~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/core/dev.c

Version: ~ [ linux-4.20-rc3 ] ~ [ linux-4.19.2 ] ~ [ linux-4.18.19 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.81 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.137 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.163 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.125 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.60 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.31.14 ] ~ [ linux-2.6.30.10 ] ~ [ linux-2.6.29.6 ] ~ [ linux-2.6.28.10 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *      NET3    Protocol independent device support routines.
  3  *
  4  *              This program is free software; you can redistribute it and/or
  5  *              modify it under the terms of the GNU General Public License
  6  *              as published by the Free Software Foundation; either version
  7  *              2 of the License, or (at your option) any later version.
  8  *
  9  *      Derived from the non IP parts of dev.c 1.0.19
 10  *              Authors:        Ross Biro
 11  *                              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 12  *                              Mark Evans, <evansmp@uhura.aston.ac.uk>
 13  *
 14  *      Additional Authors:
 15  *              Florian la Roche <rzsfl@rz.uni-sb.de>
 16  *              Alan Cox <gw4pts@gw4pts.ampr.org>
 17  *              David Hinds <dahinds@users.sourceforge.net>
 18  *              Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
 19  *              Adam Sulmicki <adam@cfar.umd.edu>
 20  *              Pekka Riikonen <priikone@poesidon.pspt.fi>
 21  *
 22  *      Changes:
 23  *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
 24  *                                      to 2 if register_netdev gets called
 25  *                                      before net_dev_init & also removed a
 26  *                                      few lines of code in the process.
 27  *              Alan Cox        :       device private ioctl copies fields back.
 28  *              Alan Cox        :       Transmit queue code does relevant
 29  *                                      stunts to keep the queue safe.
 30  *              Alan Cox        :       Fixed double lock.
 31  *              Alan Cox        :       Fixed promisc NULL pointer trap
 32  *              ????????        :       Support the full private ioctl range
 33  *              Alan Cox        :       Moved ioctl permission check into
 34  *                                      drivers
 35  *              Tim Kordas      :       SIOCADDMULTI/SIOCDELMULTI
 36  *              Alan Cox        :       100 backlog just doesn't cut it when
 37  *                                      you start doing multicast video 8)
 38  *              Alan Cox        :       Rewrote net_bh and list manager.
 39  *              Alan Cox        :       Fix ETH_P_ALL echoback lengths.
 40  *              Alan Cox        :       Took out transmit every packet pass
 41  *                                      Saved a few bytes in the ioctl handler
 42  *              Alan Cox        :       Network driver sets packet type before
 43  *                                      calling netif_rx. Saves a function
 44  *                                      call a packet.
 45  *              Alan Cox        :       Hashed net_bh()
 46  *              Richard Kooijman:       Timestamp fixes.
 47  *              Alan Cox        :       Wrong field in SIOCGIFDSTADDR
 48  *              Alan Cox        :       Device lock protection.
 49  *              Alan Cox        :       Fixed nasty side effect of device close
 50  *                                      changes.
 51  *              Rudi Cilibrasi  :       Pass the right thing to
 52  *                                      set_mac_address()
 53  *              Dave Miller     :       32bit quantity for the device lock to
 54  *                                      make it work out on a Sparc.
 55  *              Bjorn Ekwall    :       Added KERNELD hack.
 56  *              Alan Cox        :       Cleaned up the backlog initialise.
 57  *              Craig Metz      :       SIOCGIFCONF fix if space for under
 58  *                                      1 device.
 59  *          Thomas Bogendoerfer :       Return ENODEV for dev_open, if there
 60  *                                      is no device open function.
 61  *              Andi Kleen      :       Fix error reporting for SIOCGIFCONF
 62  *          Michael Chastain    :       Fix signed/unsigned for SIOCGIFCONF
 63  *              Cyrus Durgin    :       Cleaned for KMOD
 64  *              Adam Sulmicki   :       Bug Fix : Network Device Unload
 65  *                                      A network device unload needs to purge
 66  *                                      the backlog queue.
 67  *      Paul Rusty Russell      :       SIOCSIFNAME
 68  *              Pekka Riikonen  :       Netdev boot-time settings code
 69  *              Andrew Morton   :       Make unregister_netdevice wait
 70  *                                      indefinitely on dev->refcnt
 71  *              J Hadi Salim    :       - Backlog queue sampling
 72  *                                      - netif_rx() feedback
 73  */
 74 
 75 #include <linux/uaccess.h>
 76 #include <linux/bitops.h>
 77 #include <linux/capability.h>
 78 #include <linux/cpu.h>
 79 #include <linux/types.h>
 80 #include <linux/kernel.h>
 81 #include <linux/hash.h>
 82 #include <linux/slab.h>
 83 #include <linux/sched.h>
 84 #include <linux/sched/mm.h>
 85 #include <linux/mutex.h>
 86 #include <linux/string.h>
 87 #include <linux/mm.h>
 88 #include <linux/socket.h>
 89 #include <linux/sockios.h>
 90 #include <linux/errno.h>
 91 #include <linux/interrupt.h>
 92 #include <linux/if_ether.h>
 93 #include <linux/netdevice.h>
 94 #include <linux/etherdevice.h>
 95 #include <linux/ethtool.h>
 96 #include <linux/skbuff.h>
 97 #include <linux/bpf.h>
 98 #include <linux/bpf_trace.h>
 99 #include <net/net_namespace.h>
100 #include <net/sock.h>
101 #include <net/busy_poll.h>
102 #include <linux/rtnetlink.h>
103 #include <linux/stat.h>
104 #include <net/dst.h>
105 #include <net/dst_metadata.h>
106 #include <net/pkt_sched.h>
107 #include <net/pkt_cls.h>
108 #include <net/checksum.h>
109 #include <net/xfrm.h>
110 #include <linux/highmem.h>
111 #include <linux/init.h>
112 #include <linux/module.h>
113 #include <linux/netpoll.h>
114 #include <linux/rcupdate.h>
115 #include <linux/delay.h>
116 #include <net/iw_handler.h>
117 #include <asm/current.h>
118 #include <linux/audit.h>
119 #include <linux/dmaengine.h>
120 #include <linux/err.h>
121 #include <linux/ctype.h>
122 #include <linux/if_arp.h>
123 #include <linux/if_vlan.h>
124 #include <linux/ip.h>
125 #include <net/ip.h>
126 #include <net/mpls.h>
127 #include <linux/ipv6.h>
128 #include <linux/in.h>
129 #include <linux/jhash.h>
130 #include <linux/random.h>
131 #include <trace/events/napi.h>
132 #include <trace/events/net.h>
133 #include <trace/events/skb.h>
134 #include <linux/pci.h>
135 #include <linux/inetdevice.h>
136 #include <linux/cpu_rmap.h>
137 #include <linux/static_key.h>
138 #include <linux/hashtable.h>
139 #include <linux/vmalloc.h>
140 #include <linux/if_macvlan.h>
141 #include <linux/errqueue.h>
142 #include <linux/hrtimer.h>
143 #include <linux/netfilter_ingress.h>
144 #include <linux/crash_dump.h>
145 #include <linux/sctp.h>
146 #include <net/udp_tunnel.h>
147 #include <linux/net_namespace.h>
148 
149 #include "net-sysfs.h"
150 
151 #define MAX_GRO_SKBS 8
152 
153 /* This should be increased if a protocol with a bigger head is added. */
154 #define GRO_MAX_HEAD (MAX_HEADER + 128)
155 
156 static DEFINE_SPINLOCK(ptype_lock);
157 static DEFINE_SPINLOCK(offload_lock);
158 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
159 struct list_head ptype_all __read_mostly;       /* Taps */
160 static struct list_head offload_base __read_mostly;
161 
162 static int netif_rx_internal(struct sk_buff *skb);
163 static int call_netdevice_notifiers_info(unsigned long val,
164                                          struct netdev_notifier_info *info);
165 static struct napi_struct *napi_by_id(unsigned int napi_id);
166 
167 /*
168  * The @dev_base_head list is protected by @dev_base_lock and the rtnl
169  * semaphore.
170  *
171  * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
172  *
173  * Writers must hold the rtnl semaphore while they loop through the
174  * dev_base_head list, and hold dev_base_lock for writing when they do the
175  * actual updates.  This allows pure readers to access the list even
176  * while a writer is preparing to update it.
177  *
178  * To put it another way, dev_base_lock is held for writing only to
179  * protect against pure readers; the rtnl semaphore provides the
180  * protection against other writers.
181  *
182  * See, for example usages, register_netdevice() and
183  * unregister_netdevice(), which must be called with the rtnl
184  * semaphore held.
185  */
186 DEFINE_RWLOCK(dev_base_lock);
187 EXPORT_SYMBOL(dev_base_lock);
188 
189 static DEFINE_MUTEX(ifalias_mutex);
190 
191 /* protects napi_hash addition/deletion and napi_gen_id */
192 static DEFINE_SPINLOCK(napi_hash_lock);
193 
194 static unsigned int napi_gen_id = NR_CPUS;
195 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
196 
197 static seqcount_t devnet_rename_seq;
198 
199 static inline void dev_base_seq_inc(struct net *net)
200 {
201         while (++net->dev_base_seq == 0)
202                 ;
203 }
204 
205 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
206 {
207         unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
208 
209         return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
210 }
211 
212 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
213 {
214         return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
215 }
216 
217 static inline void rps_lock(struct softnet_data *sd)
218 {
219 #ifdef CONFIG_RPS
220         spin_lock(&sd->input_pkt_queue.lock);
221 #endif
222 }
223 
224 static inline void rps_unlock(struct softnet_data *sd)
225 {
226 #ifdef CONFIG_RPS
227         spin_unlock(&sd->input_pkt_queue.lock);
228 #endif
229 }
230 
231 /* Device list insertion */
232 static void list_netdevice(struct net_device *dev)
233 {
234         struct net *net = dev_net(dev);
235 
236         ASSERT_RTNL();
237 
238         write_lock_bh(&dev_base_lock);
239         list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
240         hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
241         hlist_add_head_rcu(&dev->index_hlist,
242                            dev_index_hash(net, dev->ifindex));
243         write_unlock_bh(&dev_base_lock);
244 
245         dev_base_seq_inc(net);
246 }
247 
248 /* Device list removal
249  * caller must respect a RCU grace period before freeing/reusing dev
250  */
251 static void unlist_netdevice(struct net_device *dev)
252 {
253         ASSERT_RTNL();
254 
255         /* Unlink dev from the device chain */
256         write_lock_bh(&dev_base_lock);
257         list_del_rcu(&dev->dev_list);
258         hlist_del_rcu(&dev->name_hlist);
259         hlist_del_rcu(&dev->index_hlist);
260         write_unlock_bh(&dev_base_lock);
261 
262         dev_base_seq_inc(dev_net(dev));
263 }
264 
265 /*
266  *      Our notifier list
267  */
268 
269 static RAW_NOTIFIER_HEAD(netdev_chain);
270 
271 /*
272  *      Device drivers call our routines to queue packets here. We empty the
273  *      queue in the local softnet handler.
274  */
275 
276 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
277 EXPORT_PER_CPU_SYMBOL(softnet_data);
278 
279 #ifdef CONFIG_LOCKDEP
280 /*
281  * register_netdevice() inits txq->_xmit_lock and sets lockdep class
282  * according to dev->type
283  */
284 static const unsigned short netdev_lock_type[] = {
285          ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
286          ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
287          ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
288          ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
289          ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
290          ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
291          ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
292          ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
293          ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
294          ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
295          ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
296          ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
297          ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
298          ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
299          ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
300 
301 static const char *const netdev_lock_name[] = {
302         "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
303         "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
304         "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
305         "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
306         "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
307         "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
308         "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
309         "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
310         "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
311         "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
312         "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
313         "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
314         "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
315         "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
316         "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
317 
318 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
319 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
320 
321 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
322 {
323         int i;
324 
325         for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
326                 if (netdev_lock_type[i] == dev_type)
327                         return i;
328         /* the last key is used by default */
329         return ARRAY_SIZE(netdev_lock_type) - 1;
330 }
331 
332 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
333                                                  unsigned short dev_type)
334 {
335         int i;
336 
337         i = netdev_lock_pos(dev_type);
338         lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
339                                    netdev_lock_name[i]);
340 }
341 
342 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
343 {
344         int i;
345 
346         i = netdev_lock_pos(dev->type);
347         lockdep_set_class_and_name(&dev->addr_list_lock,
348                                    &netdev_addr_lock_key[i],
349                                    netdev_lock_name[i]);
350 }
351 #else
352 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
353                                                  unsigned short dev_type)
354 {
355 }
356 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
357 {
358 }
359 #endif
360 
361 /*******************************************************************************
362  *
363  *              Protocol management and registration routines
364  *
365  *******************************************************************************/
366 
367 
368 /*
369  *      Add a protocol ID to the list. Now that the input handler is
370  *      smarter we can dispense with all the messy stuff that used to be
371  *      here.
372  *
373  *      BEWARE!!! Protocol handlers, mangling input packets,
374  *      MUST BE last in hash buckets and checking protocol handlers
375  *      MUST start from promiscuous ptype_all chain in net_bh.
376  *      It is true now, do not change it.
377  *      Explanation follows: if protocol handler, mangling packet, will
378  *      be the first on list, it is not able to sense, that packet
379  *      is cloned and should be copied-on-write, so that it will
380  *      change it and subsequent readers will get broken packet.
381  *                                                      --ANK (980803)
382  */
383 
384 static inline struct list_head *ptype_head(const struct packet_type *pt)
385 {
386         if (pt->type == htons(ETH_P_ALL))
387                 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
388         else
389                 return pt->dev ? &pt->dev->ptype_specific :
390                                  &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
391 }
392 
393 /**
394  *      dev_add_pack - add packet handler
395  *      @pt: packet type declaration
396  *
397  *      Add a protocol handler to the networking stack. The passed &packet_type
398  *      is linked into kernel lists and may not be freed until it has been
399  *      removed from the kernel lists.
400  *
401  *      This call does not sleep therefore it can not
402  *      guarantee all CPU's that are in middle of receiving packets
403  *      will see the new packet type (until the next received packet).
404  */
405 
406 void dev_add_pack(struct packet_type *pt)
407 {
408         struct list_head *head = ptype_head(pt);
409 
410         spin_lock(&ptype_lock);
411         list_add_rcu(&pt->list, head);
412         spin_unlock(&ptype_lock);
413 }
414 EXPORT_SYMBOL(dev_add_pack);
415 
416 /**
417  *      __dev_remove_pack        - remove packet handler
418  *      @pt: packet type declaration
419  *
420  *      Remove a protocol handler that was previously added to the kernel
421  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
422  *      from the kernel lists and can be freed or reused once this function
423  *      returns.
424  *
425  *      The packet type might still be in use by receivers
426  *      and must not be freed until after all the CPU's have gone
427  *      through a quiescent state.
428  */
429 void __dev_remove_pack(struct packet_type *pt)
430 {
431         struct list_head *head = ptype_head(pt);
432         struct packet_type *pt1;
433 
434         spin_lock(&ptype_lock);
435 
436         list_for_each_entry(pt1, head, list) {
437                 if (pt == pt1) {
438                         list_del_rcu(&pt->list);
439                         goto out;
440                 }
441         }
442 
443         pr_warn("dev_remove_pack: %p not found\n", pt);
444 out:
445         spin_unlock(&ptype_lock);
446 }
447 EXPORT_SYMBOL(__dev_remove_pack);
448 
449 /**
450  *      dev_remove_pack  - remove packet handler
451  *      @pt: packet type declaration
452  *
453  *      Remove a protocol handler that was previously added to the kernel
454  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
455  *      from the kernel lists and can be freed or reused once this function
456  *      returns.
457  *
458  *      This call sleeps to guarantee that no CPU is looking at the packet
459  *      type after return.
460  */
461 void dev_remove_pack(struct packet_type *pt)
462 {
463         __dev_remove_pack(pt);
464 
465         synchronize_net();
466 }
467 EXPORT_SYMBOL(dev_remove_pack);
468 
469 
470 /**
471  *      dev_add_offload - register offload handlers
472  *      @po: protocol offload declaration
473  *
474  *      Add protocol offload handlers to the networking stack. The passed
475  *      &proto_offload is linked into kernel lists and may not be freed until
476  *      it has been removed from the kernel lists.
477  *
478  *      This call does not sleep therefore it can not
479  *      guarantee all CPU's that are in middle of receiving packets
480  *      will see the new offload handlers (until the next received packet).
481  */
482 void dev_add_offload(struct packet_offload *po)
483 {
484         struct packet_offload *elem;
485 
486         spin_lock(&offload_lock);
487         list_for_each_entry(elem, &offload_base, list) {
488                 if (po->priority < elem->priority)
489                         break;
490         }
491         list_add_rcu(&po->list, elem->list.prev);
492         spin_unlock(&offload_lock);
493 }
494 EXPORT_SYMBOL(dev_add_offload);
495 
496 /**
497  *      __dev_remove_offload     - remove offload handler
498  *      @po: packet offload declaration
499  *
500  *      Remove a protocol offload handler that was previously added to the
501  *      kernel offload handlers by dev_add_offload(). The passed &offload_type
502  *      is removed from the kernel lists and can be freed or reused once this
503  *      function returns.
504  *
505  *      The packet type might still be in use by receivers
506  *      and must not be freed until after all the CPU's have gone
507  *      through a quiescent state.
508  */
509 static void __dev_remove_offload(struct packet_offload *po)
510 {
511         struct list_head *head = &offload_base;
512         struct packet_offload *po1;
513 
514         spin_lock(&offload_lock);
515 
516         list_for_each_entry(po1, head, list) {
517                 if (po == po1) {
518                         list_del_rcu(&po->list);
519                         goto out;
520                 }
521         }
522 
523         pr_warn("dev_remove_offload: %p not found\n", po);
524 out:
525         spin_unlock(&offload_lock);
526 }
527 
528 /**
529  *      dev_remove_offload       - remove packet offload handler
530  *      @po: packet offload declaration
531  *
532  *      Remove a packet offload handler that was previously added to the kernel
533  *      offload handlers by dev_add_offload(). The passed &offload_type is
534  *      removed from the kernel lists and can be freed or reused once this
535  *      function returns.
536  *
537  *      This call sleeps to guarantee that no CPU is looking at the packet
538  *      type after return.
539  */
540 void dev_remove_offload(struct packet_offload *po)
541 {
542         __dev_remove_offload(po);
543 
544         synchronize_net();
545 }
546 EXPORT_SYMBOL(dev_remove_offload);
547 
548 /******************************************************************************
549  *
550  *                    Device Boot-time Settings Routines
551  *
552  ******************************************************************************/
553 
554 /* Boot time configuration table */
555 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
556 
557 /**
558  *      netdev_boot_setup_add   - add new setup entry
559  *      @name: name of the device
560  *      @map: configured settings for the device
561  *
562  *      Adds new setup entry to the dev_boot_setup list.  The function
563  *      returns 0 on error and 1 on success.  This is a generic routine to
564  *      all netdevices.
565  */
566 static int netdev_boot_setup_add(char *name, struct ifmap *map)
567 {
568         struct netdev_boot_setup *s;
569         int i;
570 
571         s = dev_boot_setup;
572         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
573                 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
574                         memset(s[i].name, 0, sizeof(s[i].name));
575                         strlcpy(s[i].name, name, IFNAMSIZ);
576                         memcpy(&s[i].map, map, sizeof(s[i].map));
577                         break;
578                 }
579         }
580 
581         return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
582 }
583 
584 /**
585  * netdev_boot_setup_check      - check boot time settings
586  * @dev: the netdevice
587  *
588  * Check boot time settings for the device.
589  * The found settings are set for the device to be used
590  * later in the device probing.
591  * Returns 0 if no settings found, 1 if they are.
592  */
593 int netdev_boot_setup_check(struct net_device *dev)
594 {
595         struct netdev_boot_setup *s = dev_boot_setup;
596         int i;
597 
598         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
599                 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
600                     !strcmp(dev->name, s[i].name)) {
601                         dev->irq = s[i].map.irq;
602                         dev->base_addr = s[i].map.base_addr;
603                         dev->mem_start = s[i].map.mem_start;
604                         dev->mem_end = s[i].map.mem_end;
605                         return 1;
606                 }
607         }
608         return 0;
609 }
610 EXPORT_SYMBOL(netdev_boot_setup_check);
611 
612 
613 /**
614  * netdev_boot_base     - get address from boot time settings
615  * @prefix: prefix for network device
616  * @unit: id for network device
617  *
618  * Check boot time settings for the base address of device.
619  * The found settings are set for the device to be used
620  * later in the device probing.
621  * Returns 0 if no settings found.
622  */
623 unsigned long netdev_boot_base(const char *prefix, int unit)
624 {
625         const struct netdev_boot_setup *s = dev_boot_setup;
626         char name[IFNAMSIZ];
627         int i;
628 
629         sprintf(name, "%s%d", prefix, unit);
630 
631         /*
632          * If device already registered then return base of 1
633          * to indicate not to probe for this interface
634          */
635         if (__dev_get_by_name(&init_net, name))
636                 return 1;
637 
638         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
639                 if (!strcmp(name, s[i].name))
640                         return s[i].map.base_addr;
641         return 0;
642 }
643 
644 /*
645  * Saves at boot time configured settings for any netdevice.
646  */
647 int __init netdev_boot_setup(char *str)
648 {
649         int ints[5];
650         struct ifmap map;
651 
652         str = get_options(str, ARRAY_SIZE(ints), ints);
653         if (!str || !*str)
654                 return 0;
655 
656         /* Save settings */
657         memset(&map, 0, sizeof(map));
658         if (ints[0] > 0)
659                 map.irq = ints[1];
660         if (ints[0] > 1)
661                 map.base_addr = ints[2];
662         if (ints[0] > 2)
663                 map.mem_start = ints[3];
664         if (ints[0] > 3)
665                 map.mem_end = ints[4];
666 
667         /* Add new entry to the list */
668         return netdev_boot_setup_add(str, &map);
669 }
670 
671 __setup("netdev=", netdev_boot_setup);
672 
673 /*******************************************************************************
674  *
675  *                          Device Interface Subroutines
676  *
677  *******************************************************************************/
678 
679 /**
680  *      dev_get_iflink  - get 'iflink' value of a interface
681  *      @dev: targeted interface
682  *
683  *      Indicates the ifindex the interface is linked to.
684  *      Physical interfaces have the same 'ifindex' and 'iflink' values.
685  */
686 
687 int dev_get_iflink(const struct net_device *dev)
688 {
689         if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
690                 return dev->netdev_ops->ndo_get_iflink(dev);
691 
692         return dev->ifindex;
693 }
694 EXPORT_SYMBOL(dev_get_iflink);
695 
696 /**
697  *      dev_fill_metadata_dst - Retrieve tunnel egress information.
698  *      @dev: targeted interface
699  *      @skb: The packet.
700  *
701  *      For better visibility of tunnel traffic OVS needs to retrieve
702  *      egress tunnel information for a packet. Following API allows
703  *      user to get this info.
704  */
705 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
706 {
707         struct ip_tunnel_info *info;
708 
709         if (!dev->netdev_ops  || !dev->netdev_ops->ndo_fill_metadata_dst)
710                 return -EINVAL;
711 
712         info = skb_tunnel_info_unclone(skb);
713         if (!info)
714                 return -ENOMEM;
715         if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
716                 return -EINVAL;
717 
718         return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
719 }
720 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
721 
722 /**
723  *      __dev_get_by_name       - find a device by its name
724  *      @net: the applicable net namespace
725  *      @name: name to find
726  *
727  *      Find an interface by name. Must be called under RTNL semaphore
728  *      or @dev_base_lock. If the name is found a pointer to the device
729  *      is returned. If the name is not found then %NULL is returned. The
730  *      reference counters are not incremented so the caller must be
731  *      careful with locks.
732  */
733 
734 struct net_device *__dev_get_by_name(struct net *net, const char *name)
735 {
736         struct net_device *dev;
737         struct hlist_head *head = dev_name_hash(net, name);
738 
739         hlist_for_each_entry(dev, head, name_hlist)
740                 if (!strncmp(dev->name, name, IFNAMSIZ))
741                         return dev;
742 
743         return NULL;
744 }
745 EXPORT_SYMBOL(__dev_get_by_name);
746 
747 /**
748  * dev_get_by_name_rcu  - find a device by its name
749  * @net: the applicable net namespace
750  * @name: name to find
751  *
752  * Find an interface by name.
753  * If the name is found a pointer to the device is returned.
754  * If the name is not found then %NULL is returned.
755  * The reference counters are not incremented so the caller must be
756  * careful with locks. The caller must hold RCU lock.
757  */
758 
759 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
760 {
761         struct net_device *dev;
762         struct hlist_head *head = dev_name_hash(net, name);
763 
764         hlist_for_each_entry_rcu(dev, head, name_hlist)
765                 if (!strncmp(dev->name, name, IFNAMSIZ))
766                         return dev;
767 
768         return NULL;
769 }
770 EXPORT_SYMBOL(dev_get_by_name_rcu);
771 
772 /**
773  *      dev_get_by_name         - find a device by its name
774  *      @net: the applicable net namespace
775  *      @name: name to find
776  *
777  *      Find an interface by name. This can be called from any
778  *      context and does its own locking. The returned handle has
779  *      the usage count incremented and the caller must use dev_put() to
780  *      release it when it is no longer needed. %NULL is returned if no
781  *      matching device is found.
782  */
783 
784 struct net_device *dev_get_by_name(struct net *net, const char *name)
785 {
786         struct net_device *dev;
787 
788         rcu_read_lock();
789         dev = dev_get_by_name_rcu(net, name);
790         if (dev)
791                 dev_hold(dev);
792         rcu_read_unlock();
793         return dev;
794 }
795 EXPORT_SYMBOL(dev_get_by_name);
796 
797 /**
798  *      __dev_get_by_index - find a device by its ifindex
799  *      @net: the applicable net namespace
800  *      @ifindex: index of device
801  *
802  *      Search for an interface by index. Returns %NULL if the device
803  *      is not found or a pointer to the device. The device has not
804  *      had its reference counter increased so the caller must be careful
805  *      about locking. The caller must hold either the RTNL semaphore
806  *      or @dev_base_lock.
807  */
808 
809 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
810 {
811         struct net_device *dev;
812         struct hlist_head *head = dev_index_hash(net, ifindex);
813 
814         hlist_for_each_entry(dev, head, index_hlist)
815                 if (dev->ifindex == ifindex)
816                         return dev;
817 
818         return NULL;
819 }
820 EXPORT_SYMBOL(__dev_get_by_index);
821 
822 /**
823  *      dev_get_by_index_rcu - find a device by its ifindex
824  *      @net: the applicable net namespace
825  *      @ifindex: index of device
826  *
827  *      Search for an interface by index. Returns %NULL if the device
828  *      is not found or a pointer to the device. The device has not
829  *      had its reference counter increased so the caller must be careful
830  *      about locking. The caller must hold RCU lock.
831  */
832 
833 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
834 {
835         struct net_device *dev;
836         struct hlist_head *head = dev_index_hash(net, ifindex);
837 
838         hlist_for_each_entry_rcu(dev, head, index_hlist)
839                 if (dev->ifindex == ifindex)
840                         return dev;
841 
842         return NULL;
843 }
844 EXPORT_SYMBOL(dev_get_by_index_rcu);
845 
846 
847 /**
848  *      dev_get_by_index - find a device by its ifindex
849  *      @net: the applicable net namespace
850  *      @ifindex: index of device
851  *
852  *      Search for an interface by index. Returns NULL if the device
853  *      is not found or a pointer to the device. The device returned has
854  *      had a reference added and the pointer is safe until the user calls
855  *      dev_put to indicate they have finished with it.
856  */
857 
858 struct net_device *dev_get_by_index(struct net *net, int ifindex)
859 {
860         struct net_device *dev;
861 
862         rcu_read_lock();
863         dev = dev_get_by_index_rcu(net, ifindex);
864         if (dev)
865                 dev_hold(dev);
866         rcu_read_unlock();
867         return dev;
868 }
869 EXPORT_SYMBOL(dev_get_by_index);
870 
871 /**
872  *      dev_get_by_napi_id - find a device by napi_id
873  *      @napi_id: ID of the NAPI struct
874  *
875  *      Search for an interface by NAPI ID. Returns %NULL if the device
876  *      is not found or a pointer to the device. The device has not had
877  *      its reference counter increased so the caller must be careful
878  *      about locking. The caller must hold RCU lock.
879  */
880 
881 struct net_device *dev_get_by_napi_id(unsigned int napi_id)
882 {
883         struct napi_struct *napi;
884 
885         WARN_ON_ONCE(!rcu_read_lock_held());
886 
887         if (napi_id < MIN_NAPI_ID)
888                 return NULL;
889 
890         napi = napi_by_id(napi_id);
891 
892         return napi ? napi->dev : NULL;
893 }
894 EXPORT_SYMBOL(dev_get_by_napi_id);
895 
896 /**
897  *      netdev_get_name - get a netdevice name, knowing its ifindex.
898  *      @net: network namespace
899  *      @name: a pointer to the buffer where the name will be stored.
900  *      @ifindex: the ifindex of the interface to get the name from.
901  *
902  *      The use of raw_seqcount_begin() and cond_resched() before
903  *      retrying is required as we want to give the writers a chance
904  *      to complete when CONFIG_PREEMPT is not set.
905  */
906 int netdev_get_name(struct net *net, char *name, int ifindex)
907 {
908         struct net_device *dev;
909         unsigned int seq;
910 
911 retry:
912         seq = raw_seqcount_begin(&devnet_rename_seq);
913         rcu_read_lock();
914         dev = dev_get_by_index_rcu(net, ifindex);
915         if (!dev) {
916                 rcu_read_unlock();
917                 return -ENODEV;
918         }
919 
920         strcpy(name, dev->name);
921         rcu_read_unlock();
922         if (read_seqcount_retry(&devnet_rename_seq, seq)) {
923                 cond_resched();
924                 goto retry;
925         }
926 
927         return 0;
928 }
929 
930 /**
931  *      dev_getbyhwaddr_rcu - find a device by its hardware address
932  *      @net: the applicable net namespace
933  *      @type: media type of device
934  *      @ha: hardware address
935  *
936  *      Search for an interface by MAC address. Returns NULL if the device
937  *      is not found or a pointer to the device.
938  *      The caller must hold RCU or RTNL.
939  *      The returned device has not had its ref count increased
940  *      and the caller must therefore be careful about locking
941  *
942  */
943 
944 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
945                                        const char *ha)
946 {
947         struct net_device *dev;
948 
949         for_each_netdev_rcu(net, dev)
950                 if (dev->type == type &&
951                     !memcmp(dev->dev_addr, ha, dev->addr_len))
952                         return dev;
953 
954         return NULL;
955 }
956 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
957 
958 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
959 {
960         struct net_device *dev;
961 
962         ASSERT_RTNL();
963         for_each_netdev(net, dev)
964                 if (dev->type == type)
965                         return dev;
966 
967         return NULL;
968 }
969 EXPORT_SYMBOL(__dev_getfirstbyhwtype);
970 
971 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
972 {
973         struct net_device *dev, *ret = NULL;
974 
975         rcu_read_lock();
976         for_each_netdev_rcu(net, dev)
977                 if (dev->type == type) {
978                         dev_hold(dev);
979                         ret = dev;
980                         break;
981                 }
982         rcu_read_unlock();
983         return ret;
984 }
985 EXPORT_SYMBOL(dev_getfirstbyhwtype);
986 
987 /**
988  *      __dev_get_by_flags - find any device with given flags
989  *      @net: the applicable net namespace
990  *      @if_flags: IFF_* values
991  *      @mask: bitmask of bits in if_flags to check
992  *
993  *      Search for any interface with the given flags. Returns NULL if a device
994  *      is not found or a pointer to the device. Must be called inside
995  *      rtnl_lock(), and result refcount is unchanged.
996  */
997 
998 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
999                                       unsigned short mask)
1000 {
1001         struct net_device *dev, *ret;
1002 
1003         ASSERT_RTNL();
1004 
1005         ret = NULL;
1006         for_each_netdev(net, dev) {
1007                 if (((dev->flags ^ if_flags) & mask) == 0) {
1008                         ret = dev;
1009                         break;
1010                 }
1011         }
1012         return ret;
1013 }
1014 EXPORT_SYMBOL(__dev_get_by_flags);
1015 
1016 /**
1017  *      dev_valid_name - check if name is okay for network device
1018  *      @name: name string
1019  *
1020  *      Network device names need to be valid file names to
1021  *      to allow sysfs to work.  We also disallow any kind of
1022  *      whitespace.
1023  */
1024 bool dev_valid_name(const char *name)
1025 {
1026         if (*name == '\0')
1027                 return false;
1028         if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
1029                 return false;
1030         if (!strcmp(name, ".") || !strcmp(name, ".."))
1031                 return false;
1032 
1033         while (*name) {
1034                 if (*name == '/' || *name == ':' || isspace(*name))
1035                         return false;
1036                 name++;
1037         }
1038         return true;
1039 }
1040 EXPORT_SYMBOL(dev_valid_name);
1041 
1042 /**
1043  *      __dev_alloc_name - allocate a name for a device
1044  *      @net: network namespace to allocate the device name in
1045  *      @name: name format string
1046  *      @buf:  scratch buffer and result name string
1047  *
1048  *      Passed a format string - eg "lt%d" it will try and find a suitable
1049  *      id. It scans list of devices to build up a free map, then chooses
1050  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1051  *      while allocating the name and adding the device in order to avoid
1052  *      duplicates.
1053  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1054  *      Returns the number of the unit assigned or a negative errno code.
1055  */
1056 
1057 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1058 {
1059         int i = 0;
1060         const char *p;
1061         const int max_netdevices = 8*PAGE_SIZE;
1062         unsigned long *inuse;
1063         struct net_device *d;
1064 
1065         if (!dev_valid_name(name))
1066                 return -EINVAL;
1067 
1068         p = strchr(name, '%');
1069         if (p) {
1070                 /*
1071                  * Verify the string as this thing may have come from
1072                  * the user.  There must be either one "%d" and no other "%"
1073                  * characters.
1074                  */
1075                 if (p[1] != 'd' || strchr(p + 2, '%'))
1076                         return -EINVAL;
1077 
1078                 /* Use one page as a bit array of possible slots */
1079                 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1080                 if (!inuse)
1081                         return -ENOMEM;
1082 
1083                 for_each_netdev(net, d) {
1084                         if (!sscanf(d->name, name, &i))
1085                                 continue;
1086                         if (i < 0 || i >= max_netdevices)
1087                                 continue;
1088 
1089                         /*  avoid cases where sscanf is not exact inverse of printf */
1090                         snprintf(buf, IFNAMSIZ, name, i);
1091                         if (!strncmp(buf, d->name, IFNAMSIZ))
1092                                 set_bit(i, inuse);
1093                 }
1094 
1095                 i = find_first_zero_bit(inuse, max_netdevices);
1096                 free_page((unsigned long) inuse);
1097         }
1098 
1099         snprintf(buf, IFNAMSIZ, name, i);
1100         if (!__dev_get_by_name(net, buf))
1101                 return i;
1102 
1103         /* It is possible to run out of possible slots
1104          * when the name is long and there isn't enough space left
1105          * for the digits, or if all bits are used.
1106          */
1107         return -ENFILE;
1108 }
1109 
1110 static int dev_alloc_name_ns(struct net *net,
1111                              struct net_device *dev,
1112                              const char *name)
1113 {
1114         char buf[IFNAMSIZ];
1115         int ret;
1116 
1117         BUG_ON(!net);
1118         ret = __dev_alloc_name(net, name, buf);
1119         if (ret >= 0)
1120                 strlcpy(dev->name, buf, IFNAMSIZ);
1121         return ret;
1122 }
1123 
1124 /**
1125  *      dev_alloc_name - allocate a name for a device
1126  *      @dev: device
1127  *      @name: name format string
1128  *
1129  *      Passed a format string - eg "lt%d" it will try and find a suitable
1130  *      id. It scans list of devices to build up a free map, then chooses
1131  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1132  *      while allocating the name and adding the device in order to avoid
1133  *      duplicates.
1134  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1135  *      Returns the number of the unit assigned or a negative errno code.
1136  */
1137 
1138 int dev_alloc_name(struct net_device *dev, const char *name)
1139 {
1140         return dev_alloc_name_ns(dev_net(dev), dev, name);
1141 }
1142 EXPORT_SYMBOL(dev_alloc_name);
1143 
1144 int dev_get_valid_name(struct net *net, struct net_device *dev,
1145                        const char *name)
1146 {
1147         BUG_ON(!net);
1148 
1149         if (!dev_valid_name(name))
1150                 return -EINVAL;
1151 
1152         if (strchr(name, '%'))
1153                 return dev_alloc_name_ns(net, dev, name);
1154         else if (__dev_get_by_name(net, name))
1155                 return -EEXIST;
1156         else if (dev->name != name)
1157                 strlcpy(dev->name, name, IFNAMSIZ);
1158 
1159         return 0;
1160 }
1161 EXPORT_SYMBOL(dev_get_valid_name);
1162 
1163 /**
1164  *      dev_change_name - change name of a device
1165  *      @dev: device
1166  *      @newname: name (or format string) must be at least IFNAMSIZ
1167  *
1168  *      Change name of a device, can pass format strings "eth%d".
1169  *      for wildcarding.
1170  */
1171 int dev_change_name(struct net_device *dev, const char *newname)
1172 {
1173         unsigned char old_assign_type;
1174         char oldname[IFNAMSIZ];
1175         int err = 0;
1176         int ret;
1177         struct net *net;
1178 
1179         ASSERT_RTNL();
1180         BUG_ON(!dev_net(dev));
1181 
1182         net = dev_net(dev);
1183         if (dev->flags & IFF_UP)
1184                 return -EBUSY;
1185 
1186         write_seqcount_begin(&devnet_rename_seq);
1187 
1188         if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1189                 write_seqcount_end(&devnet_rename_seq);
1190                 return 0;
1191         }
1192 
1193         memcpy(oldname, dev->name, IFNAMSIZ);
1194 
1195         err = dev_get_valid_name(net, dev, newname);
1196         if (err < 0) {
1197                 write_seqcount_end(&devnet_rename_seq);
1198                 return err;
1199         }
1200 
1201         if (oldname[0] && !strchr(oldname, '%'))
1202                 netdev_info(dev, "renamed from %s\n", oldname);
1203 
1204         old_assign_type = dev->name_assign_type;
1205         dev->name_assign_type = NET_NAME_RENAMED;
1206 
1207 rollback:
1208         ret = device_rename(&dev->dev, dev->name);
1209         if (ret) {
1210                 memcpy(dev->name, oldname, IFNAMSIZ);
1211                 dev->name_assign_type = old_assign_type;
1212                 write_seqcount_end(&devnet_rename_seq);
1213                 return ret;
1214         }
1215 
1216         write_seqcount_end(&devnet_rename_seq);
1217 
1218         netdev_adjacent_rename_links(dev, oldname);
1219 
1220         write_lock_bh(&dev_base_lock);
1221         hlist_del_rcu(&dev->name_hlist);
1222         write_unlock_bh(&dev_base_lock);
1223 
1224         synchronize_rcu();
1225 
1226         write_lock_bh(&dev_base_lock);
1227         hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1228         write_unlock_bh(&dev_base_lock);
1229 
1230         ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1231         ret = notifier_to_errno(ret);
1232 
1233         if (ret) {
1234                 /* err >= 0 after dev_alloc_name() or stores the first errno */
1235                 if (err >= 0) {
1236                         err = ret;
1237                         write_seqcount_begin(&devnet_rename_seq);
1238                         memcpy(dev->name, oldname, IFNAMSIZ);
1239                         memcpy(oldname, newname, IFNAMSIZ);
1240                         dev->name_assign_type = old_assign_type;
1241                         old_assign_type = NET_NAME_RENAMED;
1242                         goto rollback;
1243                 } else {
1244                         pr_err("%s: name change rollback failed: %d\n",
1245                                dev->name, ret);
1246                 }
1247         }
1248 
1249         return err;
1250 }
1251 
1252 /**
1253  *      dev_set_alias - change ifalias of a device
1254  *      @dev: device
1255  *      @alias: name up to IFALIASZ
1256  *      @len: limit of bytes to copy from info
1257  *
1258  *      Set ifalias for a device,
1259  */
1260 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1261 {
1262         struct dev_ifalias *new_alias = NULL;
1263 
1264         if (len >= IFALIASZ)
1265                 return -EINVAL;
1266 
1267         if (len) {
1268                 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1269                 if (!new_alias)
1270                         return -ENOMEM;
1271 
1272                 memcpy(new_alias->ifalias, alias, len);
1273                 new_alias->ifalias[len] = 0;
1274         }
1275 
1276         mutex_lock(&ifalias_mutex);
1277         rcu_swap_protected(dev->ifalias, new_alias,
1278                            mutex_is_locked(&ifalias_mutex));
1279         mutex_unlock(&ifalias_mutex);
1280 
1281         if (new_alias)
1282                 kfree_rcu(new_alias, rcuhead);
1283 
1284         return len;
1285 }
1286 EXPORT_SYMBOL(dev_set_alias);
1287 
1288 /**
1289  *      dev_get_alias - get ifalias of a device
1290  *      @dev: device
1291  *      @name: buffer to store name of ifalias
1292  *      @len: size of buffer
1293  *
1294  *      get ifalias for a device.  Caller must make sure dev cannot go
1295  *      away,  e.g. rcu read lock or own a reference count to device.
1296  */
1297 int dev_get_alias(const struct net_device *dev, char *name, size_t len)
1298 {
1299         const struct dev_ifalias *alias;
1300         int ret = 0;
1301 
1302         rcu_read_lock();
1303         alias = rcu_dereference(dev->ifalias);
1304         if (alias)
1305                 ret = snprintf(name, len, "%s", alias->ifalias);
1306         rcu_read_unlock();
1307 
1308         return ret;
1309 }
1310 
1311 /**
1312  *      netdev_features_change - device changes features
1313  *      @dev: device to cause notification
1314  *
1315  *      Called to indicate a device has changed features.
1316  */
1317 void netdev_features_change(struct net_device *dev)
1318 {
1319         call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1320 }
1321 EXPORT_SYMBOL(netdev_features_change);
1322 
1323 /**
1324  *      netdev_state_change - device changes state
1325  *      @dev: device to cause notification
1326  *
1327  *      Called to indicate a device has changed state. This function calls
1328  *      the notifier chains for netdev_chain and sends a NEWLINK message
1329  *      to the routing socket.
1330  */
1331 void netdev_state_change(struct net_device *dev)
1332 {
1333         if (dev->flags & IFF_UP) {
1334                 struct netdev_notifier_change_info change_info = {
1335                         .info.dev = dev,
1336                 };
1337 
1338                 call_netdevice_notifiers_info(NETDEV_CHANGE,
1339                                               &change_info.info);
1340                 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1341         }
1342 }
1343 EXPORT_SYMBOL(netdev_state_change);
1344 
1345 /**
1346  * netdev_notify_peers - notify network peers about existence of @dev
1347  * @dev: network device
1348  *
1349  * Generate traffic such that interested network peers are aware of
1350  * @dev, such as by generating a gratuitous ARP. This may be used when
1351  * a device wants to inform the rest of the network about some sort of
1352  * reconfiguration such as a failover event or virtual machine
1353  * migration.
1354  */
1355 void netdev_notify_peers(struct net_device *dev)
1356 {
1357         rtnl_lock();
1358         call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1359         call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1360         rtnl_unlock();
1361 }
1362 EXPORT_SYMBOL(netdev_notify_peers);
1363 
1364 static int __dev_open(struct net_device *dev)
1365 {
1366         const struct net_device_ops *ops = dev->netdev_ops;
1367         int ret;
1368 
1369         ASSERT_RTNL();
1370 
1371         if (!netif_device_present(dev))
1372                 return -ENODEV;
1373 
1374         /* Block netpoll from trying to do any rx path servicing.
1375          * If we don't do this there is a chance ndo_poll_controller
1376          * or ndo_poll may be running while we open the device
1377          */
1378         netpoll_poll_disable(dev);
1379 
1380         ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1381         ret = notifier_to_errno(ret);
1382         if (ret)
1383                 return ret;
1384 
1385         set_bit(__LINK_STATE_START, &dev->state);
1386 
1387         if (ops->ndo_validate_addr)
1388                 ret = ops->ndo_validate_addr(dev);
1389 
1390         if (!ret && ops->ndo_open)
1391                 ret = ops->ndo_open(dev);
1392 
1393         netpoll_poll_enable(dev);
1394 
1395         if (ret)
1396                 clear_bit(__LINK_STATE_START, &dev->state);
1397         else {
1398                 dev->flags |= IFF_UP;
1399                 dev_set_rx_mode(dev);
1400                 dev_activate(dev);
1401                 add_device_randomness(dev->dev_addr, dev->addr_len);
1402         }
1403 
1404         return ret;
1405 }
1406 
1407 /**
1408  *      dev_open        - prepare an interface for use.
1409  *      @dev:   device to open
1410  *
1411  *      Takes a device from down to up state. The device's private open
1412  *      function is invoked and then the multicast lists are loaded. Finally
1413  *      the device is moved into the up state and a %NETDEV_UP message is
1414  *      sent to the netdev notifier chain.
1415  *
1416  *      Calling this function on an active interface is a nop. On a failure
1417  *      a negative errno code is returned.
1418  */
1419 int dev_open(struct net_device *dev)
1420 {
1421         int ret;
1422 
1423         if (dev->flags & IFF_UP)
1424                 return 0;
1425 
1426         ret = __dev_open(dev);
1427         if (ret < 0)
1428                 return ret;
1429 
1430         rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1431         call_netdevice_notifiers(NETDEV_UP, dev);
1432 
1433         return ret;
1434 }
1435 EXPORT_SYMBOL(dev_open);
1436 
1437 static void __dev_close_many(struct list_head *head)
1438 {
1439         struct net_device *dev;
1440 
1441         ASSERT_RTNL();
1442         might_sleep();
1443 
1444         list_for_each_entry(dev, head, close_list) {
1445                 /* Temporarily disable netpoll until the interface is down */
1446                 netpoll_poll_disable(dev);
1447 
1448                 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1449 
1450                 clear_bit(__LINK_STATE_START, &dev->state);
1451 
1452                 /* Synchronize to scheduled poll. We cannot touch poll list, it
1453                  * can be even on different cpu. So just clear netif_running().
1454                  *
1455                  * dev->stop() will invoke napi_disable() on all of it's
1456                  * napi_struct instances on this device.
1457                  */
1458                 smp_mb__after_atomic(); /* Commit netif_running(). */
1459         }
1460 
1461         dev_deactivate_many(head);
1462 
1463         list_for_each_entry(dev, head, close_list) {
1464                 const struct net_device_ops *ops = dev->netdev_ops;
1465 
1466                 /*
1467                  *      Call the device specific close. This cannot fail.
1468                  *      Only if device is UP
1469                  *
1470                  *      We allow it to be called even after a DETACH hot-plug
1471                  *      event.
1472                  */
1473                 if (ops->ndo_stop)
1474                         ops->ndo_stop(dev);
1475 
1476                 dev->flags &= ~IFF_UP;
1477                 netpoll_poll_enable(dev);
1478         }
1479 }
1480 
1481 static void __dev_close(struct net_device *dev)
1482 {
1483         LIST_HEAD(single);
1484 
1485         list_add(&dev->close_list, &single);
1486         __dev_close_many(&single);
1487         list_del(&single);
1488 }
1489 
1490 void dev_close_many(struct list_head *head, bool unlink)
1491 {
1492         struct net_device *dev, *tmp;
1493 
1494         /* Remove the devices that don't need to be closed */
1495         list_for_each_entry_safe(dev, tmp, head, close_list)
1496                 if (!(dev->flags & IFF_UP))
1497                         list_del_init(&dev->close_list);
1498 
1499         __dev_close_many(head);
1500 
1501         list_for_each_entry_safe(dev, tmp, head, close_list) {
1502                 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1503                 call_netdevice_notifiers(NETDEV_DOWN, dev);
1504                 if (unlink)
1505                         list_del_init(&dev->close_list);
1506         }
1507 }
1508 EXPORT_SYMBOL(dev_close_many);
1509 
1510 /**
1511  *      dev_close - shutdown an interface.
1512  *      @dev: device to shutdown
1513  *
1514  *      This function moves an active device into down state. A
1515  *      %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1516  *      is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1517  *      chain.
1518  */
1519 void dev_close(struct net_device *dev)
1520 {
1521         if (dev->flags & IFF_UP) {
1522                 LIST_HEAD(single);
1523 
1524                 list_add(&dev->close_list, &single);
1525                 dev_close_many(&single, true);
1526                 list_del(&single);
1527         }
1528 }
1529 EXPORT_SYMBOL(dev_close);
1530 
1531 
1532 /**
1533  *      dev_disable_lro - disable Large Receive Offload on a device
1534  *      @dev: device
1535  *
1536  *      Disable Large Receive Offload (LRO) on a net device.  Must be
1537  *      called under RTNL.  This is needed if received packets may be
1538  *      forwarded to another interface.
1539  */
1540 void dev_disable_lro(struct net_device *dev)
1541 {
1542         struct net_device *lower_dev;
1543         struct list_head *iter;
1544 
1545         dev->wanted_features &= ~NETIF_F_LRO;
1546         netdev_update_features(dev);
1547 
1548         if (unlikely(dev->features & NETIF_F_LRO))
1549                 netdev_WARN(dev, "failed to disable LRO!\n");
1550 
1551         netdev_for_each_lower_dev(dev, lower_dev, iter)
1552                 dev_disable_lro(lower_dev);
1553 }
1554 EXPORT_SYMBOL(dev_disable_lro);
1555 
1556 /**
1557  *      dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1558  *      @dev: device
1559  *
1560  *      Disable HW Generic Receive Offload (GRO_HW) on a net device.  Must be
1561  *      called under RTNL.  This is needed if Generic XDP is installed on
1562  *      the device.
1563  */
1564 static void dev_disable_gro_hw(struct net_device *dev)
1565 {
1566         dev->wanted_features &= ~NETIF_F_GRO_HW;
1567         netdev_update_features(dev);
1568 
1569         if (unlikely(dev->features & NETIF_F_GRO_HW))
1570                 netdev_WARN(dev, "failed to disable GRO_HW!\n");
1571 }
1572 
1573 const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1574 {
1575 #define N(val)                                          \
1576         case NETDEV_##val:                              \
1577                 return "NETDEV_" __stringify(val);
1578         switch (cmd) {
1579         N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1580         N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1581         N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1582         N(POST_INIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN) N(CHANGEUPPER)
1583         N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA) N(BONDING_INFO)
1584         N(PRECHANGEUPPER) N(CHANGELOWERSTATE) N(UDP_TUNNEL_PUSH_INFO)
1585         N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
1586         N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1587         N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
1588         }
1589 #undef N
1590         return "UNKNOWN_NETDEV_EVENT";
1591 }
1592 EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1593 
1594 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1595                                    struct net_device *dev)
1596 {
1597         struct netdev_notifier_info info = {
1598                 .dev = dev,
1599         };
1600 
1601         return nb->notifier_call(nb, val, &info);
1602 }
1603 
1604 static int dev_boot_phase = 1;
1605 
1606 /**
1607  * register_netdevice_notifier - register a network notifier block
1608  * @nb: notifier
1609  *
1610  * Register a notifier to be called when network device events occur.
1611  * The notifier passed is linked into the kernel structures and must
1612  * not be reused until it has been unregistered. A negative errno code
1613  * is returned on a failure.
1614  *
1615  * When registered all registration and up events are replayed
1616  * to the new notifier to allow device to have a race free
1617  * view of the network device list.
1618  */
1619 
1620 int register_netdevice_notifier(struct notifier_block *nb)
1621 {
1622         struct net_device *dev;
1623         struct net_device *last;
1624         struct net *net;
1625         int err;
1626 
1627         /* Close race with setup_net() and cleanup_net() */
1628         down_write(&pernet_ops_rwsem);
1629         rtnl_lock();
1630         err = raw_notifier_chain_register(&netdev_chain, nb);
1631         if (err)
1632                 goto unlock;
1633         if (dev_boot_phase)
1634                 goto unlock;
1635         for_each_net(net) {
1636                 for_each_netdev(net, dev) {
1637                         err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1638                         err = notifier_to_errno(err);
1639                         if (err)
1640                                 goto rollback;
1641 
1642                         if (!(dev->flags & IFF_UP))
1643                                 continue;
1644 
1645                         call_netdevice_notifier(nb, NETDEV_UP, dev);
1646                 }
1647         }
1648 
1649 unlock:
1650         rtnl_unlock();
1651         up_write(&pernet_ops_rwsem);
1652         return err;
1653 
1654 rollback:
1655         last = dev;
1656         for_each_net(net) {
1657                 for_each_netdev(net, dev) {
1658                         if (dev == last)
1659                                 goto outroll;
1660 
1661                         if (dev->flags & IFF_UP) {
1662                                 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1663                                                         dev);
1664                                 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1665                         }
1666                         call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1667                 }
1668         }
1669 
1670 outroll:
1671         raw_notifier_chain_unregister(&netdev_chain, nb);
1672         goto unlock;
1673 }
1674 EXPORT_SYMBOL(register_netdevice_notifier);
1675 
1676 /**
1677  * unregister_netdevice_notifier - unregister a network notifier block
1678  * @nb: notifier
1679  *
1680  * Unregister a notifier previously registered by
1681  * register_netdevice_notifier(). The notifier is unlinked into the
1682  * kernel structures and may then be reused. A negative errno code
1683  * is returned on a failure.
1684  *
1685  * After unregistering unregister and down device events are synthesized
1686  * for all devices on the device list to the removed notifier to remove
1687  * the need for special case cleanup code.
1688  */
1689 
1690 int unregister_netdevice_notifier(struct notifier_block *nb)
1691 {
1692         struct net_device *dev;
1693         struct net *net;
1694         int err;
1695 
1696         /* Close race with setup_net() and cleanup_net() */
1697         down_write(&pernet_ops_rwsem);
1698         rtnl_lock();
1699         err = raw_notifier_chain_unregister(&netdev_chain, nb);
1700         if (err)
1701                 goto unlock;
1702 
1703         for_each_net(net) {
1704                 for_each_netdev(net, dev) {
1705                         if (dev->flags & IFF_UP) {
1706                                 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1707                                                         dev);
1708                                 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1709                         }
1710                         call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1711                 }
1712         }
1713 unlock:
1714         rtnl_unlock();
1715         up_write(&pernet_ops_rwsem);
1716         return err;
1717 }
1718 EXPORT_SYMBOL(unregister_netdevice_notifier);
1719 
1720 /**
1721  *      call_netdevice_notifiers_info - call all network notifier blocks
1722  *      @val: value passed unmodified to notifier function
1723  *      @info: notifier information data
1724  *
1725  *      Call all network notifier blocks.  Parameters and return value
1726  *      are as for raw_notifier_call_chain().
1727  */
1728 
1729 static int call_netdevice_notifiers_info(unsigned long val,
1730                                          struct netdev_notifier_info *info)
1731 {
1732         ASSERT_RTNL();
1733         return raw_notifier_call_chain(&netdev_chain, val, info);
1734 }
1735 
1736 /**
1737  *      call_netdevice_notifiers - call all network notifier blocks
1738  *      @val: value passed unmodified to notifier function
1739  *      @dev: net_device pointer passed unmodified to notifier function
1740  *
1741  *      Call all network notifier blocks.  Parameters and return value
1742  *      are as for raw_notifier_call_chain().
1743  */
1744 
1745 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1746 {
1747         struct netdev_notifier_info info = {
1748                 .dev = dev,
1749         };
1750 
1751         return call_netdevice_notifiers_info(val, &info);
1752 }
1753 EXPORT_SYMBOL(call_netdevice_notifiers);
1754 
1755 /**
1756  *      call_netdevice_notifiers_mtu - call all network notifier blocks
1757  *      @val: value passed unmodified to notifier function
1758  *      @dev: net_device pointer passed unmodified to notifier function
1759  *      @arg: additional u32 argument passed to the notifier function
1760  *
1761  *      Call all network notifier blocks.  Parameters and return value
1762  *      are as for raw_notifier_call_chain().
1763  */
1764 static int call_netdevice_notifiers_mtu(unsigned long val,
1765                                         struct net_device *dev, u32 arg)
1766 {
1767         struct netdev_notifier_info_ext info = {
1768                 .info.dev = dev,
1769                 .ext.mtu = arg,
1770         };
1771 
1772         BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
1773 
1774         return call_netdevice_notifiers_info(val, &info.info);
1775 }
1776 
1777 #ifdef CONFIG_NET_INGRESS
1778 static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
1779 
1780 void net_inc_ingress_queue(void)
1781 {
1782         static_branch_inc(&ingress_needed_key);
1783 }
1784 EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1785 
1786 void net_dec_ingress_queue(void)
1787 {
1788         static_branch_dec(&ingress_needed_key);
1789 }
1790 EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1791 #endif
1792 
1793 #ifdef CONFIG_NET_EGRESS
1794 static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
1795 
1796 void net_inc_egress_queue(void)
1797 {
1798         static_branch_inc(&egress_needed_key);
1799 }
1800 EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1801 
1802 void net_dec_egress_queue(void)
1803 {
1804         static_branch_dec(&egress_needed_key);
1805 }
1806 EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1807 #endif
1808 
1809 static DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
1810 #ifdef HAVE_JUMP_LABEL
1811 static atomic_t netstamp_needed_deferred;
1812 static atomic_t netstamp_wanted;
1813 static void netstamp_clear(struct work_struct *work)
1814 {
1815         int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1816         int wanted;
1817 
1818         wanted = atomic_add_return(deferred, &netstamp_wanted);
1819         if (wanted > 0)
1820                 static_branch_enable(&netstamp_needed_key);
1821         else
1822                 static_branch_disable(&netstamp_needed_key);
1823 }
1824 static DECLARE_WORK(netstamp_work, netstamp_clear);
1825 #endif
1826 
1827 void net_enable_timestamp(void)
1828 {
1829 #ifdef HAVE_JUMP_LABEL
1830         int wanted;
1831 
1832         while (1) {
1833                 wanted = atomic_read(&netstamp_wanted);
1834                 if (wanted <= 0)
1835                         break;
1836                 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
1837                         return;
1838         }
1839         atomic_inc(&netstamp_needed_deferred);
1840         schedule_work(&netstamp_work);
1841 #else
1842         static_branch_inc(&netstamp_needed_key);
1843 #endif
1844 }
1845 EXPORT_SYMBOL(net_enable_timestamp);
1846 
1847 void net_disable_timestamp(void)
1848 {
1849 #ifdef HAVE_JUMP_LABEL
1850         int wanted;
1851 
1852         while (1) {
1853                 wanted = atomic_read(&netstamp_wanted);
1854                 if (wanted <= 1)
1855                         break;
1856                 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
1857                         return;
1858         }
1859         atomic_dec(&netstamp_needed_deferred);
1860         schedule_work(&netstamp_work);
1861 #else
1862         static_branch_dec(&netstamp_needed_key);
1863 #endif
1864 }
1865 EXPORT_SYMBOL(net_disable_timestamp);
1866 
1867 static inline void net_timestamp_set(struct sk_buff *skb)
1868 {
1869         skb->tstamp = 0;
1870         if (static_branch_unlikely(&netstamp_needed_key))
1871                 __net_timestamp(skb);
1872 }
1873 
1874 #define net_timestamp_check(COND, SKB)                          \
1875         if (static_branch_unlikely(&netstamp_needed_key)) {     \
1876                 if ((COND) && !(SKB)->tstamp)                   \
1877                         __net_timestamp(SKB);                   \
1878         }                                                       \
1879 
1880 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
1881 {
1882         unsigned int len;
1883 
1884         if (!(dev->flags & IFF_UP))
1885                 return false;
1886 
1887         len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1888         if (skb->len <= len)
1889                 return true;
1890 
1891         /* if TSO is enabled, we don't care about the length as the packet
1892          * could be forwarded without being segmented before
1893          */
1894         if (skb_is_gso(skb))
1895                 return true;
1896 
1897         return false;
1898 }
1899 EXPORT_SYMBOL_GPL(is_skb_forwardable);
1900 
1901 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1902 {
1903         int ret = ____dev_forward_skb(dev, skb);
1904 
1905         if (likely(!ret)) {
1906                 skb->protocol = eth_type_trans(skb, dev);
1907                 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
1908         }
1909 
1910         return ret;
1911 }
1912 EXPORT_SYMBOL_GPL(__dev_forward_skb);
1913 
1914 /**
1915  * dev_forward_skb - loopback an skb to another netif
1916  *
1917  * @dev: destination network device
1918  * @skb: buffer to forward
1919  *
1920  * return values:
1921  *      NET_RX_SUCCESS  (no congestion)
1922  *      NET_RX_DROP     (packet was dropped, but freed)
1923  *
1924  * dev_forward_skb can be used for injecting an skb from the
1925  * start_xmit function of one device into the receive queue
1926  * of another device.
1927  *
1928  * The receiving device may be in another namespace, so
1929  * we have to clear all information in the skb that could
1930  * impact namespace isolation.
1931  */
1932 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1933 {
1934         return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
1935 }
1936 EXPORT_SYMBOL_GPL(dev_forward_skb);
1937 
1938 static inline int deliver_skb(struct sk_buff *skb,
1939                               struct packet_type *pt_prev,
1940                               struct net_device *orig_dev)
1941 {
1942         if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
1943                 return -ENOMEM;
1944         refcount_inc(&skb->users);
1945         return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1946 }
1947 
1948 static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1949                                           struct packet_type **pt,
1950                                           struct net_device *orig_dev,
1951                                           __be16 type,
1952                                           struct list_head *ptype_list)
1953 {
1954         struct packet_type *ptype, *pt_prev = *pt;
1955 
1956         list_for_each_entry_rcu(ptype, ptype_list, list) {
1957                 if (ptype->type != type)
1958                         continue;
1959                 if (pt_prev)
1960                         deliver_skb(skb, pt_prev, orig_dev);
1961                 pt_prev = ptype;
1962         }
1963         *pt = pt_prev;
1964 }
1965 
1966 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1967 {
1968         if (!ptype->af_packet_priv || !skb->sk)
1969                 return false;
1970 
1971         if (ptype->id_match)
1972                 return ptype->id_match(ptype, skb->sk);
1973         else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1974                 return true;
1975 
1976         return false;
1977 }
1978 
1979 /**
1980  * dev_nit_active - return true if any network interface taps are in use
1981  *
1982  * @dev: network device to check for the presence of taps
1983  */
1984 bool dev_nit_active(struct net_device *dev)
1985 {
1986         return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);
1987 }
1988 EXPORT_SYMBOL_GPL(dev_nit_active);
1989 
1990 /*
1991  *      Support routine. Sends outgoing frames to any network
1992  *      taps currently in use.
1993  */
1994 
1995 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1996 {
1997         struct packet_type *ptype;
1998         struct sk_buff *skb2 = NULL;
1999         struct packet_type *pt_prev = NULL;
2000         struct list_head *ptype_list = &ptype_all;
2001 
2002         rcu_read_lock();
2003 again:
2004         list_for_each_entry_rcu(ptype, ptype_list, list) {
2005                 if (ptype->ignore_outgoing)
2006                         continue;
2007 
2008                 /* Never send packets back to the socket
2009                  * they originated from - MvS (miquels@drinkel.ow.org)
2010                  */
2011                 if (skb_loop_sk(ptype, skb))
2012                         continue;
2013 
2014                 if (pt_prev) {
2015                         deliver_skb(skb2, pt_prev, skb->dev);
2016                         pt_prev = ptype;
2017                         continue;
2018                 }
2019 
2020                 /* need to clone skb, done only once */
2021                 skb2 = skb_clone(skb, GFP_ATOMIC);
2022                 if (!skb2)
2023                         goto out_unlock;
2024 
2025                 net_timestamp_set(skb2);
2026 
2027                 /* skb->nh should be correctly
2028                  * set by sender, so that the second statement is
2029                  * just protection against buggy protocols.
2030                  */
2031                 skb_reset_mac_header(skb2);
2032 
2033                 if (skb_network_header(skb2) < skb2->data ||
2034                     skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2035                         net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2036                                              ntohs(skb2->protocol),
2037                                              dev->name);
2038                         skb_reset_network_header(skb2);
2039                 }
2040 
2041                 skb2->transport_header = skb2->network_header;
2042                 skb2->pkt_type = PACKET_OUTGOING;
2043                 pt_prev = ptype;
2044         }
2045 
2046         if (ptype_list == &ptype_all) {
2047                 ptype_list = &dev->ptype_all;
2048                 goto again;
2049         }
2050 out_unlock:
2051         if (pt_prev) {
2052                 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2053                         pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2054                 else
2055                         kfree_skb(skb2);
2056         }
2057         rcu_read_unlock();
2058 }
2059 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
2060 
2061 /**
2062  * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2063  * @dev: Network device
2064  * @txq: number of queues available
2065  *
2066  * If real_num_tx_queues is changed the tc mappings may no longer be
2067  * valid. To resolve this verify the tc mapping remains valid and if
2068  * not NULL the mapping. With no priorities mapping to this
2069  * offset/count pair it will no longer be used. In the worst case TC0
2070  * is invalid nothing can be done so disable priority mappings. If is
2071  * expected that drivers will fix this mapping if they can before
2072  * calling netif_set_real_num_tx_queues.
2073  */
2074 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
2075 {
2076         int i;
2077         struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2078 
2079         /* If TC0 is invalidated disable TC mapping */
2080         if (tc->offset + tc->count > txq) {
2081                 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2082                 dev->num_tc = 0;
2083                 return;
2084         }
2085 
2086         /* Invalidated prio to tc mappings set to TC0 */
2087         for (i = 1; i < TC_BITMASK + 1; i++) {
2088                 int q = netdev_get_prio_tc_map(dev, i);
2089 
2090                 tc = &dev->tc_to_txq[q];
2091                 if (tc->offset + tc->count > txq) {
2092                         pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2093                                 i, q);
2094                         netdev_set_prio_tc_map(dev, i, 0);
2095                 }
2096         }
2097 }
2098 
2099 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2100 {
2101         if (dev->num_tc) {
2102                 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2103                 int i;
2104 
2105                 /* walk through the TCs and see if it falls into any of them */
2106                 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2107                         if ((txq - tc->offset) < tc->count)
2108                                 return i;
2109                 }
2110 
2111                 /* didn't find it, just return -1 to indicate no match */
2112                 return -1;
2113         }
2114 
2115         return 0;
2116 }
2117 EXPORT_SYMBOL(netdev_txq_to_tc);
2118 
2119 #ifdef CONFIG_XPS
2120 struct static_key xps_needed __read_mostly;
2121 EXPORT_SYMBOL(xps_needed);
2122 struct static_key xps_rxqs_needed __read_mostly;
2123 EXPORT_SYMBOL(xps_rxqs_needed);
2124 static DEFINE_MUTEX(xps_map_mutex);
2125 #define xmap_dereference(P)             \
2126         rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2127 
2128 static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2129                              int tci, u16 index)
2130 {
2131         struct xps_map *map = NULL;
2132         int pos;
2133 
2134         if (dev_maps)
2135                 map = xmap_dereference(dev_maps->attr_map[tci]);
2136         if (!map)
2137                 return false;
2138 
2139         for (pos = map->len; pos--;) {
2140                 if (map->queues[pos] != index)
2141                         continue;
2142 
2143                 if (map->len > 1) {
2144                         map->queues[pos] = map->queues[--map->len];
2145                         break;
2146                 }
2147 
2148                 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2149                 kfree_rcu(map, rcu);
2150                 return false;
2151         }
2152 
2153         return true;
2154 }
2155 
2156 static bool remove_xps_queue_cpu(struct net_device *dev,
2157                                  struct xps_dev_maps *dev_maps,
2158                                  int cpu, u16 offset, u16 count)
2159 {
2160         int num_tc = dev->num_tc ? : 1;
2161         bool active = false;
2162         int tci;
2163 
2164         for (tci = cpu * num_tc; num_tc--; tci++) {
2165                 int i, j;
2166 
2167                 for (i = count, j = offset; i--; j++) {
2168                         if (!remove_xps_queue(dev_maps, tci, j))
2169                                 break;
2170                 }
2171 
2172                 active |= i < 0;
2173         }
2174 
2175         return active;
2176 }
2177 
2178 static void clean_xps_maps(struct net_device *dev, const unsigned long *mask,
2179                            struct xps_dev_maps *dev_maps, unsigned int nr_ids,
2180                            u16 offset, u16 count, bool is_rxqs_map)
2181 {
2182         bool active = false;
2183         int i, j;
2184 
2185         for (j = -1; j = netif_attrmask_next(j, mask, nr_ids),
2186              j < nr_ids;)
2187                 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset,
2188                                                count);
2189         if (!active) {
2190                 if (is_rxqs_map) {
2191                         RCU_INIT_POINTER(dev->xps_rxqs_map, NULL);
2192                 } else {
2193                         RCU_INIT_POINTER(dev->xps_cpus_map, NULL);
2194 
2195                         for (i = offset + (count - 1); count--; i--)
2196                                 netdev_queue_numa_node_write(
2197                                         netdev_get_tx_queue(dev, i),
2198                                                         NUMA_NO_NODE);
2199                 }
2200                 kfree_rcu(dev_maps, rcu);
2201         }
2202 }
2203 
2204 static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2205                                    u16 count)
2206 {
2207         const unsigned long *possible_mask = NULL;
2208         struct xps_dev_maps *dev_maps;
2209         unsigned int nr_ids;
2210 
2211         if (!static_key_false(&xps_needed))
2212                 return;
2213 
2214         cpus_read_lock();
2215         mutex_lock(&xps_map_mutex);
2216 
2217         if (static_key_false(&xps_rxqs_needed)) {
2218                 dev_maps = xmap_dereference(dev->xps_rxqs_map);
2219                 if (dev_maps) {
2220                         nr_ids = dev->num_rx_queues;
2221                         clean_xps_maps(dev, possible_mask, dev_maps, nr_ids,
2222                                        offset, count, true);
2223                 }
2224         }
2225 
2226         dev_maps = xmap_dereference(dev->xps_cpus_map);
2227         if (!dev_maps)
2228                 goto out_no_maps;
2229 
2230         if (num_possible_cpus() > 1)
2231                 possible_mask = cpumask_bits(cpu_possible_mask);
2232         nr_ids = nr_cpu_ids;
2233         clean_xps_maps(dev, possible_mask, dev_maps, nr_ids, offset, count,
2234                        false);
2235 
2236 out_no_maps:
2237         if (static_key_enabled(&xps_rxqs_needed))
2238                 static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2239 
2240         static_key_slow_dec_cpuslocked(&xps_needed);
2241         mutex_unlock(&xps_map_mutex);
2242         cpus_read_unlock();
2243 }
2244 
2245 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2246 {
2247         netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2248 }
2249 
2250 static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
2251                                       u16 index, bool is_rxqs_map)
2252 {
2253         struct xps_map *new_map;
2254         int alloc_len = XPS_MIN_MAP_ALLOC;
2255         int i, pos;
2256 
2257         for (pos = 0; map && pos < map->len; pos++) {
2258                 if (map->queues[pos] != index)
2259                         continue;
2260                 return map;
2261         }
2262 
2263         /* Need to add tx-queue to this CPU's/rx-queue's existing map */
2264         if (map) {
2265                 if (pos < map->alloc_len)
2266                         return map;
2267 
2268                 alloc_len = map->alloc_len * 2;
2269         }
2270 
2271         /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
2272          *  map
2273          */
2274         if (is_rxqs_map)
2275                 new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2276         else
2277                 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2278                                        cpu_to_node(attr_index));
2279         if (!new_map)
2280                 return NULL;
2281 
2282         for (i = 0; i < pos; i++)
2283                 new_map->queues[i] = map->queues[i];
2284         new_map->alloc_len = alloc_len;
2285         new_map->len = pos;
2286 
2287         return new_map;
2288 }
2289 
2290 /* Must be called under cpus_read_lock */
2291 int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
2292                           u16 index, bool is_rxqs_map)
2293 {
2294         const unsigned long *online_mask = NULL, *possible_mask = NULL;
2295         struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2296         int i, j, tci, numa_node_id = -2;
2297         int maps_sz, num_tc = 1, tc = 0;
2298         struct xps_map *map, *new_map;
2299         bool active = false;
2300         unsigned int nr_ids;
2301 
2302         if (dev->num_tc) {
2303                 /* Do not allow XPS on subordinate device directly */
2304                 num_tc = dev->num_tc;
2305                 if (num_tc < 0)
2306                         return -EINVAL;
2307 
2308                 /* If queue belongs to subordinate dev use its map */
2309                 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2310 
2311                 tc = netdev_txq_to_tc(dev, index);
2312                 if (tc < 0)
2313                         return -EINVAL;
2314         }
2315 
2316         mutex_lock(&xps_map_mutex);
2317         if (is_rxqs_map) {
2318                 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2319                 dev_maps = xmap_dereference(dev->xps_rxqs_map);
2320                 nr_ids = dev->num_rx_queues;
2321         } else {
2322                 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2323                 if (num_possible_cpus() > 1) {
2324                         online_mask = cpumask_bits(cpu_online_mask);
2325                         possible_mask = cpumask_bits(cpu_possible_mask);
2326                 }
2327                 dev_maps = xmap_dereference(dev->xps_cpus_map);
2328                 nr_ids = nr_cpu_ids;
2329         }
2330 
2331         if (maps_sz < L1_CACHE_BYTES)
2332                 maps_sz = L1_CACHE_BYTES;
2333 
2334         /* allocate memory for queue storage */
2335         for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2336              j < nr_ids;) {
2337                 if (!new_dev_maps)
2338                         new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2339                 if (!new_dev_maps) {
2340                         mutex_unlock(&xps_map_mutex);
2341                         return -ENOMEM;
2342                 }
2343 
2344                 tci = j * num_tc + tc;
2345                 map = dev_maps ? xmap_dereference(dev_maps->attr_map[tci]) :
2346                                  NULL;
2347 
2348                 map = expand_xps_map(map, j, index, is_rxqs_map);
2349                 if (!map)
2350                         goto error;
2351 
2352                 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2353         }
2354 
2355         if (!new_dev_maps)
2356                 goto out_no_new_maps;
2357 
2358         static_key_slow_inc_cpuslocked(&xps_needed);
2359         if (is_rxqs_map)
2360                 static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2361 
2362         for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2363              j < nr_ids;) {
2364                 /* copy maps belonging to foreign traffic classes */
2365                 for (i = tc, tci = j * num_tc; dev_maps && i--; tci++) {
2366                         /* fill in the new device map from the old device map */
2367                         map = xmap_dereference(dev_maps->attr_map[tci]);
2368                         RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2369                 }
2370 
2371                 /* We need to explicitly update tci as prevous loop
2372                  * could break out early if dev_maps is NULL.
2373                  */
2374                 tci = j * num_tc + tc;
2375 
2376                 if (netif_attr_test_mask(j, mask, nr_ids) &&
2377                     netif_attr_test_online(j, online_mask, nr_ids)) {
2378                         /* add tx-queue to CPU/rx-queue maps */
2379                         int pos = 0;
2380 
2381                         map = xmap_dereference(new_dev_maps->attr_map[tci]);
2382                         while ((pos < map->len) && (map->queues[pos] != index))
2383                                 pos++;
2384 
2385                         if (pos == map->len)
2386                                 map->queues[map->len++] = index;
2387 #ifdef CONFIG_NUMA
2388                         if (!is_rxqs_map) {
2389                                 if (numa_node_id == -2)
2390                                         numa_node_id = cpu_to_node(j);
2391                                 else if (numa_node_id != cpu_to_node(j))
2392                                         numa_node_id = -1;
2393                         }
2394 #endif
2395                 } else if (dev_maps) {
2396                         /* fill in the new device map from the old device map */
2397                         map = xmap_dereference(dev_maps->attr_map[tci]);
2398                         RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2399                 }
2400 
2401                 /* copy maps belonging to foreign traffic classes */
2402                 for (i = num_tc - tc, tci++; dev_maps && --i; tci++) {
2403                         /* fill in the new device map from the old device map */
2404                         map = xmap_dereference(dev_maps->attr_map[tci]);
2405                         RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2406                 }
2407         }
2408 
2409         if (is_rxqs_map)
2410                 rcu_assign_pointer(dev->xps_rxqs_map, new_dev_maps);
2411         else
2412                 rcu_assign_pointer(dev->xps_cpus_map, new_dev_maps);
2413 
2414         /* Cleanup old maps */
2415         if (!dev_maps)
2416                 goto out_no_old_maps;
2417 
2418         for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2419              j < nr_ids;) {
2420                 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2421                         new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2422                         map = xmap_dereference(dev_maps->attr_map[tci]);
2423                         if (map && map != new_map)
2424                                 kfree_rcu(map, rcu);
2425                 }
2426         }
2427 
2428         kfree_rcu(dev_maps, rcu);
2429 
2430 out_no_old_maps:
2431         dev_maps = new_dev_maps;
2432         active = true;
2433 
2434 out_no_new_maps:
2435         if (!is_rxqs_map) {
2436                 /* update Tx queue numa node */
2437                 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2438                                              (numa_node_id >= 0) ?
2439                                              numa_node_id : NUMA_NO_NODE);
2440         }
2441 
2442         if (!dev_maps)
2443                 goto out_no_maps;
2444 
2445         /* removes tx-queue from unused CPUs/rx-queues */
2446         for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2447              j < nr_ids;) {
2448                 for (i = tc, tci = j * num_tc; i--; tci++)
2449                         active |= remove_xps_queue(dev_maps, tci, index);
2450                 if (!netif_attr_test_mask(j, mask, nr_ids) ||
2451                     !netif_attr_test_online(j, online_mask, nr_ids))
2452                         active |= remove_xps_queue(dev_maps, tci, index);
2453                 for (i = num_tc - tc, tci++; --i; tci++)
2454                         active |= remove_xps_queue(dev_maps, tci, index);
2455         }
2456 
2457         /* free map if not active */
2458         if (!active) {
2459                 if (is_rxqs_map)
2460                         RCU_INIT_POINTER(dev->xps_rxqs_map, NULL);
2461                 else
2462                         RCU_INIT_POINTER(dev->xps_cpus_map, NULL);
2463                 kfree_rcu(dev_maps, rcu);
2464         }
2465 
2466 out_no_maps:
2467         mutex_unlock(&xps_map_mutex);
2468 
2469         return 0;
2470 error:
2471         /* remove any maps that we added */
2472         for (j = -1; j = netif_attrmask_next(j, possible_mask, nr_ids),
2473              j < nr_ids;) {
2474                 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2475                         new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2476                         map = dev_maps ?
2477                               xmap_dereference(dev_maps->attr_map[tci]) :
2478                               NULL;
2479                         if (new_map && new_map != map)
2480                                 kfree(new_map);
2481                 }
2482         }
2483 
2484         mutex_unlock(&xps_map_mutex);
2485 
2486         kfree(new_dev_maps);
2487         return -ENOMEM;
2488 }
2489 EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
2490 
2491 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2492                         u16 index)
2493 {
2494         int ret;
2495 
2496         cpus_read_lock();
2497         ret =  __netif_set_xps_queue(dev, cpumask_bits(mask), index, false);
2498         cpus_read_unlock();
2499 
2500         return ret;
2501 }
2502 EXPORT_SYMBOL(netif_set_xps_queue);
2503 
2504 #endif
2505 static void netdev_unbind_all_sb_channels(struct net_device *dev)
2506 {
2507         struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2508 
2509         /* Unbind any subordinate channels */
2510         while (txq-- != &dev->_tx[0]) {
2511                 if (txq->sb_dev)
2512                         netdev_unbind_sb_channel(dev, txq->sb_dev);
2513         }
2514 }
2515 
2516 void netdev_reset_tc(struct net_device *dev)
2517 {
2518 #ifdef CONFIG_XPS
2519         netif_reset_xps_queues_gt(dev, 0);
2520 #endif
2521         netdev_unbind_all_sb_channels(dev);
2522 
2523         /* Reset TC configuration of device */
2524         dev->num_tc = 0;
2525         memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2526         memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2527 }
2528 EXPORT_SYMBOL(netdev_reset_tc);
2529 
2530 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2531 {
2532         if (tc >= dev->num_tc)
2533                 return -EINVAL;
2534 
2535 #ifdef CONFIG_XPS
2536         netif_reset_xps_queues(dev, offset, count);
2537 #endif
2538         dev->tc_to_txq[tc].count = count;
2539         dev->tc_to_txq[tc].offset = offset;
2540         return 0;
2541 }
2542 EXPORT_SYMBOL(netdev_set_tc_queue);
2543 
2544 int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2545 {
2546         if (num_tc > TC_MAX_QUEUE)
2547                 return -EINVAL;
2548 
2549 #ifdef CONFIG_XPS
2550         netif_reset_xps_queues_gt(dev, 0);
2551 #endif
2552         netdev_unbind_all_sb_channels(dev);
2553 
2554         dev->num_tc = num_tc;
2555         return 0;
2556 }
2557 EXPORT_SYMBOL(netdev_set_num_tc);
2558 
2559 void netdev_unbind_sb_channel(struct net_device *dev,
2560                               struct net_device *sb_dev)
2561 {
2562         struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2563 
2564 #ifdef CONFIG_XPS
2565         netif_reset_xps_queues_gt(sb_dev, 0);
2566 #endif
2567         memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
2568         memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
2569 
2570         while (txq-- != &dev->_tx[0]) {
2571                 if (txq->sb_dev == sb_dev)
2572                         txq->sb_dev = NULL;
2573         }
2574 }
2575 EXPORT_SYMBOL(netdev_unbind_sb_channel);
2576 
2577 int netdev_bind_sb_channel_queue(struct net_device *dev,
2578                                  struct net_device *sb_dev,
2579                                  u8 tc, u16 count, u16 offset)
2580 {
2581         /* Make certain the sb_dev and dev are already configured */
2582         if (sb_dev->num_tc >= 0 || tc >= dev->num_tc)
2583                 return -EINVAL;
2584 
2585         /* We cannot hand out queues we don't have */
2586         if ((offset + count) > dev->real_num_tx_queues)
2587                 return -EINVAL;
2588 
2589         /* Record the mapping */
2590         sb_dev->tc_to_txq[tc].count = count;
2591         sb_dev->tc_to_txq[tc].offset = offset;
2592 
2593         /* Provide a way for Tx queue to find the tc_to_txq map or
2594          * XPS map for itself.
2595          */
2596         while (count--)
2597                 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
2598 
2599         return 0;
2600 }
2601 EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
2602 
2603 int netdev_set_sb_channel(struct net_device *dev, u16 channel)
2604 {
2605         /* Do not use a multiqueue device to represent a subordinate channel */
2606         if (netif_is_multiqueue(dev))
2607                 return -ENODEV;
2608 
2609         /* We allow channels 1 - 32767 to be used for subordinate channels.
2610          * Channel 0 is meant to be "native" mode and used only to represent
2611          * the main root device. We allow writing 0 to reset the device back
2612          * to normal mode after being used as a subordinate channel.
2613          */
2614         if (channel > S16_MAX)
2615                 return -EINVAL;
2616 
2617         dev->num_tc = -channel;
2618 
2619         return 0;
2620 }
2621 EXPORT_SYMBOL(netdev_set_sb_channel);
2622 
2623 /*
2624  * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2625  * greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
2626  */
2627 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2628 {
2629         bool disabling;
2630         int rc;
2631 
2632         disabling = txq < dev->real_num_tx_queues;
2633 
2634         if (txq < 1 || txq > dev->num_tx_queues)
2635                 return -EINVAL;
2636 
2637         if (dev->reg_state == NETREG_REGISTERED ||
2638             dev->reg_state == NETREG_UNREGISTERING) {
2639                 ASSERT_RTNL();
2640 
2641                 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2642                                                   txq);
2643                 if (rc)
2644                         return rc;
2645 
2646                 if (dev->num_tc)
2647                         netif_setup_tc(dev, txq);
2648 
2649                 dev->real_num_tx_queues = txq;
2650 
2651                 if (disabling) {
2652                         synchronize_net();
2653                         qdisc_reset_all_tx_gt(dev, txq);
2654 #ifdef CONFIG_XPS
2655                         netif_reset_xps_queues_gt(dev, txq);
2656 #endif
2657                 }
2658         } else {
2659                 dev->real_num_tx_queues = txq;
2660         }
2661 
2662         return 0;
2663 }
2664 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2665 
2666 #ifdef CONFIG_SYSFS
2667 /**
2668  *      netif_set_real_num_rx_queues - set actual number of RX queues used
2669  *      @dev: Network device
2670  *      @rxq: Actual number of RX queues
2671  *
2672  *      This must be called either with the rtnl_lock held or before
2673  *      registration of the net device.  Returns 0 on success, or a
2674  *      negative error code.  If called before registration, it always
2675  *      succeeds.
2676  */
2677 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2678 {
2679         int rc;
2680 
2681         if (rxq < 1 || rxq > dev->num_rx_queues)
2682                 return -EINVAL;
2683 
2684         if (dev->reg_state == NETREG_REGISTERED) {
2685                 ASSERT_RTNL();
2686 
2687                 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2688                                                   rxq);
2689                 if (rc)
2690                         return rc;
2691         }
2692 
2693         dev->real_num_rx_queues = rxq;
2694         return 0;
2695 }
2696 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2697 #endif
2698 
2699 /**
2700  * netif_get_num_default_rss_queues - default number of RSS queues
2701  *
2702  * This routine should set an upper limit on the number of RSS queues
2703  * used by default by multiqueue devices.
2704  */
2705 int netif_get_num_default_rss_queues(void)
2706 {
2707         return is_kdump_kernel() ?
2708                 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2709 }
2710 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2711 
2712 static void __netif_reschedule(struct Qdisc *q)
2713 {
2714         struct softnet_data *sd;
2715         unsigned long flags;
2716 
2717         local_irq_save(flags);
2718         sd = this_cpu_ptr(&softnet_data);
2719         q->next_sched = NULL;
2720         *sd->output_queue_tailp = q;
2721         sd->output_queue_tailp = &q->next_sched;
2722         raise_softirq_irqoff(NET_TX_SOFTIRQ);
2723         local_irq_restore(flags);
2724 }
2725 
2726 void __netif_schedule(struct Qdisc *q)
2727 {
2728         if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2729                 __netif_reschedule(q);
2730 }
2731 EXPORT_SYMBOL(__netif_schedule);
2732 
2733 struct dev_kfree_skb_cb {
2734         enum skb_free_reason reason;
2735 };
2736 
2737 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
2738 {
2739         return (struct dev_kfree_skb_cb *)skb->cb;
2740 }
2741 
2742 void netif_schedule_queue(struct netdev_queue *txq)
2743 {
2744         rcu_read_lock();
2745         if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2746                 struct Qdisc *q = rcu_dereference(txq->qdisc);
2747 
2748                 __netif_schedule(q);
2749         }
2750         rcu_read_unlock();
2751 }
2752 EXPORT_SYMBOL(netif_schedule_queue);
2753 
2754 void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2755 {
2756         if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2757                 struct Qdisc *q;
2758 
2759                 rcu_read_lock();
2760                 q = rcu_dereference(dev_queue->qdisc);
2761                 __netif_schedule(q);
2762                 rcu_read_unlock();
2763         }
2764 }
2765 EXPORT_SYMBOL(netif_tx_wake_queue);
2766 
2767 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2768 {
2769         unsigned long flags;
2770 
2771         if (unlikely(!skb))
2772                 return;
2773 
2774         if (likely(refcount_read(&skb->users) == 1)) {
2775                 smp_rmb();
2776                 refcount_set(&skb->users, 0);
2777         } else if (likely(!refcount_dec_and_test(&skb->users))) {
2778                 return;
2779         }
2780         get_kfree_skb_cb(skb)->reason = reason;
2781         local_irq_save(flags);
2782         skb->next = __this_cpu_read(softnet_data.completion_queue);
2783         __this_cpu_write(softnet_data.completion_queue, skb);
2784         raise_softirq_irqoff(NET_TX_SOFTIRQ);
2785         local_irq_restore(flags);
2786 }
2787 EXPORT_SYMBOL(__dev_kfree_skb_irq);
2788 
2789 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
2790 {
2791         if (in_irq() || irqs_disabled())
2792                 __dev_kfree_skb_irq(skb, reason);
2793         else
2794                 dev_kfree_skb(skb);
2795 }
2796 EXPORT_SYMBOL(__dev_kfree_skb_any);
2797 
2798 
2799 /**
2800  * netif_device_detach - mark device as removed
2801  * @dev: network device
2802  *
2803  * Mark device as removed from system and therefore no longer available.
2804  */
2805 void netif_device_detach(struct net_device *dev)
2806 {
2807         if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2808             netif_running(dev)) {
2809                 netif_tx_stop_all_queues(dev);
2810         }
2811 }
2812 EXPORT_SYMBOL(netif_device_detach);
2813 
2814 /**
2815  * netif_device_attach - mark device as attached
2816  * @dev: network device
2817  *
2818  * Mark device as attached from system and restart if needed.
2819  */
2820 void netif_device_attach(struct net_device *dev)
2821 {
2822         if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2823             netif_running(dev)) {
2824                 netif_tx_wake_all_queues(dev);
2825                 __netdev_watchdog_up(dev);
2826         }
2827 }
2828 EXPORT_SYMBOL(netif_device_attach);
2829 
2830 /*
2831  * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2832  * to be used as a distribution range.
2833  */
2834 static u16 skb_tx_hash(const struct net_device *dev,
2835                        const struct net_device *sb_dev,
2836                        struct sk_buff *skb)
2837 {
2838         u32 hash;
2839         u16 qoffset = 0;
2840         u16 qcount = dev->real_num_tx_queues;
2841 
2842         if (dev->num_tc) {
2843                 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2844 
2845                 qoffset = sb_dev->tc_to_txq[tc].offset;
2846                 qcount = sb_dev->tc_to_txq[tc].count;
2847         }
2848 
2849         if (skb_rx_queue_recorded(skb)) {
2850                 hash = skb_get_rx_queue(skb);
2851                 while (unlikely(hash >= qcount))
2852                         hash -= qcount;
2853                 return hash + qoffset;
2854         }
2855 
2856         return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2857 }
2858 
2859 static void skb_warn_bad_offload(const struct sk_buff *skb)
2860 {
2861         static const netdev_features_t null_features;
2862         struct net_device *dev = skb->dev;
2863         const char *name = "";
2864 
2865         if (!net_ratelimit())
2866                 return;
2867 
2868         if (dev) {
2869                 if (dev->dev.parent)
2870                         name = dev_driver_string(dev->dev.parent);
2871                 else
2872                         name = netdev_name(dev);
2873         }
2874         WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2875              "gso_type=%d ip_summed=%d\n",
2876              name, dev ? &dev->features : &null_features,
2877              skb->sk ? &skb->sk->sk_route_caps : &null_features,
2878              skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2879              skb_shinfo(skb)->gso_type, skb->ip_summed);
2880 }
2881 
2882 /*
2883  * Invalidate hardware checksum when packet is to be mangled, and
2884  * complete checksum manually on outgoing path.
2885  */
2886 int skb_checksum_help(struct sk_buff *skb)
2887 {
2888         __wsum csum;
2889         int ret = 0, offset;
2890 
2891         if (skb->ip_summed == CHECKSUM_COMPLETE)
2892                 goto out_set_summed;
2893 
2894         if (unlikely(skb_shinfo(skb)->gso_size)) {
2895                 skb_warn_bad_offload(skb);
2896                 return -EINVAL;
2897         }
2898 
2899         /* Before computing a checksum, we should make sure no frag could
2900          * be modified by an external entity : checksum could be wrong.
2901          */
2902         if (skb_has_shared_frag(skb)) {
2903                 ret = __skb_linearize(skb);
2904                 if (ret)
2905                         goto out;
2906         }
2907 
2908         offset = skb_checksum_start_offset(skb);
2909         BUG_ON(offset >= skb_headlen(skb));
2910         csum = skb_checksum(skb, offset, skb->len - offset, 0);
2911 
2912         offset += skb->csum_offset;
2913         BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2914 
2915         if (skb_cloned(skb) &&
2916             !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2917                 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2918                 if (ret)
2919                         goto out;
2920         }
2921 
2922         *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
2923 out_set_summed:
2924         skb->ip_summed = CHECKSUM_NONE;
2925 out:
2926         return ret;
2927 }
2928 EXPORT_SYMBOL(skb_checksum_help);
2929 
2930 int skb_crc32c_csum_help(struct sk_buff *skb)
2931 {
2932         __le32 crc32c_csum;
2933         int ret = 0, offset, start;
2934 
2935         if (skb->ip_summed != CHECKSUM_PARTIAL)
2936                 goto out;
2937 
2938         if (unlikely(skb_is_gso(skb)))
2939                 goto out;
2940 
2941         /* Before computing a checksum, we should make sure no frag could
2942          * be modified by an external entity : checksum could be wrong.
2943          */
2944         if (unlikely(skb_has_shared_frag(skb))) {
2945                 ret = __skb_linearize(skb);
2946                 if (ret)
2947                         goto out;
2948         }
2949         start = skb_checksum_start_offset(skb);
2950         offset = start + offsetof(struct sctphdr, checksum);
2951         if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
2952                 ret = -EINVAL;
2953                 goto out;
2954         }
2955         if (skb_cloned(skb) &&
2956             !skb_clone_writable(skb, offset + sizeof(__le32))) {
2957                 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2958                 if (ret)
2959                         goto out;
2960         }
2961         crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
2962                                                   skb->len - start, ~(__u32)0,
2963                                                   crc32c_csum_stub));
2964         *(__le32 *)(skb->data + offset) = crc32c_csum;
2965         skb->ip_summed = CHECKSUM_NONE;
2966         skb->csum_not_inet = 0;
2967 out:
2968         return ret;
2969 }
2970 
2971 __be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2972 {
2973         __be16 type = skb->protocol;
2974 
2975         /* Tunnel gso handlers can set protocol to ethernet. */
2976         if (type == htons(ETH_P_TEB)) {
2977                 struct ethhdr *eth;
2978 
2979                 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2980                         return 0;
2981 
2982                 eth = (struct ethhdr *)skb->data;
2983                 type = eth->h_proto;
2984         }
2985 
2986         return __vlan_get_protocol(skb, type, depth);
2987 }
2988 
2989 /**
2990  *      skb_mac_gso_segment - mac layer segmentation handler.
2991  *      @skb: buffer to segment
2992  *      @features: features for the output path (see dev->features)
2993  */
2994 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2995                                     netdev_features_t features)
2996 {
2997         struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2998         struct packet_offload *ptype;
2999         int vlan_depth = skb->mac_len;
3000         __be16 type = skb_network_protocol(skb, &vlan_depth);
3001 
3002         if (unlikely(!type))
3003                 return ERR_PTR(-EINVAL);
3004 
3005         __skb_pull(skb, vlan_depth);
3006 
3007         rcu_read_lock();
3008         list_for_each_entry_rcu(ptype, &offload_base, list) {
3009                 if (ptype->type == type && ptype->callbacks.gso_segment) {
3010                         segs = ptype->callbacks.gso_segment(skb, features);
3011                         break;
3012                 }
3013         }
3014         rcu_read_unlock();
3015 
3016         __skb_push(skb, skb->data - skb_mac_header(skb));
3017 
3018         return segs;
3019 }
3020 EXPORT_SYMBOL(skb_mac_gso_segment);
3021 
3022 
3023 /* openvswitch calls this on rx path, so we need a different check.
3024  */
3025 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
3026 {
3027         if (tx_path)
3028                 return skb->ip_summed != CHECKSUM_PARTIAL &&
3029                        skb->ip_summed != CHECKSUM_UNNECESSARY;
3030 
3031         return skb->ip_summed == CHECKSUM_NONE;
3032 }
3033 
3034 /**
3035  *      __skb_gso_segment - Perform segmentation on skb.
3036  *      @skb: buffer to segment
3037  *      @features: features for the output path (see dev->features)
3038  *      @tx_path: whether it is called in TX path
3039  *
3040  *      This function segments the given skb and returns a list of segments.
3041  *
3042  *      It may return NULL if the skb requires no segmentation.  This is
3043  *      only possible when GSO is used for verifying header integrity.
3044  *
3045  *      Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
3046  */
3047 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
3048                                   netdev_features_t features, bool tx_path)
3049 {
3050         struct sk_buff *segs;
3051 
3052         if (unlikely(skb_needs_check(skb, tx_path))) {
3053                 int err;
3054 
3055                 /* We're going to init ->check field in TCP or UDP header */
3056                 err = skb_cow_head(skb, 0);
3057                 if (err < 0)
3058                         return ERR_PTR(err);
3059         }
3060 
3061         /* Only report GSO partial support if it will enable us to
3062          * support segmentation on this frame without needing additional
3063          * work.
3064          */
3065         if (features & NETIF_F_GSO_PARTIAL) {
3066                 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
3067                 struct net_device *dev = skb->dev;
3068 
3069                 partial_features |= dev->features & dev->gso_partial_features;
3070                 if (!skb_gso_ok(skb, features | partial_features))
3071                         features &= ~NETIF_F_GSO_PARTIAL;
3072         }
3073 
3074         BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
3075                      sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
3076 
3077         SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
3078         SKB_GSO_CB(skb)->encap_level = 0;
3079 
3080         skb_reset_mac_header(skb);
3081         skb_reset_mac_len(skb);
3082 
3083         segs = skb_mac_gso_segment(skb, features);
3084 
3085         if (unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
3086                 skb_warn_bad_offload(skb);
3087 
3088         return segs;
3089 }
3090 EXPORT_SYMBOL(__skb_gso_segment);
3091 
3092 /* Take action when hardware reception checksum errors are detected. */
3093 #ifdef CONFIG_BUG
3094 void netdev_rx_csum_fault(struct net_device *dev)
3095 {
3096         if (net_ratelimit()) {
3097                 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
3098                 dump_stack();
3099         }
3100 }
3101 EXPORT_SYMBOL(netdev_rx_csum_fault);
3102 #endif
3103 
3104 /* XXX: check that highmem exists at all on the given machine. */
3105 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
3106 {
3107 #ifdef CONFIG_HIGHMEM
3108         int i;
3109 
3110         if (!(dev->features & NETIF_F_HIGHDMA)) {
3111                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3112                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3113 
3114                         if (PageHighMem(skb_frag_page(frag)))
3115                                 return 1;
3116                 }
3117         }
3118 #endif
3119         return 0;
3120 }
3121 
3122 /* If MPLS offload request, verify we are testing hardware MPLS features
3123  * instead of standard features for the netdev.
3124  */
3125 #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
3126 static netdev_features_t net_mpls_features(struct sk_buff *skb,
3127                                            netdev_features_t features,
3128                                            __be16 type)
3129 {
3130         if (eth_p_mpls(type))
3131                 features &= skb->dev->mpls_features;
3132 
3133         return features;
3134 }
3135 #else
3136 static netdev_features_t net_mpls_features(struct sk_buff *skb,
3137                                            netdev_features_t features,
3138                                            __be16 type)
3139 {
3140         return features;
3141 }
3142 #endif
3143 
3144 static netdev_features_t harmonize_features(struct sk_buff *skb,
3145         netdev_features_t features)
3146 {
3147         int tmp;
3148         __be16 type;
3149 
3150         type = skb_network_protocol(skb, &tmp);
3151         features = net_mpls_features(skb, features, type);
3152 
3153         if (skb->ip_summed != CHECKSUM_NONE &&
3154             !can_checksum_protocol(features, type)) {
3155                 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3156         }
3157         if (illegal_highdma(skb->dev, skb))
3158                 features &= ~NETIF_F_SG;
3159 
3160         return features;
3161 }
3162 
3163 netdev_features_t passthru_features_check(struct sk_buff *skb,
3164                                           struct net_device *dev,
3165                                           netdev_features_t features)
3166 {
3167         return features;
3168 }
3169 EXPORT_SYMBOL(passthru_features_check);
3170 
3171 static netdev_features_t dflt_features_check(struct sk_buff *skb,
3172                                              struct net_device *dev,
3173                                              netdev_features_t features)
3174 {
3175         return vlan_features_check(skb, features);
3176 }
3177 
3178 static netdev_features_t gso_features_check(const struct sk_buff *skb,
3179                                             struct net_device *dev,
3180                                             netdev_features_t features)
3181 {
3182         u16 gso_segs = skb_shinfo(skb)->gso_segs;
3183 
3184         if (gso_segs > dev->gso_max_segs)
3185                 return features & ~NETIF_F_GSO_MASK;
3186 
3187         /* Support for GSO partial features requires software
3188          * intervention before we can actually process the packets
3189          * so we need to strip support for any partial features now
3190          * and we can pull them back in after we have partially
3191          * segmented the frame.
3192          */
3193         if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3194                 features &= ~dev->gso_partial_features;
3195 
3196         /* Make sure to clear the IPv4 ID mangling feature if the
3197          * IPv4 header has the potential to be fragmented.
3198          */
3199         if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3200                 struct iphdr *iph = skb->encapsulation ?
3201                                     inner_ip_hdr(skb) : ip_hdr(skb);
3202 
3203                 if (!(iph->frag_off & htons(IP_DF)))
3204                         features &= ~NETIF_F_TSO_MANGLEID;
3205         }
3206 
3207         return features;
3208 }
3209 
3210 netdev_features_t netif_skb_features(struct sk_buff *skb)
3211 {
3212         struct net_device *dev = skb->dev;
3213         netdev_features_t features = dev->features;
3214 
3215         if (skb_is_gso(skb))
3216                 features = gso_features_check(skb, dev, features);
3217 
3218         /* If encapsulation offload request, verify we are testing
3219          * hardware encapsulation features instead of standard
3220          * features for the netdev
3221          */
3222         if (skb->encapsulation)
3223                 features &= dev->hw_enc_features;
3224 
3225         if (skb_vlan_tagged(skb))
3226                 features = netdev_intersect_features(features,
3227                                                      dev->vlan_features |
3228                                                      NETIF_F_HW_VLAN_CTAG_TX |
3229                                                      NETIF_F_HW_VLAN_STAG_TX);
3230 
3231         if (dev->netdev_ops->ndo_features_check)
3232                 features &= dev->netdev_ops->ndo_features_check(skb, dev,
3233                                                                 features);
3234         else
3235                 features &= dflt_features_check(skb, dev, features);
3236 
3237         return harmonize_features(skb, features);
3238 }
3239 EXPORT_SYMBOL(netif_skb_features);
3240 
3241 static int xmit_one(struct sk_buff *skb, struct net_device *dev,
3242                     struct netdev_queue *txq, bool more)
3243 {
3244         unsigned int len;
3245         int rc;
3246 
3247         if (dev_nit_active(dev))
3248                 dev_queue_xmit_nit(skb, dev);
3249 
3250         len = skb->len;
3251         trace_net_dev_start_xmit(skb, dev);
3252         rc = netdev_start_xmit(skb, dev, txq, more);
3253         trace_net_dev_xmit(skb, rc, dev, len);
3254 
3255         return rc;
3256 }
3257 
3258 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3259                                     struct netdev_queue *txq, int *ret)
3260 {
3261         struct sk_buff *skb = first;
3262         int rc = NETDEV_TX_OK;
3263 
3264         while (skb) {
3265                 struct sk_buff *next = skb->next;
3266 
3267                 skb_mark_not_on_list(skb);
3268                 rc = xmit_one(skb, dev, txq, next != NULL);
3269                 if (unlikely(!dev_xmit_complete(rc))) {
3270                         skb->next = next;
3271                         goto out;
3272                 }
3273 
3274                 skb = next;
3275                 if (netif_tx_queue_stopped(txq) && skb) {
3276                         rc = NETDEV_TX_BUSY;
3277                         break;
3278                 }
3279         }
3280 
3281 out:
3282         *ret = rc;
3283         return skb;
3284 }
3285 
3286 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3287                                           netdev_features_t features)
3288 {
3289         if (skb_vlan_tag_present(skb) &&
3290             !vlan_hw_offload_capable(features, skb->vlan_proto))
3291                 skb = __vlan_hwaccel_push_inside(skb);
3292         return skb;
3293 }
3294 
3295 int skb_csum_hwoffload_help(struct sk_buff *skb,
3296                             const netdev_features_t features)
3297 {
3298         if (unlikely(skb->csum_not_inet))
3299                 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3300                         skb_crc32c_csum_help(skb);
3301 
3302         return !!(features & NETIF_F_CSUM_MASK) ? 0 : skb_checksum_help(skb);
3303 }
3304 EXPORT_SYMBOL(skb_csum_hwoffload_help);
3305 
3306 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
3307 {
3308         netdev_features_t features;
3309 
3310         features = netif_skb_features(skb);
3311         skb = validate_xmit_vlan(skb, features);
3312         if (unlikely(!skb))
3313                 goto out_null;
3314 
3315         skb = sk_validate_xmit_skb(skb, dev);
3316         if (unlikely(!skb))
3317                 goto out_null;
3318 
3319         if (netif_needs_gso(skb, features)) {
3320                 struct sk_buff *segs;
3321 
3322                 segs = skb_gso_segment(skb, features);
3323                 if (IS_ERR(segs)) {
3324                         goto out_kfree_skb;
3325                 } else if (segs) {
3326                         consume_skb(skb);
3327                         skb = segs;
3328                 }
3329         } else {
3330                 if (skb_needs_linearize(skb, features) &&
3331                     __skb_linearize(skb))
3332                         goto out_kfree_skb;
3333 
3334                 /* If packet is not checksummed and device does not
3335                  * support checksumming for this protocol, complete
3336                  * checksumming here.
3337                  */
3338                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3339                         if (skb->encapsulation)
3340                                 skb_set_inner_transport_header(skb,
3341                                                                skb_checksum_start_offset(skb));
3342                         else
3343                                 skb_set_transport_header(skb,
3344                                                          skb_checksum_start_offset(skb));
3345                         if (skb_csum_hwoffload_help(skb, features))
3346                                 goto out_kfree_skb;
3347                 }
3348         }
3349 
3350         skb = validate_xmit_xfrm(skb, features, again);
3351 
3352         return skb;
3353 
3354 out_kfree_skb:
3355         kfree_skb(skb);
3356 out_null:
3357         atomic_long_inc(&dev->tx_dropped);
3358         return NULL;
3359 }
3360 
3361 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
3362 {
3363         struct sk_buff *next, *head = NULL, *tail;
3364 
3365         for (; skb != NULL; skb = next) {
3366                 next = skb->next;
3367                 skb_mark_not_on_list(skb);
3368 
3369                 /* in case skb wont be segmented, point to itself */
3370                 skb->prev = skb;
3371 
3372                 skb = validate_xmit_skb(skb, dev, again);
3373                 if (!skb)
3374                         continue;
3375 
3376                 if (!head)
3377                         head = skb;
3378                 else
3379                         tail->next = skb;
3380                 /* If skb was segmented, skb->prev points to
3381                  * the last segment. If not, it still contains skb.
3382                  */
3383                 tail = skb->prev;
3384         }
3385         return head;
3386 }
3387 EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
3388 
3389 static void qdisc_pkt_len_init(struct sk_buff *skb)
3390 {
3391         const struct skb_shared_info *shinfo = skb_shinfo(skb);
3392 
3393         qdisc_skb_cb(skb)->pkt_len = skb->len;
3394 
3395         /* To get more precise estimation of bytes sent on wire,
3396          * we add to pkt_len the headers size of all segments
3397          */
3398         if (shinfo->gso_size)  {
3399                 unsigned int hdr_len;
3400                 u16 gso_segs = shinfo->gso_segs;
3401 
3402                 /* mac layer + network layer */
3403                 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3404 
3405                 /* + transport layer */
3406                 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
3407                         const struct tcphdr *th;
3408                         struct tcphdr _tcphdr;
3409 
3410                         th = skb_header_pointer(skb, skb_transport_offset(skb),
3411                                                 sizeof(_tcphdr), &_tcphdr);
3412                         if (likely(th))
3413                                 hdr_len += __tcp_hdrlen(th);
3414                 } else {
3415                         struct udphdr _udphdr;
3416 
3417                         if (skb_header_pointer(skb, skb_transport_offset(skb),
3418                                                sizeof(_udphdr), &_udphdr))
3419                                 hdr_len += sizeof(struct udphdr);
3420                 }
3421 
3422                 if (shinfo->gso_type & SKB_GSO_DODGY)
3423                         gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3424                                                 shinfo->gso_size);
3425 
3426                 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3427         }
3428 }
3429 
3430 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3431                                  struct net_device *dev,
3432                                  struct netdev_queue *txq)
3433 {
3434         spinlock_t *root_lock = qdisc_lock(q);
3435         struct sk_buff *to_free = NULL;
3436         bool contended;
3437         int rc;
3438 
3439         qdisc_calculate_pkt_len(skb, q);
3440 
3441         if (q->flags & TCQ_F_NOLOCK) {
3442                 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3443                         __qdisc_drop(skb, &to_free);
3444                         rc = NET_XMIT_DROP;
3445                 } else {
3446                         rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
3447                         qdisc_run(q);
3448                 }
3449 
3450                 if (unlikely(to_free))
3451                         kfree_skb_list(to_free);
3452                 return rc;
3453         }
3454 
3455         /*
3456          * Heuristic to force contended enqueues to serialize on a
3457          * separate lock before trying to get qdisc main lock.
3458          * This permits qdisc->running owner to get the lock more
3459          * often and dequeue packets faster.
3460          */
3461         contended = qdisc_is_running(q);
3462         if (unlikely(contended))
3463                 spin_lock(&q->busylock);
3464 
3465         spin_lock(root_lock);
3466         if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3467                 __qdisc_drop(skb, &to_free);
3468                 rc = NET_XMIT_DROP;
3469         } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3470                    qdisc_run_begin(q)) {
3471                 /*
3472                  * This is a work-conserving queue; there are no old skbs
3473                  * waiting to be sent out; and the qdisc is not running -
3474                  * xmit the skb directly.
3475                  */
3476 
3477                 qdisc_bstats_update(q, skb);
3478 
3479                 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3480                         if (unlikely(contended)) {
3481                                 spin_unlock(&q->busylock);
3482                                 contended = false;
3483                         }
3484                         __qdisc_run(q);
3485                 }
3486 
3487                 qdisc_run_end(q);
3488                 rc = NET_XMIT_SUCCESS;
3489         } else {
3490                 rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
3491                 if (qdisc_run_begin(q)) {
3492                         if (unlikely(contended)) {
3493                                 spin_unlock(&q->busylock);
3494                                 contended = false;
3495                         }
3496                         __qdisc_run(q);
3497                         qdisc_run_end(q);
3498                 }
3499         }
3500         spin_unlock(root_lock);
3501         if (unlikely(to_free))
3502                 kfree_skb_list(to_free);
3503         if (unlikely(contended))
3504                 spin_unlock(&q->busylock);
3505         return rc;
3506 }
3507 
3508 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3509 static void skb_update_prio(struct sk_buff *skb)
3510 {
3511         const struct netprio_map *map;
3512         const struct sock *sk;
3513         unsigned int prioidx;
3514 
3515         if (skb->priority)
3516                 return;
3517         map = rcu_dereference_bh(skb->dev->priomap);
3518         if (!map)
3519                 return;
3520         sk = skb_to_full_sk(skb);
3521         if (!sk)
3522                 return;
3523 
3524         prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3525 
3526         if (prioidx < map->priomap_len)
3527                 skb->priority = map->priomap[prioidx];
3528 }
3529 #else
3530 #define skb_update_prio(skb)
3531 #endif
3532 
3533 DEFINE_PER_CPU(int, xmit_recursion);
3534 EXPORT_SYMBOL(xmit_recursion);
3535 
3536 /**
3537  *      dev_loopback_xmit - loop back @skb
3538  *      @net: network namespace this loopback is happening in
3539  *      @sk:  sk needed to be a netfilter okfn
3540  *      @skb: buffer to transmit
3541  */
3542 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3543 {
3544         skb_reset_mac_header(skb);
3545         __skb_pull(skb, skb_network_offset(skb));
3546         skb->pkt_type = PACKET_LOOPBACK;
3547         skb->ip_summed = CHECKSUM_UNNECESSARY;
3548         WARN_ON(!skb_dst(skb));
3549         skb_dst_force(skb);
3550         netif_rx_ni(skb);
3551         return 0;
3552 }
3553 EXPORT_SYMBOL(dev_loopback_xmit);
3554 
3555 #ifdef CONFIG_NET_EGRESS
3556 static struct sk_buff *
3557 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3558 {
3559         struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress);
3560         struct tcf_result cl_res;
3561 
3562         if (!miniq)
3563                 return skb;
3564 
3565         /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
3566         mini_qdisc_bstats_cpu_update(miniq, skb);
3567 
3568         switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {
3569         case TC_ACT_OK:
3570         case TC_ACT_RECLASSIFY:
3571                 skb->tc_index = TC_H_MIN(cl_res.classid);
3572                 break;
3573         case TC_ACT_SHOT:
3574                 mini_qdisc_qstats_cpu_drop(miniq);
3575                 *ret = NET_XMIT_DROP;
3576                 kfree_skb(skb);
3577                 return NULL;
3578         case TC_ACT_STOLEN:
3579         case TC_ACT_QUEUED:
3580         case TC_ACT_TRAP:
3581                 *ret = NET_XMIT_SUCCESS;
3582                 consume_skb(skb);
3583                 return NULL;
3584         case TC_ACT_REDIRECT:
3585                 /* No need to push/pop skb's mac_header here on egress! */
3586                 skb_do_redirect(skb);
3587                 *ret = NET_XMIT_SUCCESS;
3588                 return NULL;
3589         default:
3590                 break;
3591         }
3592 
3593         return skb;
3594 }
3595 #endif /* CONFIG_NET_EGRESS */
3596 
3597 #ifdef CONFIG_XPS
3598 static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
3599                                struct xps_dev_maps *dev_maps, unsigned int tci)
3600 {
3601         struct xps_map *map;
3602         int queue_index = -1;
3603 
3604         if (dev->num_tc) {
3605                 tci *= dev->num_tc;
3606                 tci += netdev_get_prio_tc_map(dev, skb->priority);
3607         }
3608 
3609         map = rcu_dereference(dev_maps->attr_map[tci]);
3610         if (map) {
3611                 if (map->len == 1)
3612                         queue_index = map->queues[0];
3613                 else
3614                         queue_index = map->queues[reciprocal_scale(
3615                                                 skb_get_hash(skb), map->len)];
3616                 if (unlikely(queue_index >= dev->real_num_tx_queues))
3617                         queue_index = -1;
3618         }
3619         return queue_index;
3620 }
3621 #endif
3622 
3623 static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev,
3624                          struct sk_buff *skb)
3625 {
3626 #ifdef CONFIG_XPS
3627         struct xps_dev_maps *dev_maps;
3628         struct sock *sk = skb->sk;
3629         int queue_index = -1;
3630 
3631         if (!static_key_false(&xps_needed))
3632                 return -1;
3633 
3634         rcu_read_lock();
3635         if (!static_key_false(&xps_rxqs_needed))
3636                 goto get_cpus_map;
3637 
3638         dev_maps = rcu_dereference(sb_dev->xps_rxqs_map);
3639         if (dev_maps) {
3640                 int tci = sk_rx_queue_get(sk);
3641 
3642                 if (tci >= 0 && tci < dev->num_rx_queues)
3643                         queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
3644                                                           tci);
3645         }
3646 
3647 get_cpus_map:
3648         if (queue_index < 0) {
3649                 dev_maps = rcu_dereference(sb_dev->xps_cpus_map);
3650                 if (dev_maps) {
3651                         unsigned int tci = skb->sender_cpu - 1;
3652 
3653                         queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
3654                                                           tci);
3655                 }
3656         }
3657         rcu_read_unlock();
3658 
3659         return queue_index;
3660 #else
3661         return -1;
3662 #endif
3663 }
3664 
3665 u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
3666                      struct net_device *sb_dev,
3667                      select_queue_fallback_t fallback)
3668 {
3669         return 0;
3670 }
3671 EXPORT_SYMBOL(dev_pick_tx_zero);
3672 
3673 u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
3674                        struct net_device *sb_dev,
3675                        select_queue_fallback_t fallback)
3676 {
3677         return (u16)raw_smp_processor_id() % dev->real_num_tx_queues;
3678 }
3679 EXPORT_SYMBOL(dev_pick_tx_cpu_id);
3680 
3681 static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
3682                             struct net_device *sb_dev)
3683 {
3684         struct sock *sk = skb->sk;
3685         int queue_index = sk_tx_queue_get(sk);
3686 
3687         sb_dev = sb_dev ? : dev;
3688 
3689         if (queue_index < 0 || skb->ooo_okay ||
3690             queue_index >= dev->real_num_tx_queues) {
3691                 int new_index = get_xps_queue(dev, sb_dev, skb);
3692 
3693                 if (new_index < 0)
3694                         new_index = skb_tx_hash(dev, sb_dev, skb);
3695 
3696                 if (queue_index != new_index && sk &&
3697                     sk_fullsock(sk) &&
3698                     rcu_access_pointer(sk->sk_dst_cache))
3699                         sk_tx_queue_set(sk, new_index);
3700 
3701                 queue_index = new_index;
3702         }
3703 
3704         return queue_index;
3705 }
3706 
3707 struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3708                                     struct sk_buff *skb,
3709                                     struct net_device *sb_dev)
3710 {
3711         int queue_index = 0;
3712 
3713 #ifdef CONFIG_XPS
3714         u32 sender_cpu = skb->sender_cpu - 1;
3715 
3716         if (sender_cpu >= (u32)NR_CPUS)
3717                 skb->sender_cpu = raw_smp_processor_id() + 1;
3718 #endif
3719 
3720         if (dev->real_num_tx_queues != 1) {
3721                 const struct net_device_ops *ops = dev->netdev_ops;
3722 
3723                 if (ops->ndo_select_queue)
3724                         queue_index = ops->ndo_select_queue(dev, skb, sb_dev,
3725                                                             __netdev_pick_tx);
3726                 else
3727                         queue_index = __netdev_pick_tx(dev, skb, sb_dev);
3728 
3729                 queue_index = netdev_cap_txqueue(dev, queue_index);
3730         }
3731 
3732         skb_set_queue_mapping(skb, queue_index);
3733         return netdev_get_tx_queue(dev, queue_index);
3734 }
3735 
3736 /**
3737  *      __dev_queue_xmit - transmit a buffer
3738  *      @skb: buffer to transmit
3739  *      @sb_dev: suboordinate device used for L2 forwarding offload
3740  *
3741  *      Queue a buffer for transmission to a network device. The caller must
3742  *      have set the device and priority and built the buffer before calling
3743  *      this function. The function can be called from an interrupt.
3744  *
3745  *      A negative errno code is returned on a failure. A success does not
3746  *      guarantee the frame will be transmitted as it may be dropped due
3747  *      to congestion or traffic shaping.
3748  *
3749  * -----------------------------------------------------------------------------------
3750  *      I notice this method can also return errors from the queue disciplines,
3751  *      including NET_XMIT_DROP, which is a positive value.  So, errors can also
3752  *      be positive.
3753  *
3754  *      Regardless of the return value, the skb is consumed, so it is currently
3755  *      difficult to retry a send to this method.  (You can bump the ref count
3756  *      before sending to hold a reference for retry if you are careful.)
3757  *
3758  *      When calling this method, interrupts MUST be enabled.  This is because
3759  *      the BH enable code must have IRQs enabled so that it will not deadlock.
3760  *          --BLG
3761  */
3762 static int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
3763 {
3764         struct net_device *dev = skb->dev;
3765         struct netdev_queue *txq;
3766         struct Qdisc *q;
3767         int rc = -ENOMEM;
3768         bool again = false;
3769 
3770         skb_reset_mac_header(skb);
3771 
3772         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3773                 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3774 
3775         /* Disable soft irqs for various locks below. Also
3776          * stops preemption for RCU.
3777          */
3778         rcu_read_lock_bh();
3779 
3780         skb_update_prio(skb);
3781 
3782         qdisc_pkt_len_init(skb);
3783 #ifdef CONFIG_NET_CLS_ACT
3784         skb->tc_at_ingress = 0;
3785 # ifdef CONFIG_NET_EGRESS
3786         if (static_branch_unlikely(&egress_needed_key)) {
3787                 skb = sch_handle_egress(skb, &rc, dev);
3788                 if (!skb)
3789                         goto out;
3790         }
3791 # endif
3792 #endif
3793         /* If device/qdisc don't need skb->dst, release it right now while
3794          * its hot in this cpu cache.
3795          */
3796         if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3797                 skb_dst_drop(skb);
3798         else
3799                 skb_dst_force(skb);
3800 
3801         txq = netdev_pick_tx(dev, skb, sb_dev);
3802         q = rcu_dereference_bh(txq->qdisc);
3803 
3804         trace_net_dev_queue(skb);
3805         if (q->enqueue) {
3806                 rc = __dev_xmit_skb(skb, q, dev, txq);
3807                 goto out;
3808         }
3809 
3810         /* The device has no queue. Common case for software devices:
3811          * loopback, all the sorts of tunnels...
3812 
3813          * Really, it is unlikely that netif_tx_lock protection is necessary
3814          * here.  (f.e. loopback and IP tunnels are clean ignoring statistics
3815          * counters.)
3816          * However, it is possible, that they rely on protection
3817          * made by us here.
3818 
3819          * Check this and shot the lock. It is not prone from deadlocks.
3820          *Either shot noqueue qdisc, it is even simpler 8)
3821          */
3822         if (dev->flags & IFF_UP) {
3823                 int cpu = smp_processor_id(); /* ok because BHs are off */
3824 
3825                 if (txq->xmit_lock_owner != cpu) {
3826                         if (unlikely(__this_cpu_read(xmit_recursion) >
3827                                      XMIT_RECURSION_LIMIT))
3828                                 goto recursion_alert;
3829 
3830                         skb = validate_xmit_skb(skb, dev, &again);
3831                         if (!skb)
3832                                 goto out;
3833 
3834                         HARD_TX_LOCK(dev, txq, cpu);
3835 
3836                         if (!netif_xmit_stopped(txq)) {
3837                                 __this_cpu_inc(xmit_recursion);
3838                                 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
3839                                 __this_cpu_dec(xmit_recursion);
3840                                 if (dev_xmit_complete(rc)) {
3841                                         HARD_TX_UNLOCK(dev, txq);
3842                                         goto out;
3843                                 }
3844                         }
3845                         HARD_TX_UNLOCK(dev, txq);
3846                         net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3847                                              dev->name);
3848                 } else {
3849                         /* Recursion is detected! It is possible,
3850                          * unfortunately
3851                          */
3852 recursion_alert:
3853                         net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3854                                              dev->name);
3855                 }
3856         }
3857 
3858         rc = -ENETDOWN;
3859         rcu_read_unlock_bh();
3860 
3861         atomic_long_inc(&dev->tx_dropped);
3862         kfree_skb_list(skb);
3863         return rc;
3864 out:
3865         rcu_read_unlock_bh();
3866         return rc;
3867 }
3868 
3869 int dev_queue_xmit(struct sk_buff *skb)
3870 {
3871         return __dev_queue_xmit(skb, NULL);
3872 }
3873 EXPORT_SYMBOL(dev_queue_xmit);
3874 
3875 int dev_queue_xmit_accel(struct sk_buff *skb, struct net_device *sb_dev)
3876 {
3877         return __dev_queue_xmit(skb, sb_dev);
3878 }
3879 EXPORT_SYMBOL(dev_queue_xmit_accel);
3880 
3881 int dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
3882 {
3883         struct net_device *dev = skb->dev;
3884         struct sk_buff *orig_skb = skb;
3885         struct netdev_queue *txq;
3886         int ret = NETDEV_TX_BUSY;
3887         bool again = false;
3888 
3889         if (unlikely(!netif_running(dev) ||
3890                      !netif_carrier_ok(dev)))
3891                 goto drop;
3892 
3893         skb = validate_xmit_skb_list(skb, dev, &again);
3894         if (skb != orig_skb)
3895                 goto drop;
3896 
3897         skb_set_queue_mapping(skb, queue_id);
3898         txq = skb_get_tx_queue(dev, skb);
3899 
3900         local_bh_disable();
3901 
3902         HARD_TX_LOCK(dev, txq, smp_processor_id());
3903         if (!netif_xmit_frozen_or_drv_stopped(txq))
3904                 ret = netdev_start_xmit(skb, dev, txq, false);
3905         HARD_TX_UNLOCK(dev, txq);
3906 
3907         local_bh_enable();
3908 
3909         if (!dev_xmit_complete(ret))
3910                 kfree_skb(skb);
3911 
3912         return ret;
3913 drop:
3914         atomic_long_inc(&dev->tx_dropped);
3915         kfree_skb_list(skb);
3916         return NET_XMIT_DROP;
3917 }
3918 EXPORT_SYMBOL(dev_direct_xmit);
3919 
3920 /*************************************************************************
3921  *                      Receiver routines
3922  *************************************************************************/
3923 
3924 int netdev_max_backlog __read_mostly = 1000;
3925 EXPORT_SYMBOL(netdev_max_backlog);
3926 
3927 int netdev_tstamp_prequeue __read_mostly = 1;
3928 int netdev_budget __read_mostly = 300;
3929 unsigned int __read_mostly netdev_budget_usecs = 2000;
3930 int weight_p __read_mostly = 64;           /* old backlog weight */
3931 int dev_weight_rx_bias __read_mostly = 1;  /* bias for backlog weight */
3932 int dev_weight_tx_bias __read_mostly = 1;  /* bias for output_queue quota */
3933 int dev_rx_weight __read_mostly = 64;
3934 int dev_tx_weight __read_mostly = 64;
3935 
3936 /* Called with irq disabled */
3937 static inline void ____napi_schedule(struct softnet_data *sd,
3938                                      struct napi_struct *napi)
3939 {
3940         list_add_tail(&napi->poll_list, &sd->poll_list);
3941         __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3942 }
3943 
3944 #ifdef CONFIG_RPS
3945 
3946 /* One global table that all flow-based protocols share. */
3947 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
3948 EXPORT_SYMBOL(rps_sock_flow_table);
3949 u32 rps_cpu_mask __read_mostly;
3950 EXPORT_SYMBOL(rps_cpu_mask);
3951 
3952 struct static_key rps_needed __read_mostly;
3953 EXPORT_SYMBOL(rps_needed);
3954 struct static_key rfs_needed __read_mostly;
3955 EXPORT_SYMBOL(rfs_needed);
3956 
3957 static struct rps_dev_flow *
3958 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3959             struct rps_dev_flow *rflow, u16 next_cpu)
3960 {
3961         if (next_cpu < nr_cpu_ids) {
3962 #ifdef CONFIG_RFS_ACCEL
3963                 struct netdev_rx_queue *rxqueue;
3964                 struct rps_dev_flow_table *flow_table;
3965                 struct rps_dev_flow *old_rflow;
3966                 u32 flow_id;
3967                 u16 rxq_index;
3968                 int rc;
3969 
3970                 /* Should we steer this flow to a different hardware queue? */
3971                 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3972                     !(dev->features & NETIF_F_NTUPLE))
3973                         goto out;
3974                 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3975                 if (rxq_index == skb_get_rx_queue(skb))
3976                         goto out;
3977 
3978                 rxqueue = dev->_rx + rxq_index;
3979                 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3980                 if (!flow_table)
3981                         goto out;
3982                 flow_id = skb_get_hash(skb) & flow_table->mask;
3983                 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3984                                                         rxq_index, flow_id);
3985                 if (rc < 0)
3986                         goto out;
3987                 old_rflow = rflow;
3988                 rflow = &flow_table->flows[flow_id];
3989                 rflow->filter = rc;
3990                 if (old_rflow->filter == rflow->filter)
3991                         old_rflow->filter = RPS_NO_FILTER;
3992         out:
3993 #endif
3994                 rflow->last_qtail =
3995                         per_cpu(softnet_data, next_cpu).input_queue_head;
3996         }
3997 
3998         rflow->cpu = next_cpu;
3999         return rflow;
4000 }
4001 
4002 /*
4003  * get_rps_cpu is called from netif_receive_skb and returns the target
4004  * CPU from the RPS map of the receiving queue for a given skb.
4005  * rcu_read_lock must be held on entry.
4006  */
4007 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4008                        struct rps_dev_flow **rflowp)
4009 {
4010         const struct rps_sock_flow_table *sock_flow_table;
4011         struct netdev_rx_queue *rxqueue = dev->_rx;
4012         struct rps_dev_flow_table *flow_table;
4013         struct rps_map *map;
4014         int cpu = -1;
4015         u32 tcpu;
4016         u32 hash;
4017 
4018         if (skb_rx_queue_recorded(skb)) {
4019                 u16 index = skb_get_rx_queue(skb);
4020 
4021                 if (unlikely(index >= dev->real_num_rx_queues)) {
4022                         WARN_ONCE(dev->real_num_rx_queues > 1,
4023                                   "%s received packet on queue %u, but number "
4024                                   "of RX queues is %u\n",
4025                                   dev->name, index, dev->real_num_rx_queues);
4026                         goto done;
4027                 }
4028                 rxqueue += index;
4029         }
4030 
4031         /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
4032 
4033         flow_table = rcu_dereference(rxqueue->rps_flow_table);
4034         map = rcu_dereference(rxqueue->rps_map);
4035         if (!flow_table && !map)
4036                 goto done;
4037 
4038         skb_reset_network_header(skb);
4039         hash = skb_get_hash(skb);
4040         if (!hash)
4041                 goto done;
4042 
4043         sock_flow_table = rcu_dereference(rps_sock_flow_table);
4044         if (flow_table && sock_flow_table) {
4045                 struct rps_dev_flow *rflow;
4046                 u32 next_cpu;
4047                 u32 ident;
4048 
4049                 /* First check into global flow table if there is a match */
4050                 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
4051                 if ((ident ^ hash) & ~rps_cpu_mask)
4052                         goto try_rps;
4053 
4054                 next_cpu = ident & rps_cpu_mask;
4055 
4056                 /* OK, now we know there is a match,
4057                  * we can look at the local (per receive queue) flow table
4058                  */
4059                 rflow = &flow_table->flows[hash & flow_table->mask];
4060                 tcpu = rflow->cpu;
4061 
4062                 /*
4063                  * If the desired CPU (where last recvmsg was done) is
4064                  * different from current CPU (one in the rx-queue flow
4065                  * table entry), switch if one of the following holds:
4066                  *   - Current CPU is unset (>= nr_cpu_ids).
4067                  *   - Current CPU is offline.
4068                  *   - The current CPU's queue tail has advanced beyond the
4069                  *     last packet that was enqueued using this table entry.
4070                  *     This guarantees that all previous packets for the flow
4071                  *     have been dequeued, thus preserving in order delivery.
4072                  */
4073                 if (unlikely(tcpu != next_cpu) &&
4074                     (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
4075                      ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
4076                       rflow->last_qtail)) >= 0)) {
4077                         tcpu = next_cpu;
4078                         rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
4079                 }
4080 
4081                 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
4082                         *rflowp = rflow;
4083                         cpu = tcpu;
4084                         goto done;
4085                 }
4086         }
4087 
4088 try_rps:
4089 
4090         if (map) {
4091                 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
4092                 if (cpu_online(tcpu)) {
4093                         cpu = tcpu;
4094                         goto done;
4095                 }
4096         }
4097 
4098 done:
4099         return cpu;
4100 }
4101 
4102 #ifdef CONFIG_RFS_ACCEL
4103 
4104 /**
4105  * rps_may_expire_flow - check whether an RFS hardware filter may be removed
4106  * @dev: Device on which the filter was set
4107  * @rxq_index: RX queue index
4108  * @flow_id: Flow ID passed to ndo_rx_flow_steer()
4109  * @filter_id: Filter ID returned by ndo_rx_flow_steer()
4110  *
4111  * Drivers that implement ndo_rx_flow_steer() should periodically call
4112  * this function for each installed filter and remove the filters for
4113  * which it returns %true.
4114  */
4115 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
4116                          u32 flow_id, u16 filter_id)
4117 {
4118         struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
4119         struct rps_dev_flow_table *flow_table;
4120         struct rps_dev_flow *rflow;
4121         bool expire = true;
4122         unsigned int cpu;
4123 
4124         rcu_read_lock();
4125         flow_table = rcu_dereference(rxqueue->rps_flow_table);
4126         if (flow_table && flow_id <= flow_table->mask) {
4127                 rflow = &flow_table->flows[flow_id];
4128                 cpu = READ_ONCE(rflow->cpu);
4129                 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
4130                     ((int)(per_cpu(softnet_data, cpu).input_queue_head -
4131                            rflow->last_qtail) <
4132                      (int)(10 * flow_table->mask)))
4133                         expire = false;
4134         }
4135         rcu_read_unlock();
4136         return expire;
4137 }
4138 EXPORT_SYMBOL(rps_may_expire_flow);
4139 
4140 #endif /* CONFIG_RFS_ACCEL */
4141 
4142 /* Called from hardirq (IPI) context */
4143 static void rps_trigger_softirq(void *data)
4144 {
4145         struct softnet_data *sd = data;
4146 
4147         ____napi_schedule(sd, &sd->backlog);
4148         sd->received_rps++;
4149 }
4150 
4151 #endif /* CONFIG_RPS */
4152 
4153 /*
4154  * Check if this softnet_data structure is another cpu one
4155  * If yes, queue it to our IPI list and return 1
4156  * If no, return 0
4157  */
4158 static int rps_ipi_queued(struct softnet_data *sd)
4159 {
4160 #ifdef CONFIG_RPS
4161         struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
4162 
4163         if (sd != mysd) {
4164                 sd->rps_ipi_next = mysd->rps_ipi_list;
4165                 mysd->rps_ipi_list = sd;
4166 
4167                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4168                 return 1;
4169         }
4170 #endif /* CONFIG_RPS */
4171         return 0;
4172 }
4173 
4174 #ifdef CONFIG_NET_FLOW_LIMIT
4175 int netdev_flow_limit_table_len __read_mostly = (1 << 12);
4176 #endif
4177 
4178 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
4179 {
4180 #ifdef CONFIG_NET_FLOW_LIMIT
4181         struct sd_flow_limit *fl;
4182         struct softnet_data *sd;
4183         unsigned int old_flow, new_flow;
4184 
4185         if (qlen < (netdev_max_backlog >> 1))
4186                 return false;
4187 
4188         sd = this_cpu_ptr(&softnet_data);
4189 
4190         rcu_read_lock();
4191         fl = rcu_dereference(sd->flow_limit);
4192         if (fl) {
4193                 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
4194                 old_flow = fl->history[fl->history_head];
4195                 fl->history[fl->history_head] = new_flow;
4196 
4197                 fl->history_head++;
4198                 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
4199 
4200                 if (likely(fl->buckets[old_flow]))
4201                         fl->buckets[old_flow]--;
4202 
4203                 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
4204                         fl->count++;
4205                         rcu_read_unlock();
4206                         return true;
4207                 }
4208         }
4209         rcu_read_unlock();
4210 #endif
4211         return false;
4212 }
4213 
4214 /*
4215  * enqueue_to_backlog is called to queue an skb to a per CPU backlog
4216  * queue (may be a remote CPU queue).
4217  */
4218 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
4219                               unsigned int *qtail)
4220 {
4221         struct softnet_data *sd;
4222         unsigned long flags;
4223         unsigned int qlen;
4224 
4225         sd = &per_cpu(softnet_data, cpu);
4226 
4227         local_irq_save(flags);
4228 
4229         rps_lock(sd);
4230         if (!netif_running(skb->dev))
4231                 goto drop;
4232         qlen = skb_queue_len(&sd->input_pkt_queue);
4233         if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
4234                 if (qlen) {
4235 enqueue:
4236                         __skb_queue_tail(&sd->input_pkt_queue, skb);
4237                         input_queue_tail_incr_save(sd, qtail);
4238                         rps_unlock(sd);
4239                         local_irq_restore(flags);
4240                         return NET_RX_SUCCESS;
4241                 }
4242 
4243                 /* Schedule NAPI for backlog device
4244                  * We can use non atomic operation since we own the queue lock
4245                  */
4246                 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
4247                         if (!rps_ipi_queued(sd))
4248                                 ____napi_schedule(sd, &sd->backlog);
4249                 }
4250                 goto enqueue;
4251         }
4252 
4253 drop:
4254         sd->dropped++;
4255         rps_unlock(sd);
4256 
4257         local_irq_restore(flags);
4258 
4259         atomic_long_inc(&skb->dev->rx_dropped);
4260         kfree_skb(skb);
4261         return NET_RX_DROP;
4262 }
4263 
4264 static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
4265 {
4266         struct net_device *dev = skb->dev;
4267         struct netdev_rx_queue *rxqueue;
4268 
4269         rxqueue = dev->_rx;
4270 
4271         if (skb_rx_queue_recorded(skb)) {
4272                 u16 index = skb_get_rx_queue(skb);
4273 
4274                 if (unlikely(index >= dev->real_num_rx_queues)) {
4275                         WARN_ONCE(dev->real_num_rx_queues > 1,
4276                                   "%s received packet on queue %u, but number "
4277                                   "of RX queues is %u\n",
4278                                   dev->name, index, dev->real_num_rx_queues);
4279 
4280                         return rxqueue; /* Return first rxqueue */
4281                 }
4282                 rxqueue += index;
4283         }
4284         return rxqueue;
4285 }
4286 
4287 static u32 netif_receive_generic_xdp(struct sk_buff *skb,
4288                                      struct xdp_buff *xdp,
4289                                      struct bpf_prog *xdp_prog)
4290 {
4291         struct netdev_rx_queue *rxqueue;
4292         void *orig_data, *orig_data_end;
4293         u32 metalen, act = XDP_DROP;
4294         __be16 orig_eth_type;
4295         struct ethhdr *eth;
4296         bool orig_bcast;
4297         int hlen, off;
4298         u32 mac_len;
4299 
4300         /* Reinjected packets coming from act_mirred or similar should
4301          * not get XDP generic processing.
4302          */
4303         if (skb_cloned(skb) || skb_is_tc_redirected(skb))
4304                 return XDP_PASS;
4305 
4306         /* XDP packets must be linear and must have sufficient headroom
4307          * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also
4308          * native XDP provides, thus we need to do it here as well.
4309          */
4310         if (skb_is_nonlinear(skb) ||
4311             skb_headroom(skb) < XDP_PACKET_HEADROOM) {
4312                 int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
4313                 int troom = skb->tail + skb->data_len - skb->end;
4314 
4315                 /* In case we have to go down the path and also linearize,
4316                  * then lets do the pskb_expand_head() work just once here.
4317                  */
4318                 if (pskb_expand_head(skb,
4319                                      hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
4320                                      troom > 0 ? troom + 128 : 0, GFP_ATOMIC))
4321                         goto do_drop;
4322                 if (skb_linearize(skb))
4323                         goto do_drop;
4324         }
4325 
4326         /* The XDP program wants to see the packet starting at the MAC
4327          * header.
4328          */
4329         mac_len = skb->data - skb_mac_header(skb);
4330         hlen = skb_headlen(skb) + mac_len;
4331         xdp->data = skb->data - mac_len;
4332         xdp->data_meta = xdp->data;
4333         xdp->data_end = xdp->data + hlen;
4334         xdp->data_hard_start = skb->data - skb_headroom(skb);
4335         orig_data_end = xdp->data_end;
4336         orig_data = xdp->data;
4337         eth = (struct ethhdr *)xdp->data;
4338         orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
4339         orig_eth_type = eth->h_proto;
4340 
4341         rxqueue = netif_get_rxqueue(skb);
4342         xdp->rxq = &rxqueue->xdp_rxq;
4343 
4344         act = bpf_prog_run_xdp(xdp_prog, xdp);
4345 
4346         off = xdp->data - orig_data;
4347         if (off > 0)
4348                 __skb_pull(skb, off);
4349         else if (off < 0)
4350                 __skb_push(skb, -off);
4351         skb->mac_header += off;
4352 
4353         /* check if bpf_xdp_adjust_tail was used. it can only "shrink"
4354          * pckt.
4355          */
4356         off = orig_data_end - xdp->data_end;
4357         if (off != 0) {
4358                 skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
4359                 skb->len -= off;
4360 
4361         }
4362 
4363         /* check if XDP changed eth hdr such SKB needs update */
4364         eth = (struct ethhdr *)xdp->data;
4365         if ((orig_eth_type != eth->h_proto) ||
4366             (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
4367                 __skb_push(skb, ETH_HLEN);
4368                 skb->protocol = eth_type_trans(skb, skb->dev);
4369         }
4370 
4371         switch (act) {
4372         case XDP_REDIRECT:
4373         case XDP_TX:
4374                 __skb_push(skb, mac_len);
4375                 break;
4376         case XDP_PASS:
4377                 metalen = xdp->data - xdp->data_meta;
4378                 if (metalen)
4379                         skb_metadata_set(skb, metalen);
4380                 break;
4381         default:
4382                 bpf_warn_invalid_xdp_action(act);
4383                 /* fall through */
4384         case XDP_ABORTED:
4385                 trace_xdp_exception(skb->dev, xdp_prog, act);
4386                 /* fall through */
4387         case XDP_DROP:
4388         do_drop:
4389                 kfree_skb(skb);
4390                 break;
4391         }
4392 
4393         return act;
4394 }
4395 
4396 /* When doing generic XDP we have to bypass the qdisc layer and the
4397  * network taps in order to match in-driver-XDP behavior.
4398  */
4399 void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
4400 {
4401         struct net_device *dev = skb->dev;
4402         struct netdev_queue *txq;
4403         bool free_skb = true;
4404         int cpu, rc;
4405 
4406         txq = netdev_pick_tx(dev, skb, NULL);
4407         cpu = smp_processor_id();
4408         HARD_TX_LOCK(dev, txq, cpu);
4409         if (!netif_xmit_stopped(txq)) {
4410                 rc = netdev_start_xmit(skb, dev, txq, 0);
4411                 if (dev_xmit_complete(rc))
4412                         free_skb = false;
4413         }
4414         HARD_TX_UNLOCK(dev, txq);
4415         if (free_skb) {
4416                 trace_xdp_exception(dev, xdp_prog, XDP_TX);
4417                 kfree_skb(skb);
4418         }
4419 }
4420 EXPORT_SYMBOL_GPL(generic_xdp_tx);
4421 
4422 static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
4423 
4424 int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
4425 {
4426         if (xdp_prog) {
4427                 struct xdp_buff xdp;
4428                 u32 act;
4429                 int err;
4430 
4431                 act = netif_receive_generic_xdp(skb, &xdp, xdp_prog);
4432                 if (act != XDP_PASS) {
4433                         switch (act) {
4434                         case XDP_REDIRECT:
4435                                 err = xdp_do_generic_redirect(skb->dev, skb,
4436                                                               &xdp, xdp_prog);
4437                                 if (err)
4438                                         goto out_redir;
4439                                 break;
4440                         case XDP_TX:
4441                                 generic_xdp_tx(skb, xdp_prog);
4442                                 break;
4443                         }
4444                         return XDP_DROP;
4445                 }
4446         }
4447         return XDP_PASS;
4448 out_redir:
4449         kfree_skb(skb);
4450         return XDP_DROP;
4451 }
4452 EXPORT_SYMBOL_GPL(do_xdp_generic);
4453 
4454 static int netif_rx_internal(struct sk_buff *skb)
4455 {
4456         int ret;
4457 
4458         net_timestamp_check(netdev_tstamp_prequeue, skb);
4459 
4460         trace_netif_rx(skb);
4461 
4462         if (static_branch_unlikely(&generic_xdp_needed_key)) {
4463                 int ret;
4464 
4465                 preempt_disable();
4466                 rcu_read_lock();
4467                 ret = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
4468                 rcu_read_unlock();
4469                 preempt_enable();
4470 
4471                 /* Consider XDP consuming the packet a success from
4472                  * the netdev point of view we do not want to count
4473                  * this as an error.
4474                  */
4475                 if (ret != XDP_PASS)
4476                         return NET_RX_SUCCESS;
4477         }
4478 
4479 #ifdef CONFIG_RPS
4480         if (static_key_false(&rps_needed)) {
4481                 struct rps_dev_flow voidflow, *rflow = &voidflow;
4482                 int cpu;
4483 
4484                 preempt_disable();
4485                 rcu_read_lock();
4486 
4487                 cpu = get_rps_cpu(skb->dev, skb, &rflow);
4488                 if (cpu < 0)
4489                         cpu = smp_processor_id();
4490 
4491                 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4492 
4493                 rcu_read_unlock();
4494                 preempt_enable();
4495         } else
4496 #endif
4497         {
4498                 unsigned int qtail;
4499 
4500                 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
4501                 put_cpu();
4502         }
4503         return ret;
4504 }
4505 
4506 /**
4507  *      netif_rx        -       post buffer to the network code
4508  *      @skb: buffer to post
4509  *
4510  *      This function receives a packet from a device driver and queues it for
4511  *      the upper (protocol) levels to process.  It always succeeds. The buffer
4512  *      may be dropped during processing for congestion control or by the
4513  *      protocol layers.
4514  *
4515  *      return values:
4516  *      NET_RX_SUCCESS  (no congestion)
4517  *      NET_RX_DROP     (packet was dropped)
4518  *
4519  */
4520 
4521 int netif_rx(struct sk_buff *skb)
4522 {
4523         trace_netif_rx_entry(skb);
4524 
4525         return netif_rx_internal(skb);
4526 }
4527 EXPORT_SYMBOL(netif_rx);
4528 
4529 int netif_rx_ni(struct sk_buff *skb)
4530 {
4531         int err;
4532 
4533         trace_netif_rx_ni_entry(skb);
4534 
4535         preempt_disable();
4536         err = netif_rx_internal(skb);
4537         if (local_softirq_pending())
4538                 do_softirq();
4539         preempt_enable();
4540 
4541         return err;
4542 }
4543 EXPORT_SYMBOL(netif_rx_ni);
4544 
4545 static __latent_entropy void net_tx_action(struct softirq_action *h)
4546 {
4547         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
4548 
4549         if (sd->completion_queue) {
4550                 struct sk_buff *clist;
4551 
4552                 local_irq_disable();
4553                 clist = sd->completion_queue;
4554                 sd->completion_queue = NULL;
4555                 local_irq_enable();
4556 
4557                 while (clist) {
4558                         struct sk_buff *skb = clist;
4559 
4560                         clist = clist->next;
4561 
4562                         WARN_ON(refcount_read(&skb->users));
4563                         if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
4564                                 trace_consume_skb(skb);
4565                         else
4566                                 trace_kfree_skb(skb, net_tx_action);
4567 
4568                         if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
4569                                 __kfree_skb(skb);
4570                         else
4571                                 __kfree_skb_defer(skb);
4572                 }
4573 
4574                 __kfree_skb_flush();
4575         }
4576 
4577         if (sd->output_queue) {
4578                 struct Qdisc *head;
4579 
4580                 local_irq_disable();
4581                 head = sd->output_queue;
4582                 sd->output_queue = NULL;
4583                 sd->output_queue_tailp = &sd->output_queue;
4584                 local_irq_enable();
4585 
4586                 while (head) {
4587                         struct Qdisc *q = head;
4588                         spinlock_t *root_lock = NULL;
4589 
4590                         head = head->next_sched;
4591 
4592                         if (!(q->flags & TCQ_F_NOLOCK)) {
4593                                 root_lock = qdisc_lock(q);
4594                                 spin_lock(root_lock);
4595                         }
4596                         /* We need to make sure head->next_sched is read
4597                          * before clearing __QDISC_STATE_SCHED
4598                          */
4599                         smp_mb__before_atomic();
4600                         clear_bit(__QDISC_STATE_SCHED, &q->state);
4601                         qdisc_run(q);
4602                         if (root_lock)
4603                                 spin_unlock(root_lock);
4604                 }
4605         }
4606 
4607         xfrm_dev_backlog(sd);
4608 }
4609 
4610 #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
4611 /* This hook is defined here for ATM LANE */
4612 int (*br_fdb_test_addr_hook)(struct net_device *dev,
4613                              unsigned char *addr) __read_mostly;
4614 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
4615 #endif
4616 
4617 static inline struct sk_buff *
4618 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
4619                    struct net_device *orig_dev)
4620 {
4621 #ifdef CONFIG_NET_CLS_ACT
4622         struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress);
4623         struct tcf_result cl_res;
4624 
4625         /* If there's at least one ingress present somewhere (so
4626          * we get here via enabled static key), remaining devices
4627          * that are not configured with an ingress qdisc will bail
4628          * out here.
4629          */
4630         if (!miniq)
4631                 return skb;
4632 
4633         if (*pt_prev) {
4634                 *ret = deliver_skb(skb, *pt_prev, orig_dev);
4635                 *pt_prev = NULL;
4636         }
4637 
4638         qdisc_skb_cb(skb)->pkt_len = skb->len;
4639         skb->tc_at_ingress = 1;
4640         mini_qdisc_bstats_cpu_update(miniq, skb);
4641 
4642         switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {
4643         case TC_ACT_OK:
4644         case TC_ACT_RECLASSIFY:
4645                 skb->tc_index = TC_H_MIN(cl_res.classid);
4646                 break;
4647         case TC_ACT_SHOT:
4648                 mini_qdisc_qstats_cpu_drop(miniq);
4649                 kfree_skb(skb);
4650                 return NULL;
4651         case TC_ACT_STOLEN:
4652         case TC_ACT_QUEUED:
4653         case TC_ACT_TRAP:
4654                 consume_skb(skb);
4655                 return NULL;
4656         case TC_ACT_REDIRECT:
4657                 /* skb_mac_header check was done by cls/act_bpf, so
4658                  * we can safely push the L2 header back before
4659                  * redirecting to another netdev
4660                  */
4661                 __skb_push(skb, skb->mac_len);
4662                 skb_do_redirect(skb);
4663                 return NULL;
4664         case TC_ACT_REINSERT:
4665                 /* this does not scrub the packet, and updates stats on error */
4666                 skb_tc_reinsert(skb, &cl_res);
4667                 return NULL;
4668         default:
4669                 break;
4670         }
4671 #endif /* CONFIG_NET_CLS_ACT */
4672         return skb;
4673 }
4674 
4675 /**
4676  *      netdev_is_rx_handler_busy - check if receive handler is registered
4677  *      @dev: device to check
4678  *
4679  *      Check if a receive handler is already registered for a given device.
4680  *      Return true if there one.
4681  *
4682  *      The caller must hold the rtnl_mutex.
4683  */
4684 bool netdev_is_rx_handler_busy(struct net_device *dev)
4685 {
4686         ASSERT_RTNL();
4687         return dev && rtnl_dereference(dev->rx_handler);
4688 }
4689 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
4690 
4691 /**
4692  *      netdev_rx_handler_register - register receive handler
4693  *      @dev: device to register a handler for
4694  *      @rx_handler: receive handler to register
4695  *      @rx_handler_data: data pointer that is used by rx handler
4696  *
4697  *      Register a receive handler for a device. This handler will then be
4698  *      called from __netif_receive_skb. A negative errno code is returned
4699  *      on a failure.
4700  *
4701  *      The caller must hold the rtnl_mutex.
4702  *
4703  *      For a general description of rx_handler, see enum rx_handler_result.
4704  */
4705 int netdev_rx_handler_register(struct net_device *dev,
4706                                rx_handler_func_t *rx_handler,
4707                                void *rx_handler_data)
4708 {
4709         if (netdev_is_rx_handler_busy(dev))
4710                 return -EBUSY;
4711 
4712         if (dev->priv_flags & IFF_NO_RX_HANDLER)
4713                 return -EINVAL;
4714 
4715         /* Note: rx_handler_data must be set before rx_handler */
4716         rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
4717         rcu_assign_pointer(dev->rx_handler, rx_handler);
4718 
4719         return 0;
4720 }
4721 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
4722 
4723 /**
4724  *      netdev_rx_handler_unregister - unregister receive handler
4725  *      @dev: device to unregister a handler from
4726  *
4727  *      Unregister a receive handler from a device.
4728  *
4729  *      The caller must hold the rtnl_mutex.
4730  */
4731 void netdev_rx_handler_unregister(struct net_device *dev)
4732 {
4733 
4734         ASSERT_RTNL();
4735         RCU_INIT_POINTER(dev->rx_handler, NULL);
4736         /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
4737          * section has a guarantee to see a non NULL rx_handler_data
4738          * as well.
4739          */
4740         synchronize_net();
4741         RCU_INIT_POINTER(dev->rx_handler_data, NULL);
4742 }
4743 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
4744 
4745 /*
4746  * Limit the use of PFMEMALLOC reserves to those protocols that implement
4747  * the special handling of PFMEMALLOC skbs.
4748  */
4749 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
4750 {
4751         switch (skb->protocol) {
4752         case htons(ETH_P_ARP):
4753         case htons(ETH_P_IP):
4754         case htons(ETH_P_IPV6):
4755         case htons(ETH_P_8021Q):
4756         case htons(ETH_P_8021AD):
4757                 return true;
4758         default:
4759                 return false;
4760         }
4761 }
4762 
4763 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
4764                              int *ret, struct net_device *orig_dev)
4765 {
4766 #ifdef CONFIG_NETFILTER_INGRESS
4767         if (nf_hook_ingress_active(skb)) {
4768                 int ingress_retval;
4769 
4770                 if (*pt_prev) {
4771                         *ret = deliver_skb(skb, *pt_prev, orig_dev);
4772                         *pt_prev = NULL;
4773                 }
4774 
4775                 rcu_read_lock();
4776                 ingress_retval = nf_hook_ingress(skb);
4777                 rcu_read_unlock();
4778                 return ingress_retval;
4779         }
4780 #endif /* CONFIG_NETFILTER_INGRESS */
4781         return 0;
4782 }
4783 
4784 static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc,
4785                                     struct packet_type **ppt_prev)
4786 {
4787         struct packet_type *ptype, *pt_prev;
4788         rx_handler_func_t *rx_handler;
4789         struct net_device *orig_dev;
4790         bool deliver_exact = false;
4791         int ret = NET_RX_DROP;
4792         __be16 type;
4793 
4794         net_timestamp_check(!netdev_tstamp_prequeue, skb);
4795 
4796         trace_netif_receive_skb(skb);
4797 
4798         orig_dev = skb->dev;
4799 
4800         skb_reset_network_header(skb);
4801         if (!skb_transport_header_was_set(skb))
4802                 skb_reset_transport_header(skb);
4803         skb_reset_mac_len(skb);
4804 
4805         pt_prev = NULL;
4806 
4807 another_round:
4808         skb->skb_iif = skb->dev->ifindex;
4809 
4810         __this_cpu_inc(softnet_data.processed);
4811 
4812         if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
4813             skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
4814                 skb = skb_vlan_untag(skb);
4815                 if (unlikely(!skb))
4816                         goto out;
4817         }
4818 
4819         if (skb_skip_tc_classify(skb))
4820                 goto skip_classify;
4821 
4822         if (pfmemalloc)
4823                 goto skip_taps;
4824 
4825         list_for_each_entry_rcu(ptype, &ptype_all, list) {
4826                 if (pt_prev)
4827                         ret = deliver_skb(skb, pt_prev, orig_dev);
4828                 pt_prev = ptype;
4829         }
4830 
4831         list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
4832                 if (pt_prev)
4833                         ret = deliver_skb(skb, pt_prev, orig_dev);
4834                 pt_prev = ptype;
4835         }
4836 
4837 skip_taps:
4838 #ifdef CONFIG_NET_INGRESS
4839         if (static_branch_unlikely(&ingress_needed_key)) {
4840                 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
4841                 if (!skb)
4842                         goto out;
4843 
4844                 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
4845                         goto out;
4846         }
4847 #endif
4848         skb_reset_tc(skb);
4849 skip_classify:
4850         if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
4851                 goto drop;
4852 
4853         if (skb_vlan_tag_present(skb)) {
4854                 if (pt_prev) {
4855                         ret = deliver_skb(skb, pt_prev, orig_dev);
4856                         pt_prev = NULL;
4857                 }
4858                 if (vlan_do_receive(&skb))
4859                         goto another_round;
4860                 else if (unlikely(!skb))
4861                         goto out;
4862         }
4863 
4864         rx_handler = rcu_dereference(skb->dev->rx_handler);
4865         if (rx_handler) {
4866                 if (pt_prev) {
4867                         ret = deliver_skb(skb, pt_prev, orig_dev);
4868                         pt_prev = NULL;
4869                 }
4870                 switch (rx_handler(&skb)) {
4871                 case RX_HANDLER_CONSUMED:
4872                         ret = NET_RX_SUCCESS;
4873                         goto out;
4874                 case RX_HANDLER_ANOTHER:
4875                         goto another_round;
4876                 case RX_HANDLER_EXACT:
4877                         deliver_exact = true;
4878                 case RX_HANDLER_PASS:
4879                         break;
4880                 default:
4881                         BUG();
4882                 }
4883         }
4884 
4885         if (unlikely(skb_vlan_tag_present(skb))) {
4886                 if (skb_vlan_tag_get_id(skb))
4887                         skb->pkt_type = PACKET_OTHERHOST;
4888                 /* Note: we might in the future use prio bits
4889                  * and set skb->priority like in vlan_do_receive()
4890                  * For the time being, just ignore Priority Code Point
4891                  */
4892                 skb->vlan_tci = 0;
4893         }
4894 
4895         type = skb->protocol;
4896 
4897         /* deliver only exact match when indicated */
4898         if (likely(!deliver_exact)) {
4899                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4900                                        &ptype_base[ntohs(type) &
4901                                                    PTYPE_HASH_MASK]);
4902         }
4903 
4904         deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4905                                &orig_dev->ptype_specific);
4906 
4907         if (unlikely(skb->dev != orig_dev)) {
4908                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4909                                        &skb->dev->ptype_specific);
4910         }
4911 
4912         if (pt_prev) {
4913                 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
4914                         goto drop;
4915                 *ppt_prev = pt_prev;
4916         } else {
4917 drop:
4918                 if (!deliver_exact)
4919                         atomic_long_inc(&skb->dev->rx_dropped);
4920                 else
4921                         atomic_long_inc(&skb->dev->rx_nohandler);
4922                 kfree_skb(skb);
4923                 /* Jamal, now you will not able to escape explaining
4924                  * me how you were going to use this. :-)
4925                  */
4926                 ret = NET_RX_DROP;
4927         }
4928 
4929 out:
4930         return ret;
4931 }
4932 
4933 static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
4934 {
4935         struct net_device *orig_dev = skb->dev;
4936         struct packet_type *pt_prev = NULL;
4937         int ret;
4938 
4939         ret = __netif_receive_skb_core(skb, pfmemalloc, &pt_prev);
4940         if (pt_prev)
4941                 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
4942         return ret;
4943 }
4944 
4945 /**
4946  *      netif_receive_skb_core - special purpose version of netif_receive_skb
4947  *      @skb: buffer to process
4948  *
4949  *      More direct receive version of netif_receive_skb().  It should
4950  *      only be used by callers that have a need to skip RPS and Generic XDP.
4951  *      Caller must also take care of handling if (page_is_)pfmemalloc.
4952  *
4953  *      This function may only be called from softirq context and interrupts
4954  *      should be enabled.
4955  *
4956  *      Return values (usually ignored):
4957  *      NET_RX_SUCCESS: no congestion
4958  *      NET_RX_DROP: packet was dropped
4959  */
4960 int netif_receive_skb_core(struct sk_buff *skb)
4961 {
4962         int ret;
4963 
4964         rcu_read_lock();
4965         ret = __netif_receive_skb_one_core(skb, false);
4966         rcu_read_unlock();
4967 
4968         return ret;
4969 }
4970 EXPORT_SYMBOL(netif_receive_skb_core);
4971 
4972 static inline void __netif_receive_skb_list_ptype(struct list_head *head,
4973                                                   struct packet_type *pt_prev,
4974                                                   struct net_device *orig_dev)
4975 {
4976         struct sk_buff *skb, *next;
4977 
4978         if (!pt_prev)
4979                 return;
4980         if (list_empty(head))
4981                 return;
4982         if (pt_prev->list_func != NULL)
4983                 pt_prev->list_func(head, pt_prev, orig_dev);
4984         else
4985                 list_for_each_entry_safe(skb, next, head, list)
4986                         pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
4987 }
4988 
4989 static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc)
4990 {
4991         /* Fast-path assumptions:
4992          * - There is no RX handler.
4993          * - Only one packet_type matches.
4994          * If either of these fails, we will end up doing some per-packet
4995          * processing in-line, then handling the 'last ptype' for the whole
4996          * sublist.  This can't cause out-of-order delivery to any single ptype,
4997          * because the 'last ptype' must be constant across the sublist, and all
4998          * other ptypes are handled per-packet.
4999          */
5000         /* Current (common) ptype of sublist */
5001         struct packet_type *pt_curr = NULL;
5002         /* Current (common) orig_dev of sublist */
5003         struct net_device *od_curr = NULL;
5004         struct list_head sublist;
5005         struct sk_buff *skb, *next;
5006 
5007         INIT_LIST_HEAD(&sublist);
5008         list_for_each_entry_safe(skb, next, head, list) {
5009                 struct net_device *orig_dev = skb->dev;
5010                 struct packet_type *pt_prev = NULL;
5011 
5012                 list_del(&skb->list);
5013                 __netif_receive_skb_core(skb, pfmemalloc, &pt_prev);
5014                 if (!pt_prev)
5015                         continue;
5016                 if (pt_curr != pt_prev || od_curr != orig_dev) {
5017                         /* dispatch old sublist */
5018                         __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5019                         /* start new sublist */
5020                         INIT_LIST_HEAD(&sublist);
5021                         pt_curr = pt_prev;
5022                         od_curr = orig_dev;
5023                 }
5024                 list_add_tail(&skb->list, &sublist);
5025         }
5026 
5027         /* dispatch final sublist */
5028         __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5029 }
5030 
5031 static int __netif_receive_skb(struct sk_buff *skb)
5032 {
5033         int ret;
5034 
5035         if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
5036                 unsigned int noreclaim_flag;
5037 
5038                 /*
5039                  * PFMEMALLOC skbs are special, they should
5040                  * - be delivered to SOCK_MEMALLOC sockets only
5041                  * - stay away from userspace
5042                  * - have bounded memory usage
5043                  *
5044                  * Use PF_MEMALLOC as this saves us from propagating the allocation
5045                  * context down to all allocation sites.
5046                  */
5047                 noreclaim_flag = memalloc_noreclaim_save();
5048                 ret = __netif_receive_skb_one_core(skb, true);
5049                 memalloc_noreclaim_restore(noreclaim_flag);
5050         } else
5051                 ret = __netif_receive_skb_one_core(skb, false);
5052 
5053         return ret;
5054 }
5055 
5056 static void __netif_receive_skb_list(struct list_head *head)
5057 {
5058         unsigned long noreclaim_flag = 0;
5059         struct sk_buff *skb, *next;
5060         bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
5061 
5062         list_for_each_entry_safe(skb, next, head, list) {
5063                 if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) {
5064                         struct list_head sublist;
5065 
5066                         /* Handle the previous sublist */
5067                         list_cut_before(&sublist, head, &skb->list);
5068                         if (!list_empty(&sublist))
5069                                 __netif_receive_skb_list_core(&sublist, pfmemalloc);
5070                         pfmemalloc = !pfmemalloc;
5071                         /* See comments in __netif_receive_skb */
5072                         if (pfmemalloc)
5073                                 noreclaim_flag = memalloc_noreclaim_save();
5074                         else
5075                                 memalloc_noreclaim_restore(noreclaim_flag);
5076                 }
5077         }
5078         /* Handle the remaining sublist */
5079         if (!list_empty(head))
5080                 __netif_receive_skb_list_core(head, pfmemalloc);
5081         /* Restore pflags */
5082         if (pfmemalloc)
5083                 memalloc_noreclaim_restore(noreclaim_flag);
5084 }
5085 
5086 static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
5087 {
5088         struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
5089         struct bpf_prog *new = xdp->prog;
5090         int ret = 0;
5091 
5092         switch (xdp->command) {
5093         case XDP_SETUP_PROG:
5094                 rcu_assign_pointer(dev->xdp_prog, new);
5095                 if (old)
5096                         bpf_prog_put(old);
5097 
5098                 if (old && !new) {
5099                         static_branch_dec(&generic_xdp_needed_key);
5100                 } else if (new && !old) {
5101                         static_branch_inc(&generic_xdp_needed_key);
5102                         dev_disable_lro(dev);
5103                         dev_disable_gro_hw(dev);
5104                 }
5105                 break;
5106 
5107         case XDP_QUERY_PROG:
5108                 xdp->prog_id = old ? old->aux->id : 0;
5109                 break;
5110 
5111         default:
5112                 ret = -EINVAL;
5113                 break;
5114         }
5115 
5116         return ret;
5117 }
5118 
5119 static int netif_receive_skb_internal(struct sk_buff *skb)
5120 {
5121         int ret;
5122 
5123         net_timestamp_check(netdev_tstamp_prequeue, skb);
5124 
5125         if (skb_defer_rx_timestamp(skb))
5126                 return NET_RX_SUCCESS;
5127 
5128         if (static_branch_unlikely(&generic_xdp_needed_key)) {
5129                 int ret;
5130 
5131                 preempt_disable();
5132                 rcu_read_lock();
5133                 ret = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
5134                 rcu_read_unlock();
5135                 preempt_enable();
5136 
5137                 if (ret != XDP_PASS)
5138                         return NET_RX_DROP;
5139         }
5140 
5141         rcu_read_lock();
5142 #ifdef CONFIG_RPS
5143         if (static_key_false(&rps_needed)) {
5144                 struct rps_dev_flow voidflow, *rflow = &voidflow;
5145                 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5146 
5147                 if (cpu >= 0) {
5148                         ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5149                         rcu_read_unlock();
5150                         return ret;
5151                 }
5152         }
5153 #endif
5154         ret = __netif_receive_skb(skb);
5155         rcu_read_unlock();
5156         return ret;
5157 }
5158 
5159 static void netif_receive_skb_list_internal(struct list_head *head)
5160 {
5161         struct bpf_prog *xdp_prog = NULL;
5162         struct sk_buff *skb, *next;
5163         struct list_head sublist;
5164 
5165         INIT_LIST_HEAD(&sublist);
5166         list_for_each_entry_safe(skb, next, head, list) {
5167                 net_timestamp_check(netdev_tstamp_prequeue, skb);
5168                 list_del(&skb->list);
5169                 if (!skb_defer_rx_timestamp(skb))
5170                         list_add_tail(&skb->list, &sublist);
5171         }
5172         list_splice_init(&sublist, head);
5173 
5174         if (static_branch_unlikely(&generic_xdp_needed_key)) {
5175                 preempt_disable();
5176                 rcu_read_lock();
5177                 list_for_each_entry_safe(skb, next, head, list) {
5178                         xdp_prog = rcu_dereference(skb->dev->xdp_prog);
5179                         list_del(&skb->list);
5180                         if (do_xdp_generic(xdp_prog, skb) == XDP_PASS)
5181                                 list_add_tail(&skb->list, &sublist);
5182                 }
5183                 rcu_read_unlock();
5184                 preempt_enable();
5185                 /* Put passed packets back on main list */
5186                 list_splice_init(&sublist, head);
5187         }
5188 
5189         rcu_read_lock();
5190 #ifdef CONFIG_RPS
5191         if (static_key_false(&rps_needed)) {
5192                 list_for_each_entry_safe(skb, next, head, list) {
5193                         struct rps_dev_flow voidflow, *rflow = &voidflow;
5194                         int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5195 
5196                         if (cpu >= 0) {
5197                                 /* Will be handled, remove from list */
5198                                 list_del(&skb->list);
5199                                 enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5200                         }
5201                 }
5202         }
5203 #endif
5204         __netif_receive_skb_list(head);
5205         rcu_read_unlock();
5206 }
5207 
5208 /**
5209  *      netif_receive_skb - process receive buffer from network
5210  *      @skb: buffer to process
5211  *
5212  *      netif_receive_skb() is the main receive data processing function.
5213  *      It always succeeds. The buffer may be dropped during processing
5214  *      for congestion control or by the protocol layers.
5215  *
5216  *      This function may only be called from softirq context and interrupts
5217  *      should be enabled.
5218  *
5219  *      Return values (usually ignored):
5220  *      NET_RX_SUCCESS: no congestion
5221  *      NET_RX_DROP: packet was dropped
5222  */
5223 int netif_receive_skb(struct sk_buff *skb)
5224 {
5225         trace_netif_receive_skb_entry(skb);
5226 
5227         return netif_receive_skb_internal(skb);
5228 }
5229 EXPORT_SYMBOL(netif_receive_skb);
5230 
5231 /**
5232  *      netif_receive_skb_list - process many receive buffers from network
5233  *      @head: list of skbs to process.
5234  *
5235  *      Since return value of netif_receive_skb() is normally ignored, and
5236  *      wouldn't be meaningful for a list, this function returns void.
5237  *
5238  *      This function may only be called from softirq context and interrupts
5239  *      should be enabled.
5240  */
5241 void netif_receive_skb_list(struct list_head *head)
5242 {
5243         struct sk_buff *skb;
5244 
5245         if (list_empty(head))
5246                 return;
5247         list_for_each_entry(skb, head, list)
5248                 trace_netif_receive_skb_list_entry(skb);
5249         netif_receive_skb_list_internal(head);
5250 }
5251 EXPORT_SYMBOL(netif_receive_skb_list);
5252 
5253 DEFINE_PER_CPU(struct work_struct, flush_works);
5254 
5255 /* Network device is going away, flush any packets still pending */
5256 static void flush_backlog(struct work_struct *work)
5257 {
5258         struct sk_buff *skb, *tmp;
5259         struct softnet_data *sd;
5260 
5261         local_bh_disable();
5262         sd = this_cpu_ptr(&softnet_data);
5263 
5264         local_irq_disable();
5265         rps_lock(sd);
5266         skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
5267                 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5268                         __skb_unlink(skb, &sd->input_pkt_queue);
5269                         kfree_skb(skb);
5270                         input_queue_head_incr(sd);
5271                 }
5272         }
5273         rps_unlock(sd);
5274         local_irq_enable();
5275 
5276         skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
5277                 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
5278                         __skb_unlink(skb, &sd->process_queue);
5279                         kfree_skb(skb);
5280                         input_queue_head_incr(sd);
5281                 }
5282         }
5283         local_bh_enable();
5284 }
5285 
5286 static void flush_all_backlogs(void)
5287 {
5288         unsigned int cpu;
5289 
5290         get_online_cpus();
5291 
5292         for_each_online_cpu(cpu)
5293                 queue_work_on(cpu, system_highpri_wq,
5294                               per_cpu_ptr(&flush_works, cpu));
5295 
5296         for_each_online_cpu(cpu)
5297                 flush_work(per_cpu_ptr(&flush_works, cpu));
5298 
5299         put_online_cpus();
5300 }
5301 
5302 static int napi_gro_complete(struct sk_buff *skb)
5303 {
5304         struct packet_offload *ptype;
5305         __be16 type = skb->protocol;
5306         struct list_head *head = &offload_base;
5307         int err = -ENOENT;
5308 
5309         BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
5310 
5311         if (NAPI_GRO_CB(skb)->count == 1) {
5312                 skb_shinfo(skb)->gso_size = 0;
5313                 goto out;
5314         }
5315 
5316         rcu_read_lock();
5317         list_for_each_entry_rcu(ptype, head, list) {
5318                 if (ptype->type != type || !ptype->callbacks.gro_complete)
5319                         continue;
5320 
5321                 err = ptype->callbacks.gro_complete(skb, 0);
5322                 break;
5323         }
5324         rcu_read_unlock();
5325 
5326         if (err) {
5327                 WARN_ON(&ptype->list == head);
5328                 kfree_skb(skb);
5329                 return NET_RX_SUCCESS;
5330         }
5331 
5332 out:
5333         return netif_receive_skb_internal(skb);
5334 }
5335 
5336 static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index,
5337                                    bool flush_old)
5338 {
5339         struct list_head *head = &napi->gro_hash[index].list;
5340         struct sk_buff *skb, *p;
5341 
5342         list_for_each_entry_safe_reverse(skb, p, head, list) {
5343                 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
5344                         return;
5345                 skb_list_del_init(skb);
5346                 napi_gro_complete(skb);
5347                 napi->gro_hash[index].count--;
5348         }
5349 
5350         if (!napi->gro_hash[index].count)
5351                 __clear_bit(index, &napi->gro_bitmask);
5352 }
5353 
5354 /* napi->gro_hash[].list contains packets ordered by age.
5355  * youngest packets at the head of it.
5356  * Complete skbs in reverse order to reduce latencies.
5357  */
5358 void napi_gro_flush(struct napi_struct *napi, bool flush_old)
5359 {
5360         u32 i;
5361 
5362         for (i = 0; i < GRO_HASH_BUCKETS; i++) {
5363                 if (test_bit(i, &napi->gro_bitmask))
5364                         __napi_gro_flush_chain(napi, i, flush_old);
5365         }
5366 }
5367 EXPORT_SYMBOL(napi_gro_flush);
5368 
5369 static struct list_head *gro_list_prepare(struct napi_struct *napi,
5370                                           struct sk_buff *skb)
5371 {
5372         unsigned int maclen = skb->dev->hard_header_len;
5373         u32 hash = skb_get_hash_raw(skb);
5374         struct list_head *head;
5375         struct sk_buff *p;
5376 
5377         head = &napi->gro_hash[hash & (GRO_HASH_BUCKETS - 1)].list;
5378         list_for_each_entry(p, head, list) {
5379                 unsigned long diffs;
5380 
5381                 NAPI_GRO_CB(p)->flush = 0;
5382 
5383                 if (hash != skb_get_hash_raw(p)) {
5384                         NAPI_GRO_CB(p)->same_flow = 0;
5385                         continue;
5386                 }
5387 
5388                 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
5389                 diffs |= p->vlan_tci ^ skb->vlan_tci;
5390                 diffs |= skb_metadata_dst_cmp(p, skb);
5391                 diffs |= skb_metadata_differs(p, skb);
5392                 if (maclen == ETH_HLEN)
5393                         diffs |= compare_ether_header(skb_mac_header(p),
5394                                                       skb_mac_header(skb));
5395                 else if (!diffs)
5396                         diffs = memcmp(skb_mac_header(p),
5397                                        skb_mac_header(skb),
5398                                        maclen);
5399                 NAPI_GRO_CB(p)->same_flow = !diffs;
5400         }
5401 
5402         return head;
5403 }
5404 
5405 static void skb_gro_reset_offset(struct sk_buff *skb)
5406 {
5407         const struct skb_shared_info *pinfo = skb_shinfo(skb);
5408         const skb_frag_t *frag0 = &pinfo->frags[0];
5409 
5410         NAPI_GRO_CB(skb)->data_offset = 0;
5411         NAPI_GRO_CB(skb)->frag0 = NULL;
5412         NAPI_GRO_CB(skb)->frag0_len = 0;
5413 
5414         if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
5415             pinfo->nr_frags &&
5416             !PageHighMem(skb_frag_page(frag0))) {
5417                 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
5418                 NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
5419                                                     skb_frag_size(frag0),
5420                                                     skb->end - skb->tail);
5421         }
5422 }
5423 
5424 static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
5425 {
5426         struct skb_shared_info *pinfo = skb_shinfo(skb);
5427 
5428         BUG_ON(skb->end - skb->tail < grow);
5429 
5430         memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
5431 
5432         skb->data_len -= grow;
5433         skb->tail += grow;
5434 
5435         pinfo->frags[0].page_offset += grow;
5436         skb_frag_size_sub(&pinfo->frags[0], grow);
5437 
5438         if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
5439                 skb_frag_unref(skb, 0);
5440                 memmove(pinfo->frags, pinfo->frags + 1,
5441                         --pinfo->nr_frags * sizeof(pinfo->frags[0]));
5442         }
5443 }
5444 
5445 static void gro_flush_oldest(struct list_head *head)
5446 {
5447         struct sk_buff *oldest;
5448 
5449         oldest = list_last_entry(head, struct sk_buff, list);
5450 
5451         /* We are called with head length >= MAX_GRO_SKBS, so this is
5452          * impossible.
5453          */
5454         if (WARN_ON_ONCE(!oldest))
5455                 return;
5456 
5457         /* Do not adjust napi->gro_hash[].count, caller is adding a new
5458          * SKB to the chain.
5459          */
5460         skb_list_del_init(oldest);
5461         napi_gro_complete(oldest);
5462 }
5463 
5464 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
5465 {
5466         u32 hash = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1);
5467         struct list_head *head = &offload_base;
5468         struct packet_offload *ptype;
5469         __be16 type = skb->protocol;
5470         struct list_head *gro_head;
5471         struct sk_buff *pp = NULL;
5472         enum gro_result ret;
5473         int same_flow;
5474         int grow;
5475 
5476         if (netif_elide_gro(skb->dev))
5477                 goto normal;
5478 
5479         gro_head = gro_list_prepare(napi, skb);
5480 
5481         rcu_read_lock();
5482         list_for_each_entry_rcu(ptype, head, list) {
5483                 if (ptype->type != type || !ptype->callbacks.gro_receive)
5484                         continue;
5485 
5486                 skb_set_network_header(skb, skb_gro_offset(skb));
5487                 skb_reset_mac_len(skb);
5488                 NAPI_GRO_CB(skb)->same_flow = 0;
5489                 NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
5490                 NAPI_GRO_CB(skb)->free = 0;
5491                 NAPI_GRO_CB(skb)->encap_mark = 0;
5492                 NAPI_GRO_CB(skb)->recursion_counter = 0;
5493                 NAPI_GRO_CB(skb)->is_fou = 0;
5494                 NAPI_GRO_CB(skb)->is_atomic = 1;
5495                 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
5496 
5497                 /* Setup for GRO checksum validation */
5498                 switch (skb->ip_summed) {
5499                 case CHECKSUM_COMPLETE:
5500                         NAPI_GRO_CB(skb)->csum = skb->csum;
5501                         NAPI_GRO_CB(skb)->csum_valid = 1;
5502                         NAPI_GRO_CB(skb)->csum_cnt = 0;
5503                         break;
5504                 case CHECKSUM_UNNECESSARY:
5505                         NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
5506                         NAPI_GRO_CB(skb)->csum_valid = 0;
5507                         break;
5508                 default:
5509                         NAPI_GRO_CB(skb)->csum_cnt = 0;
5510                         NAPI_GRO_CB(skb)->csum_valid = 0;
5511                 }
5512 
5513                 pp = ptype->callbacks.gro_receive(gro_head, skb);
5514                 break;
5515         }
5516         rcu_read_unlock();
5517 
5518         if (&ptype->list == head)
5519                 goto normal;
5520 
5521         if (IS_ERR(pp) && PTR_ERR(pp) == -EINPROGRESS) {
5522                 ret = GRO_CONSUMED;
5523                 goto ok;
5524         }
5525 
5526         same_flow = NAPI_GRO_CB(skb)->same_flow;
5527         ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
5528 
5529         if (pp) {
5530                 skb_list_del_init(pp);
5531                 napi_gro_complete(pp);
5532                 napi->gro_hash[hash].count--;
5533         }
5534 
5535         if (same_flow)
5536                 goto ok;
5537 
5538         if (NAPI_GRO_CB(skb)->flush)
5539                 goto normal;
5540 
5541         if (unlikely(napi->gro_hash[hash].count >= MAX_GRO_SKBS)) {
5542                 gro_flush_oldest(gro_head);
5543         } else {
5544                 napi->gro_hash[hash].count++;
5545         }
5546         NAPI_GRO_CB(skb)->count = 1;
5547         NAPI_GRO_CB(skb)->age = jiffies;
5548         NAPI_GRO_CB(skb)->last = skb;
5549         skb_shinfo(skb)->gso_size = skb_gro_len(skb);
5550         list_add(&skb->list, gro_head);
5551         ret = GRO_HELD;
5552 
5553 pull:
5554         grow = skb_gro_offset(skb) - skb_headlen(skb);
5555         if (grow > 0)
5556                 gro_pull_from_frag0(skb, grow);
5557 ok:
5558         if (napi->gro_hash[hash].count) {
5559                 if (!test_bit(hash, &napi->gro_bitmask))
5560                         __set_bit(hash, &napi->gro_bitmask);
5561         } else if (test_bit(hash, &napi->gro_bitmask)) {
5562                 __clear_bit(hash, &napi->gro_bitmask);
5563         }
5564 
5565         return ret;
5566 
5567 normal:
5568         ret = GRO_NORMAL;
5569         goto pull;
5570 }
5571 
5572 struct packet_offload *gro_find_receive_by_type(__be16 type)
5573 {
5574         struct list_head *offload_head = &offload_base;
5575         struct packet_offload *ptype;
5576 
5577         list_for_each_entry_rcu(ptype, offload_head, list) {
5578                 if (ptype->type != type || !ptype->callbacks.gro_receive)
5579                         continue;
5580                 return ptype;
5581         }
5582         return NULL;
5583 }
5584 EXPORT_SYMBOL(gro_find_receive_by_type);
5585 
5586 struct packet_offload *gro_find_complete_by_type(__be16 type)
5587 {
5588         struct list_head *offload_head = &offload_base;
5589         struct packet_offload *ptype;
5590 
5591         list_for_each_entry_rcu(ptype, offload_head, list) {
5592                 if (ptype->type != type || !ptype->callbacks.gro_complete)
5593                         continue;
5594                 return ptype;
5595         }
5596         return NULL;
5597 }
5598 EXPORT_SYMBOL(gro_find_complete_by_type);
5599 
5600 static void napi_skb_free_stolen_head(struct sk_buff *skb)
5601 {
5602         skb_dst_drop(skb);
5603         secpath_reset(skb);
5604         kmem_cache_free(skbuff_head_cache, skb);
5605 }
5606 
5607 static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
5608 {
5609         switch (ret) {
5610         case GRO_NORMAL:
5611                 if (netif_receive_skb_internal(skb))
5612                         ret = GRO_DROP;
5613                 break;
5614 
5615         case GRO_DROP:
5616                 kfree_skb(skb);
5617                 break;
5618 
5619         case GRO_MERGED_FREE:
5620                 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
5621                         napi_skb_free_stolen_head(skb);
5622                 else
5623                         __kfree_skb(skb);
5624                 break;
5625 
5626         case GRO_HELD:
5627         case GRO_MERGED:
5628         case GRO_CONSUMED:
5629                 break;
5630         }
5631 
5632         return ret;
5633 }
5634 
5635 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
5636 {
5637         skb_mark_napi_id(skb, napi);
5638         trace_napi_gro_receive_entry(skb);
5639 
5640         skb_gro_reset_offset(skb);
5641 
5642         return napi_skb_finish(dev_gro_receive(napi, skb), skb);
5643 }
5644 EXPORT_SYMBOL(napi_gro_receive);
5645 
5646 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
5647 {
5648         if (unlikely(skb->pfmemalloc)) {
5649                 consume_skb(skb);
5650                 return;
5651         }
5652         __skb_pull(skb, skb_headlen(skb));
5653         /* restore the reserve we had after netdev_alloc_skb_ip_align() */
5654         skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
5655         skb->vlan_tci = 0;
5656         skb->dev = napi->dev;
5657         skb->skb_iif = 0;
5658         skb->encapsulation = 0;
5659         skb_shinfo(skb)->gso_type = 0;
5660         skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
5661         secpath_reset(skb);
5662 
5663         napi->skb = skb;
5664 }
5665 
5666 struct sk_buff *napi_get_frags(struct napi_struct *napi)
5667 {
5668         struct sk_buff *skb = napi->skb;
5669 
5670         if (!skb) {
5671                 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
5672                 if (skb) {
5673                         napi->skb = skb;
5674                         skb_mark_napi_id(skb, napi);
5675                 }
5676         }
5677         return skb;
5678 }
5679 EXPORT_SYMBOL(napi_get_frags);
5680 
5681 static gro_result_t napi_frags_finish(struct napi_struct *napi,
5682                                       struct sk_buff *skb,
5683                                       gro_result_t ret)
5684 {
5685         switch (ret) {
5686         case GRO_NORMAL:
5687         case GRO_HELD:
5688                 __skb_push(skb, ETH_HLEN);
5689                 skb->protocol = eth_type_trans(skb, skb->dev);
5690                 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
5691                         ret = GRO_DROP;
5692                 break;
5693 
5694         case GRO_DROP:
5695                 napi_reuse_skb(napi, skb);
5696                 break;
5697 
5698         case GRO_MERGED_FREE:
5699                 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
5700                         napi_skb_free_stolen_head(skb);
5701                 else
5702                         napi_reuse_skb(napi, skb);
5703                 break;
5704 
5705         case GRO_MERGED:
5706         case GRO_CONSUMED:
5707                 break;
5708         }
5709 
5710         return ret;
5711 }
5712 
5713 /* Upper GRO stack assumes network header starts at gro_offset=0
5714  * Drivers could call both napi_gro_frags() and napi_gro_receive()
5715  * We copy ethernet header into skb->data to have a common layout.
5716  */
5717 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
5718 {
5719         struct sk_buff *skb = napi->skb;
5720         const struct ethhdr *eth;
5721         unsigned int hlen = sizeof(*eth);
5722 
5723         napi->skb = NULL;
5724 
5725         skb_reset_mac_header(skb);
5726         skb_gro_reset_offset(skb);
5727 
5728         eth = skb_gro_header_fast(skb, 0);
5729         if (unlikely(skb_gro_header_hard(skb, hlen))) {
5730                 eth = skb_gro_header_slow(skb, hlen, 0);
5731                 if (unlikely(!eth)) {
5732                         net_warn_ratelimited("%s: dropping impossible skb from %s\n",
5733                                              __func__, napi->dev->name);
5734                         napi_reuse_skb(napi, skb);
5735                         return NULL;
5736                 }
5737         } else {
5738                 gro_pull_from_frag0(skb, hlen);
5739                 NAPI_GRO_CB(skb)->frag0 += hlen;
5740                 NAPI_GRO_CB(skb)->frag0_len -= hlen;
5741         }
5742         __skb_pull(skb, hlen);
5743 
5744         /*
5745          * This works because the only protocols we care about don't require
5746          * special handling.
5747          * We'll fix it up properly in napi_frags_finish()
5748          */
5749         skb->protocol = eth->h_proto;
5750 
5751         return skb;
5752 }
5753 
5754 gro_result_t napi_gro_frags(struct napi_struct *napi)
5755 {
5756         struct sk_buff *skb = napi_frags_skb(napi);
5757 
5758         if (!skb)
5759                 return GRO_DROP;
5760 
5761         trace_napi_gro_frags_entry(skb);
5762 
5763         return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
5764 }
5765 EXPORT_SYMBOL(napi_gro_frags);
5766 
5767 /* Compute the checksum from gro_offset and return the folded value
5768  * after adding in any pseudo checksum.
5769  */
5770 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
5771 {
5772         __wsum wsum;
5773         __sum16 sum;
5774 
5775         wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
5776 
5777         /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
5778         sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
5779         if (likely(!sum)) {
5780                 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
5781                     !skb->csum_complete_sw)
5782                         netdev_rx_csum_fault(skb->dev);
5783         }
5784 
5785         NAPI_GRO_CB(skb)->csum = wsum;
5786         NAPI_GRO_CB(skb)->csum_valid = 1;
5787 
5788         return sum;
5789 }
5790 EXPORT_SYMBOL(__skb_gro_checksum_complete);
5791 
5792 static void net_rps_send_ipi(struct softnet_data *remsd)
5793 {
5794 #ifdef CONFIG_RPS
5795         while (remsd) {
5796                 struct softnet_data *next = remsd->rps_ipi_next;
5797 
5798                 if (cpu_online(remsd->cpu))
5799                         smp_call_function_single_async(remsd->cpu, &remsd->csd);
5800                 remsd = next;
5801         }
5802 #endif
5803 }
5804 
5805 /*
5806  * net_rps_action_and_irq_enable sends any pending IPI's for rps.
5807  * Note: called with local irq disabled, but exits with local irq enabled.
5808  */
5809 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
5810 {
5811 #ifdef CONFIG_RPS
5812         struct softnet_data *remsd = sd->rps_ipi_list;
5813 
5814         if (remsd) {
5815                 sd->rps_ipi_list = NULL;
5816 
5817                 local_irq_enable();
5818 
5819                 /* Send pending IPI's to kick RPS processing on remote cpus. */
5820                 net_rps_send_ipi(remsd);
5821         } else
5822 #endif
5823                 local_irq_enable();
5824 }
5825 
5826 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
5827 {
5828 #ifdef CONFIG_RPS
5829         return sd->rps_ipi_list != NULL;
5830 #else
5831         return false;
5832 #endif
5833 }
5834 
5835 static int process_backlog(struct napi_struct *napi, int quota)
5836 {
5837         struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
5838         bool again = true;
5839         int work = 0;
5840 
5841         /* Check if we have pending ipi, its better to send them now,
5842          * not waiting net_rx_action() end.
5843          */
5844         if (sd_has_rps_ipi_waiting(sd)) {
5845                 local_irq_disable();
5846                 net_rps_action_and_irq_enable(sd);
5847         }
5848 
5849         napi->weight = dev_rx_weight;
5850         while (again) {
5851                 struct sk_buff *skb;
5852 
5853                 while ((skb = __skb_dequeue(&sd->process_queue))) {
5854                         rcu_read_lock();
5855                         __netif_receive_skb(skb);
5856                         rcu_read_unlock();
5857                         input_queue_head_incr(sd);
5858                         if (++work >= quota)
5859                                 return work;
5860 
5861                 }
5862 
5863                 local_irq_disable();
5864                 rps_lock(sd);
5865                 if (skb_queue_empty(&sd->input_pkt_queue)) {
5866                         /*
5867                          * Inline a custom version of __napi_complete().
5868                          * only current cpu owns and manipulates this napi,
5869                          * and NAPI_STATE_SCHED is the only possible flag set
5870                          * on backlog.
5871                          * We can use a plain write instead of clear_bit(),
5872                          * and we dont need an smp_mb() memory barrier.
5873                          */
5874                         napi->state = 0;
5875                         again = false;
5876                 } else {
5877                         skb_queue_splice_tail_init(&sd->input_pkt_queue,
5878                                                    &sd->process_queue);
5879                 }
5880                 rps_unlock(sd);
5881                 local_irq_enable();
5882         }
5883 
5884         return work;
5885 }
5886 
5887 /**
5888  * __napi_schedule - schedule for receive
5889  * @n: entry to schedule
5890  *
5891  * The entry's receive function will be scheduled to run.
5892  * Consider using __napi_schedule_irqoff() if hard irqs are masked.
5893  */
5894 void __napi_schedule(struct napi_struct *n)
5895 {
5896         unsigned long flags;
5897 
5898         local_irq_save(flags);
5899         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
5900         local_irq_restore(flags);
5901 }
5902 EXPORT_SYMBOL(__napi_schedule);
5903 
5904 /**
5905  *      napi_schedule_prep - check if napi can be scheduled
5906  *      @n: napi context
5907  *
5908  * Test if NAPI routine is already running, and if not mark
5909  * it as running.  This is used as a condition variable
5910  * insure only one NAPI poll instance runs.  We also make
5911  * sure there is no pending NAPI disable.
5912  */
5913 bool napi_schedule_prep(struct napi_struct *n)
5914 {
5915         unsigned long val, new;
5916 
5917         do {
5918                 val = READ_ONCE(n->state);
5919                 if (unlikely(val & NAPIF_STATE_DISABLE))
5920                         return false;
5921                 new = val | NAPIF_STATE_SCHED;
5922 
5923                 /* Sets STATE_MISSED bit if STATE_SCHED was already set
5924                  * This was suggested by Alexander Duyck, as compiler
5925                  * emits better code than :
5926                  * if (val & NAPIF_STATE_SCHED)
5927                  *     new |= NAPIF_STATE_MISSED;
5928                  */
5929                 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
5930                                                    NAPIF_STATE_MISSED;
5931         } while (cmpxchg(&n->state, val, new) != val);
5932 
5933         return !(val & NAPIF_STATE_SCHED);
5934 }
5935 EXPORT_SYMBOL(napi_schedule_prep);
5936 
5937 /**
5938  * __napi_schedule_irqoff - schedule for receive
5939  * @n: entry to schedule
5940  *
5941  * Variant of __napi_schedule() assuming hard irqs are masked
5942  */
5943 void __napi_schedule_irqoff(struct napi_struct *n)
5944 {
5945         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
5946 }
5947 EXPORT_SYMBOL(__napi_schedule_irqoff);
5948 
5949 bool napi_complete_done(struct napi_struct *n, int work_done)
5950 {
5951         unsigned long flags, val, new;
5952 
5953         /*
5954          * 1) Don't let napi dequeue from the cpu poll list
5955          *    just in case its running on a different cpu.
5956          * 2) If we are busy polling, do nothing here, we have
5957          *    the guarantee we will be called later.
5958          */
5959         if (unlikely(n->state & (NAPIF_STATE_NPSVC |
5960                                  NAPIF_STATE_IN_BUSY_POLL)))
5961                 return false;
5962 
5963         if (n->gro_bitmask) {
5964                 unsigned long timeout = 0;
5965 
5966                 if (work_done)
5967                         timeout = n->dev->gro_flush_timeout;
5968 
5969                 if (timeout)
5970                         hrtimer_start(&n->timer, ns_to_ktime(timeout),
5971                                       HRTIMER_MODE_REL_PINNED);
5972                 else
5973                         napi_gro_flush(n, false);
5974         }
5975         if (unlikely(!list_empty(&n->poll_list))) {
5976                 /* If n->poll_list is not empty, we need to mask irqs */
5977                 local_irq_save(flags);
5978                 list_del_init(&n->poll_list);
5979                 local_irq_restore(flags);
5980         }
5981 
5982         do {
5983                 val = READ_ONCE(n->state);
5984 
5985                 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
5986 
5987                 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED);
5988 
5989                 /* If STATE_MISSED was set, leave STATE_SCHED set,
5990                  * because we will call napi->poll() one more time.
5991                  * This C code was suggested by Alexander Duyck to help gcc.
5992                  */
5993                 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
5994                                                     NAPIF_STATE_SCHED;
5995         } while (cmpxchg(&n->state, val, new) != val);
5996 
5997         if (unlikely(val & NAPIF_STATE_MISSED)) {
5998                 __napi_schedule(n);
5999                 return false;
6000         }
6001 
6002         return true;
6003 }
6004 EXPORT_SYMBOL(napi_complete_done);
6005 
6006 /* must be called under rcu_read_lock(), as we dont take a reference */
6007 static struct napi_struct *napi_by_id(unsigned int napi_id)
6008 {
6009         unsigned int hash = napi_id % HASH_SIZE(napi_hash);
6010         struct napi_struct *napi;
6011 
6012         hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
6013                 if (napi->napi_id == napi_id)
6014                         return napi;
6015 
6016         return NULL;
6017 }
6018 
6019 #if defined(CONFIG_NET_RX_BUSY_POLL)
6020 
6021 #define BUSY_POLL_BUDGET 8
6022 
6023 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock)
6024 {
6025         int rc;
6026 
6027         /* Busy polling means there is a high chance device driver hard irq
6028          * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
6029          * set in napi_schedule_prep().
6030          * Since we are about to call napi->poll() once more, we can safely
6031          * clear NAPI_STATE_MISSED.
6032          *
6033          * Note: x86 could use a single "lock and ..." instruction
6034          * to perform these two clear_bit()
6035          */
6036         clear_bit(NAPI_STATE_MISSED, &napi->state);
6037         clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
6038 
6039         local_bh_disable();
6040 
6041         /* All we really want here is to re-enable device interrupts.
6042          * Ideally, a new ndo_busy_poll_stop() could avoid another round.
6043          */
6044         rc = napi->poll(napi, BUSY_POLL_BUDGET);
6045         trace_napi_poll(napi, rc, BUSY_POLL_BUDGET);
6046         netpoll_poll_unlock(have_poll_lock);
6047         if (rc == BUSY_POLL_BUDGET)
6048                 __napi_schedule(napi);
6049         local_bh_enable();
6050 }
6051 
6052 void napi_busy_loop(unsigned int napi_id,
6053                     bool (*loop_end)(void *, unsigned long),
6054                     void *loop_end_arg)
6055 {
6056         unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
6057         int (*napi_poll)(struct napi_struct *napi, int budget);
6058         void *have_poll_lock = NULL;
6059         struct napi_struct *napi;
6060 
6061 restart:
6062         napi_poll = NULL;
6063 
6064         rcu_read_lock();
6065 
6066         napi = napi_by_id(napi_id);
6067         if (!napi)
6068                 goto out;
6069 
6070         preempt_disable();
6071         for (;;) {
6072                 int work = 0;
6073 
6074                 local_bh_disable();
6075                 if (!napi_poll) {
6076                         unsigned long val = READ_ONCE(napi->state);
6077 
6078                         /* If multiple threads are competing for this napi,
6079                          * we avoid dirtying napi->state as much as we can.
6080                          */
6081                         if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
6082                                    NAPIF_STATE_IN_BUSY_POLL))
6083                                 goto count;
6084                         if (cmpxchg(&napi->state, val,
6085                                     val | NAPIF_STATE_IN_BUSY_POLL |
6086                                           NAPIF_STATE_SCHED) != val)
6087                                 goto count;
6088                         have_poll_lock = netpoll_poll_lock(napi);
6089                         napi_poll = napi->poll;
6090                 }
6091                 work = napi_poll(napi, BUSY_POLL_BUDGET);
6092                 trace_napi_poll(napi, work, BUSY_POLL_BUDGET);
6093 count:
6094                 if (work > 0)
6095                         __NET_ADD_STATS(dev_net(napi->dev),
6096                                         LINUX_MIB_BUSYPOLLRXPACKETS, work);
6097                 local_bh_enable();
6098 
6099                 if (!loop_end || loop_end(loop_end_arg, start_time))
6100                         break;
6101 
6102                 if (unlikely(need_resched())) {
6103                         if (napi_poll)
6104                                 busy_poll_stop(napi, have_poll_lock);
6105                         preempt_enable();
6106                         rcu_read_unlock();
6107                         cond_resched();
6108                         if (loop_end(loop_end_arg, start_time))
6109                                 return;
6110                         goto restart;
6111                 }
6112                 cpu_relax();
6113         }
6114         if (napi_poll)
6115                 busy_poll_stop(napi, have_poll_lock);
6116         preempt_enable();
6117 out:
6118         rcu_read_unlock();
6119 }
6120 EXPORT_SYMBOL(napi_busy_loop);
6121 
6122 #endif /* CONFIG_NET_RX_BUSY_POLL */
6123 
6124 static void napi_hash_add(struct napi_struct *napi)
6125 {
6126         if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) ||
6127             test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
6128                 return;
6129 
6130         spin_lock(&napi_hash_lock);
6131 
6132         /* 0..NR_CPUS range is reserved for sender_cpu use */
6133         do {
6134                 if (unlikely(++napi_gen_id < MIN_NAPI_ID))
6135                         napi_gen_id = MIN_NAPI_ID;
6136         } while (napi_by_id(napi_gen_id));
6137         napi->napi_id = napi_gen_id;
6138 
6139         hlist_add_head_rcu(&napi->napi_hash_node,
6140                            &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
6141 
6142         spin_unlock(&napi_hash_lock);
6143 }
6144 
6145 /* Warning : caller is responsible to make sure rcu grace period
6146  * is respected before freeing memory containing @napi
6147  */
6148 bool napi_hash_del(struct napi_struct *napi)
6149 {
6150         bool rcu_sync_needed = false;
6151 
6152         spin_lock(&napi_hash_lock);
6153 
6154         if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) {
6155                 rcu_sync_needed = true;
6156                 hlist_del_rcu(&napi->napi_hash_node);
6157         }
6158         spin_unlock(&napi_hash_lock);
6159         return rcu_sync_needed;
6160 }
6161 EXPORT_SYMBOL_GPL(napi_hash_del);
6162 
6163 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
6164 {
6165         struct napi_struct *napi;
6166 
6167         napi = container_of(timer, struct napi_struct, timer);
6168 
6169         /* Note : we use a relaxed variant of napi_schedule_prep() not setting
6170          * NAPI_STATE_MISSED, since we do not react to a device IRQ.
6171          */
6172         if (napi->gro_bitmask && !napi_disable_pending(napi) &&
6173             !test_and_set_bit(NAPI_STATE_SCHED, &napi->state))
6174                 __napi_schedule_irqoff(napi);
6175 
6176         return HRTIMER_NORESTART;
6177 }
6178 
6179 static void init_gro_hash(struct napi_struct *napi)
6180 {
6181         int i;
6182 
6183         for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6184                 INIT_LIST_HEAD(&napi->gro_hash[i].list);
6185                 napi->gro_hash[i].count = 0;
6186         }
6187         napi->gro_bitmask = 0;
6188 }
6189 
6190 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
6191                     int (*poll)(struct napi_struct *, int), int weight)
6192 {
6193         INIT_LIST_HEAD(&napi->poll_list);
6194         hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
6195         napi->timer.function = napi_watchdog;
6196         init_gro_hash(napi);
6197         napi->skb = NULL;
6198         napi->poll = poll;
6199         if (weight > NAPI_POLL_WEIGHT)
6200                 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
6201                             weight, dev->name);
6202         napi->weight = weight;
6203         list_add(&napi->dev_list, &dev->napi_list);
6204         napi->dev = dev;
6205 #ifdef CONFIG_NETPOLL
6206         napi->poll_owner = -1;
6207 #endif
6208         set_bit(NAPI_STATE_SCHED, &napi->state);
6209         napi_hash_add(napi);
6210 }
6211 EXPORT_SYMBOL(netif_napi_add);
6212 
6213 void napi_disable(struct napi_struct *n)
6214 {
6215         might_sleep();
6216         set_bit(NAPI_STATE_DISABLE, &n->state);
6217 
6218         while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
6219                 msleep(1);
6220         while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
6221                 msleep(1);
6222 
6223         hrtimer_cancel(&n->timer);
6224 
6225         clear_bit(NAPI_STATE_DISABLE, &n->state);
6226 }
6227 EXPORT_SYMBOL(napi_disable);
6228 
6229 static void flush_gro_hash(struct napi_struct *napi)
6230 {
6231         int i;
6232 
6233         for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6234                 struct sk_buff *skb, *n;
6235 
6236                 list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list)
6237                         kfree_skb(skb);
6238                 napi->gro_hash[i].count = 0;
6239         }
6240 }
6241 
6242 /* Must be called in process context */
6243 void netif_napi_del(struct napi_struct *napi)
6244 {
6245         might_sleep();
6246         if (napi_hash_del(napi))
6247                 synchronize_net();
6248         list_del_init(&napi->dev_list);
6249         napi_free_frags(napi);
6250 
6251         flush_gro_hash(napi);
6252         napi->gro_bitmask = 0;
6253 }
6254 EXPORT_SYMBOL(netif_napi_del);
6255 
6256 static int napi_poll(struct napi_struct *n, struct list_head *repoll)
6257 {
6258         void *have;
6259         int work, weight;
6260 
6261         list_del_init(&n->poll_list);
6262 
6263         have = netpoll_poll_lock(n);
6264 
6265         weight = n->weight;
6266 
6267         /* This NAPI_STATE_SCHED test is for avoiding a race
6268          * with netpoll's poll_napi().  Only the entity which
6269          * obtains the lock and sees NAPI_STATE_SCHED set will
6270          * actually make the ->poll() call.  Therefore we avoid
6271          * accidentally calling ->poll() when NAPI is not scheduled.
6272          */
6273         work = 0;
6274         if (test_bit(NAPI_STATE_SCHED, &n->state)) {
6275                 work = n->poll(n, weight);
6276                 trace_napi_poll(n, work, weight);
6277         }
6278 
6279         WARN_ON_ONCE(work > weight);
6280 
6281         if (likely(work < weight))
6282                 goto out_unlock;
6283 
6284         /* Drivers must not modify the NAPI state if they
6285          * consume the entire weight.  In such cases this code
6286          * still "owns" the NAPI instance and therefore can
6287          * move the instance around on the list at-will.
6288          */
6289         if (unlikely(napi_disable_pending(n))) {
6290                 napi_complete(n);
6291                 goto out_unlock;
6292         }
6293 
6294         if (n->gro_bitmask) {
6295                 /* flush too old packets
6296                  * If HZ < 1000, flush all packets.
6297                  */
6298                 napi_gro_flush(n, HZ >= 1000);
6299         }
6300 
6301         /* Some drivers may have called napi_schedule
6302          * prior to exhausting their budget.
6303          */
6304         if (unlikely(!list_empty(&n->poll_list))) {
6305                 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
6306                              n->dev ? n->dev->name : "backlog");
6307                 goto out_unlock;
6308         }
6309 
6310         list_add_tail(&n->poll_list, repoll);
6311 
6312 out_unlock:
6313         netpoll_poll_unlock(have);
6314 
6315         return work;
6316 }
6317 
6318 static __latent_entropy void net_rx_action(struct softirq_action *h)
6319 {
6320         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
6321         unsigned long time_limit = jiffies +
6322                 usecs_to_jiffies(netdev_budget_usecs);
6323         int budget = netdev_budget;
6324         LIST_HEAD(list);
6325         LIST_HEAD(repoll);
6326 
6327         local_irq_disable();
6328         list_splice_init(&sd->poll_list, &list);
6329         local_irq_enable();
6330 
6331         for (;;) {
6332                 struct napi_struct *n;
6333 
6334                 if (list_empty(&list)) {
6335                         if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
6336                                 goto out;
6337                         break;
6338                 }
6339 
6340                 n = list_first_entry(&list, struct napi_struct, poll_list);
6341                 budget -= napi_poll(n, &repoll);
6342 
6343                 /* If softirq window is exhausted then punt.
6344                  * Allow this to run for 2 jiffies since which will allow
6345                  * an average latency of 1.5/HZ.
6346                  */
6347                 if (unlikely(budget <= 0 ||
6348                              time_after_eq(jiffies, time_limit))) {
6349                         sd->time_squeeze++;
6350                         break;
6351                 }
6352         }
6353 
6354         local_irq_disable();
6355 
6356         list_splice_tail_init(&sd->poll_list, &list);
6357         list_splice_tail(&repoll, &list);
6358         list_splice(&list, &sd->poll_list);
6359         if (!list_empty(&sd->poll_list))
6360                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
6361 
6362         net_rps_action_and_irq_enable(sd);
6363 out:
6364         __kfree_skb_flush();
6365 }
6366 
6367 struct netdev_adjacent {
6368         struct net_device *dev;
6369 
6370         /* upper master flag, there can only be one master device per list */
6371         bool master;
6372 
6373         /* counter for the number of times this device was added to us */
6374         u16 ref_nr;
6375 
6376         /* private field for the users */
6377         void *private;
6378 
6379         struct list_head list;
6380         struct rcu_head rcu;
6381 };
6382 
6383 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
6384                                                  struct list_head *adj_list)
6385 {
6386         struct netdev_adjacent *adj;
6387 
6388         list_for_each_entry(adj, adj_list, list) {
6389                 if (adj->dev == adj_dev)
6390                         return adj;
6391         }
6392         return NULL;
6393 }
6394 
6395 static int __netdev_has_upper_dev(struct net_device *upper_dev, void *data)
6396 {
6397         struct net_device *dev = data;
6398 
6399         return upper_dev == dev;
6400 }
6401 
6402 /**
6403  * netdev_has_upper_dev - Check if device is linked to an upper device
6404  * @dev: device
6405  * @upper_dev: upper device to check
6406  *
6407  * Find out if a device is linked to specified upper device and return true
6408  * in case it is. Note that this checks only immediate upper device,
6409  * not through a complete stack of devices. The caller must hold the RTNL lock.
6410  */
6411 bool netdev_has_upper_dev(struct net_device *dev,
6412                           struct net_device *upper_dev)
6413 {
6414         ASSERT_RTNL();
6415 
6416         return netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
6417                                              upper_dev);
6418 }
6419 EXPORT_SYMBOL(netdev_has_upper_dev);
6420 
6421 /**
6422  * netdev_has_upper_dev_all - Check if device is linked to an upper device
6423  * @dev: device
6424  * @upper_dev: upper device to check
6425  *
6426  * Find out if a device is linked to specified upper device and return true
6427  * in case it is. Note that this checks the entire upper device chain.
6428  * The caller must hold rcu lock.
6429  */
6430 
6431 bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
6432                                   struct net_device *upper_dev)
6433 {
6434         return !!netdev_walk_all_upper_dev_rcu(dev, __netdev_has_upper_dev,
6435                                                upper_dev);
6436 }
6437 EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
6438 
6439 /**
6440  * netdev_has_any_upper_dev - Check if device is linked to some device
6441  * @dev: device
6442  *
6443  * Find out if a device is linked to an upper device and return true in case
6444  * it is. The caller must hold the RTNL lock.
6445  */
6446 bool netdev_has_any_upper_dev(struct net_device *dev)
6447 {
6448         ASSERT_RTNL();
6449 
6450         return !list_empty(&dev->adj_list.upper);
6451 }
6452 EXPORT_SYMBOL(netdev_has_any_upper_dev);
6453 
6454 /**
6455  * netdev_master_upper_dev_get - Get master upper device
6456  * @dev: device
6457  *
6458  * Find a master upper device and return pointer to it or NULL in case
6459  * it's not there. The caller must hold the RTNL lock.
6460  */
6461 struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
6462 {
6463         struct netdev_adjacent *upper;
6464 
6465         ASSERT_RTNL();
6466 
6467         if (list_empty(&dev->adj_list.upper))
6468                 return NULL;
6469 
6470         upper = list_first_entry(&dev->adj_list.upper,
6471                                  struct netdev_adjacent, list);
6472         if (likely(upper->master))
6473                 return upper->dev;
6474         return NULL;
6475 }
6476 EXPORT_SYMBOL(netdev_master_upper_dev_get);
6477 
6478 /**
6479  * netdev_has_any_lower_dev - Check if device is linked to some device
6480  * @dev: device
6481  *
6482  * Find out if a device is linked to a lower device and return true in case
6483  * it is. The caller must hold the RTNL lock.
6484  */
6485 static bool netdev_has_any_lower_dev(struct net_device *dev)
6486 {
6487         ASSERT_RTNL();
6488 
6489         return !list_empty(&dev->adj_list.lower);
6490 }
6491 
6492 void *netdev_adjacent_get_private(struct list_head *adj_list)
6493 {
6494         struct netdev_adjacent *adj;
6495 
6496         adj = list_entry(adj_list, struct netdev_adjacent, list);
6497 
6498         return adj->private;
6499 }
6500 EXPORT_SYMBOL(netdev_adjacent_get_private);
6501 
6502 /**
6503  * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
6504  * @dev: device
6505  * @iter: list_head ** of the current position
6506  *
6507  * Gets the next device from the dev's upper list, starting from iter
6508  * position. The caller must hold RCU read lock.
6509  */
6510 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
6511                                                  struct list_head **iter)
6512 {
6513         struct netdev_adjacent *upper;
6514 
6515         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
6516 
6517         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6518 
6519         if (&upper->list == &dev->adj_list.upper)
6520                 return NULL;
6521 
6522         *iter = &upper->list;
6523 
6524         return upper->dev;
6525 }
6526 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
6527 
6528 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
6529                                                     struct list_head **iter)
6530 {
6531         struct netdev_adjacent *upper;
6532 
6533         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
6534 
6535         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6536 
6537         if (&upper->list == &dev->adj_list.upper)
6538                 return NULL;
6539 
6540         *iter = &upper->list;
6541 
6542         return upper->dev;
6543 }
6544 
6545 int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
6546                                   int (*fn)(struct net_device *dev,
6547                                             void *data),
6548                                   void *data)
6549 {
6550         struct net_device *udev;
6551         struct list_head *iter;
6552         int ret;
6553 
6554         for (iter = &dev->adj_list.upper,
6555              udev = netdev_next_upper_dev_rcu(dev, &iter);
6556              udev;
6557              udev = netdev_next_upper_dev_rcu(dev, &iter)) {
6558                 /* first is the upper device itself */
6559                 ret = fn(udev, data);
6560                 if (ret)
6561                         return ret;
6562 
6563                 /* then look at all of its upper devices */
6564                 ret = netdev_walk_all_upper_dev_rcu(udev, fn, data);
6565                 if (ret)
6566                         return ret;
6567         }
6568 
6569         return 0;
6570 }
6571 EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
6572 
6573 /**
6574  * netdev_lower_get_next_private - Get the next ->private from the
6575  *                                 lower neighbour list
6576  * @dev: device
6577  * @iter: list_head ** of the current position
6578  *
6579  * Gets the next netdev_adjacent->private from the dev's lower neighbour
6580  * list, starting from iter position. The caller must hold either hold the
6581  * RTNL lock or its own locking that guarantees that the neighbour lower
6582  * list will remain unchanged.
6583  */
6584 void *netdev_lower_get_next_private(struct net_device *dev,
6585                                     struct list_head **iter)
6586 {
6587         struct netdev_adjacent *lower;
6588 
6589         lower = list_entry(*iter, struct netdev_adjacent, list);
6590 
6591         if (&lower->list == &dev->adj_list.lower)
6592                 return NULL;
6593 
6594         *iter = lower->list.next;
6595 
6596         return lower->private;
6597 }
6598 EXPORT_SYMBOL(netdev_lower_get_next_private);
6599 
6600 /**
6601  * netdev_lower_get_next_private_rcu - Get the next ->private from the
6602  *                                     lower neighbour list, RCU
6603  *                                     variant
6604  * @dev: device
6605  * @iter: list_head ** of the current position
6606  *
6607  * Gets the next netdev_adjacent->private from the dev's lower neighbour
6608  * list, starting from iter position. The caller must hold RCU read lock.
6609  */
6610 void *netdev_lower_get_next_private_rcu(struct net_device *dev,
6611                                         struct list_head **iter)
6612 {
6613         struct netdev_adjacent *lower;
6614 
6615         WARN_ON_ONCE(!rcu_read_lock_held());
6616 
6617         lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6618 
6619         if (&lower->list == &dev->adj_list.lower)
6620                 return NULL;
6621 
6622         *iter = &lower->list;
6623 
6624         return lower->private;
6625 }
6626 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
6627 
6628 /**
6629  * netdev_lower_get_next - Get the next device from the lower neighbour
6630  *                         list
6631  * @dev: device
6632  * @iter: list_head ** of the current position
6633  *
6634  * Gets the next netdev_adjacent from the dev's lower neighbour
6635  * list, starting from iter position. The caller must hold RTNL lock or
6636  * its own locking that guarantees that the neighbour lower
6637  * list will remain unchanged.
6638  */
6639 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
6640 {
6641         struct netdev_adjacent *lower;
6642 
6643         lower = list_entry(*iter, struct netdev_adjacent, list);
6644 
6645         if (&lower->list == &dev->adj_list.lower)
6646                 return NULL;
6647 
6648         *iter = lower->list.next;
6649 
6650         return lower->dev;
6651 }
6652 EXPORT_SYMBOL(netdev_lower_get_next);
6653 
6654 static struct net_device *netdev_next_lower_dev(struct net_device *dev,
6655                                                 struct list_head **iter)
6656 {
6657         struct netdev_adjacent *lower;
6658 
6659         lower = list_entry((*iter)->next, struct netdev_adjacent, list);
6660 
6661         if (&lower->list == &dev->adj_list.lower)
6662                 return NULL;
6663 
6664         *iter = &lower->list;
6665 
6666         return lower->dev;
6667 }
6668 
6669 int netdev_walk_all_lower_dev(struct net_device *dev,
6670                               int (*fn)(struct net_device *dev,
6671                                         void *data),
6672                               void *data)
6673 {
6674         struct net_device *ldev;
6675         struct list_head *iter;
6676         int ret;
6677 
6678         for (iter = &dev->adj_list.lower,
6679              ldev = netdev_next_lower_dev(dev, &iter);
6680              ldev;
6681              ldev = netdev_next_lower_dev(dev, &iter)) {
6682                 /* first is the lower device itself */
6683                 ret = fn(ldev, data);
6684                 if (ret)
6685                         return ret;
6686 
6687                 /* then look at all of its lower devices */
6688                 ret = netdev_walk_all_lower_dev(ldev, fn, data);
6689                 if (ret)
6690                         return ret;
6691         }
6692 
6693         return 0;
6694 }
6695 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
6696 
6697 static struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
6698                                                     struct list_head **iter)
6699 {
6700         struct netdev_adjacent *lower;
6701 
6702         lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
6703         if (&lower->list == &dev->adj_list.lower)
6704                 return NULL;
6705 
6706         *iter = &lower->list;
6707 
6708         return lower->dev;
6709 }
6710 
6711 int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
6712                                   int (*fn)(struct net_device *dev,
6713                                             void *data),
6714                                   void *data)
6715 {
6716         struct net_device *ldev;
6717         struct list_head *iter;
6718         int ret;
6719 
6720         for (iter = &dev->adj_list.lower,
6721              ldev = netdev_next_lower_dev_rcu(dev, &iter);
6722              ldev;
6723              ldev = netdev_next_lower_dev_rcu(dev, &iter)) {
6724                 /* first is the lower device itself */
6725                 ret = fn(ldev, data);
6726                 if (ret)
6727                         return ret;
6728 
6729                 /* then look at all of its lower devices */
6730                 ret = netdev_walk_all_lower_dev_rcu(ldev, fn, data);
6731                 if (ret)
6732                         return ret;
6733         }
6734 
6735         return 0;
6736 }
6737 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
6738 
6739 /**
6740  * netdev_lower_get_first_private_rcu - Get the first ->private from the
6741  *                                     lower neighbour list, RCU
6742  *                                     variant
6743  * @dev: device
6744  *
6745  * Gets the first netdev_adjacent->private from the dev's lower neighbour
6746  * list. The caller must hold RCU read lock.
6747  */
6748 void *netdev_lower_get_first_private_rcu(struct net_device *dev)
6749 {
6750         struct netdev_adjacent *lower;
6751 
6752         lower = list_first_or_null_rcu(&dev->adj_list.lower,
6753                         struct netdev_adjacent, list);
6754         if (lower)
6755                 return lower->private;
6756         return NULL;
6757 }
6758 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
6759 
6760 /**
6761  * netdev_master_upper_dev_get_rcu - Get master upper device
6762  * @dev: device
6763  *
6764  * Find a master upper device and return pointer to it or NULL in case
6765  * it's not there. The caller must hold the RCU read lock.
6766  */
6767 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
6768 {
6769         struct netdev_adjacent *upper;
6770 
6771         upper = list_first_or_null_rcu(&dev->adj_list.upper,
6772                                        struct netdev_adjacent, list);
6773         if (upper && likely(upper->master))
6774                 return upper->dev;
6775         return NULL;
6776 }
6777 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
6778 
6779 static int netdev_adjacent_sysfs_add(struct net_device *dev,
6780                               struct net_device *adj_dev,
6781                               struct list_head *dev_list)
6782 {
6783         char linkname[IFNAMSIZ+7];
6784 
6785         sprintf(linkname, dev_list == &dev->adj_list.upper ?
6786                 "upper_%s" : "lower_%s", adj_dev->name);
6787         return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
6788                                  linkname);
6789 }
6790 static void netdev_adjacent_sysfs_del(struct net_device *dev,
6791                                char *name,
6792                                struct list_head *dev_list)
6793 {
6794         char linkname[IFNAMSIZ+7];
6795 
6796         sprintf(linkname, dev_list == &dev->adj_list.upper ?
6797                 "upper_%s" : "lower_%s", name);
6798         sysfs_remove_link(&(dev->dev.kobj), linkname);
6799 }
6800 
6801 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
6802                                                  struct net_device *adj_dev,
6803                                                  struct list_head *dev_list)
6804 {
6805         return (dev_list == &dev->adj_list.upper ||
6806                 dev_list == &dev->adj_list.lower) &&
6807                 net_eq(dev_net(dev), dev_net(adj_dev));
6808 }
6809 
6810 static int __netdev_adjacent_dev_insert(struct net_device *dev,
6811                                         struct net_device *adj_dev,
6812                                         struct list_head *dev_list,
6813                                         void *private, bool master)
6814 {
6815         struct netdev_adjacent *adj;
6816         int ret;
6817 
6818         adj = __netdev_find_adj(adj_dev, dev_list);
6819 
6820         if (adj) {
6821                 adj->ref_nr += 1;
6822                 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
6823                          dev->name, adj_dev->name, adj->ref_nr);
6824 
6825                 return 0;
6826         }
6827 
6828         adj = kmalloc(sizeof(*adj), GFP_KERNEL);
6829         if (!adj)
6830                 return -ENOMEM;
6831 
6832         adj->dev = adj_dev;
6833         adj->master = master;
6834         adj->ref_nr = 1;
6835         adj->private = private;
6836         dev_hold(adj_dev);
6837 
6838         pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
6839                  dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
6840 
6841         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
6842                 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
6843                 if (ret)
6844                         goto free_adj;
6845         }
6846 
6847         /* Ensure that master link is always the first item in list. */
6848         if (master) {
6849                 ret = sysfs_create_link(&(dev->dev.kobj),
6850                                         &(adj_dev->dev.kobj), "master");
6851                 if (ret)
6852                         goto remove_symlinks;
6853 
6854                 list_add_rcu(&adj->list, dev_list);
6855         } else {
6856                 list_add_tail_rcu(&adj->list, dev_list);
6857         }
6858 
6859         return 0;
6860 
6861 remove_symlinks:
6862         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
6863                 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
6864 free_adj:
6865         kfree(adj);
6866         dev_put(adj_dev);
6867 
6868         return ret;
6869 }
6870 
6871 static void __netdev_adjacent_dev_remove(struct net_device *dev,
6872                                          struct net_device *adj_dev,
6873                                          u16 ref_nr,
6874                                          struct list_head *dev_list)
6875 {
6876         struct netdev_adjacent *adj;
6877 
6878         pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
6879                  dev->name, adj_dev->name, ref_nr);
6880 
6881         adj = __netdev_find_adj(adj_dev, dev_list);
6882 
6883         if (!adj) {
6884                 pr_err("Adjacency does not exist for device %s from %s\n",
6885                        dev->name, adj_dev->name);
6886                 WARN_ON(1);
6887                 return;
6888         }
6889 
6890         if (adj->ref_nr > ref_nr) {
6891                 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
6892                          dev->name, adj_dev->name, ref_nr,
6893                          adj->ref_nr - ref_nr);
6894                 adj->ref_nr -= ref_nr;
6895                 return;
6896         }
6897 
6898         if (adj->master)
6899                 sysfs_remove_link(&(dev->dev.kobj), "master");
6900 
6901         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
6902                 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
6903 
6904         list_del_rcu(&adj->list);
6905         pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
6906                  adj_dev->name, dev->name, adj_dev->name);
6907         dev_put(adj_dev);
6908         kfree_rcu(adj, rcu);
6909 }
6910 
6911 static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
6912                                             struct net_device *upper_dev,
6913                                             struct list_head *up_list,
6914                                             struct list_head *down_list,
6915                                             void *private, bool master)
6916 {
6917         int ret;
6918 
6919         ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
6920                                            private, master);
6921         if (ret)
6922                 return ret;
6923 
6924         ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
6925                                            private, false);
6926         if (ret) {
6927                 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
6928                 return ret;
6929         }
6930 
6931         return 0;
6932 }
6933 
6934 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
6935                                                struct net_device *upper_dev,
6936                                                u16 ref_nr,
6937                                                struct list_head *up_list,
6938                                                struct list_head *down_list)
6939 {
6940         __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
6941         __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
6942 }
6943 
6944 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
6945                                                 struct net_device *upper_dev,
6946                                                 void *private, bool master)
6947 {
6948         return __netdev_adjacent_dev_link_lists(dev, upper_dev,
6949                                                 &dev->adj_list.upper,
6950                                                 &upper_dev->adj_list.lower,
6951                                                 private, master);
6952 }
6953 
6954 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
6955                                                    struct net_device *upper_dev)
6956 {
6957         __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
6958                                            &dev->adj_list.upper,
6959                                            &upper_dev->adj_list.lower);
6960 }
6961 
6962 static int __netdev_upper_dev_link(struct net_device *dev,
6963                                    struct net_device *upper_dev, bool master,
6964                                    void *upper_priv, void *upper_info,
6965                                    struct netlink_ext_ack *extack)
6966 {
6967         struct netdev_notifier_changeupper_info changeupper_info = {
6968                 .info = {
6969                         .dev = dev,
6970                         .extack = extack,
6971                 },
6972                 .upper_dev = upper_dev,
6973                 .master = master,
6974                 .linking = true,
6975                 .upper_info = upper_info,
6976         };
6977         struct net_device *master_dev;
6978         int ret = 0;
6979 
6980         ASSERT_RTNL();
6981 
6982         if (dev == upper_dev)
6983                 return -EBUSY;
6984 
6985         /* To prevent loops, check if dev is not upper device to upper_dev. */
6986         if (netdev_has_upper_dev(upper_dev, dev))
6987                 return -EBUSY;
6988 
6989         if (!master) {
6990                 if (netdev_has_upper_dev(dev, upper_dev))
6991                         return -EEXIST;
6992         } else {
6993                 master_dev = netdev_master_upper_dev_get(dev);
6994                 if (master_dev)
6995                         return master_dev == upper_dev ? -EEXIST : -EBUSY;
6996         }
6997 
6998         ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
6999                                             &changeupper_info.info);
7000         ret = notifier_to_errno(ret);
7001         if (ret)
7002                 return ret;
7003 
7004         ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
7005                                                    master);
7006         if (ret)
7007                 return ret;
7008 
7009         ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
7010                                             &changeupper_info.info);
7011         ret = notifier_to_errno(ret);
7012         if (ret)
7013                 goto rollback;
7014 
7015         return 0;
7016 
7017 rollback:
7018         __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
7019 
7020         return ret;
7021 }
7022 
7023 /**
7024  * netdev_upper_dev_link - Add a link to the upper device
7025  * @dev: device
7026  * @upper_dev: new upper device
7027  * @extack: netlink extended ack
7028  *
7029  * Adds a link to device which is upper to this one. The caller must hold
7030  * the RTNL lock. On a failure a negative errno code is returned.
7031  * On success the reference counts are adjusted and the function
7032  * returns zero.
7033  */
7034 int netdev_upper_dev_link(struct net_device *dev,
7035                           struct net_device *upper_dev,
7036                           struct netlink_ext_ack *extack)
7037 {
7038         return __netdev_upper_dev_link(dev, upper_dev, false,
7039                                        NULL, NULL, extack);
7040 }
7041 EXPORT_SYMBOL(netdev_upper_dev_link);
7042 
7043 /**
7044  * netdev_master_upper_dev_link - Add a master link to the upper device
7045  * @dev: device
7046  * @upper_dev: new upper device
7047  * @upper_priv: upper device private
7048  * @upper_info: upper info to be passed down via notifier
7049  * @extack: netlink extended ack
7050  *
7051  * Adds a link to device which is upper to this one. In this case, only
7052  * one master upper device can be linked, although other non-master devices
7053  * might be linked as well. The caller must hold the RTNL lock.
7054  * On a failure a negative errno code is returned. On success the reference
7055  * counts are adjusted and the function returns zero.
7056  */
7057 int netdev_master_upper_dev_link(struct net_device *dev,
7058                                  struct net_device *upper_dev,
7059                                  void *upper_priv, void *upper_info,
7060                                  struct netlink_ext_ack *extack)
7061 {
7062         return __netdev_upper_dev_link(dev, upper_dev, true,
7063                                        upper_priv, upper_info, extack);
7064 }
7065 EXPORT_SYMBOL(netdev_master_upper_dev_link);
7066 
7067 /**
7068  * netdev_upper_dev_unlink - Removes a link to upper device
7069  * @dev: device
7070  * @upper_dev: new upper device
7071  *
7072  * Removes a link to device which is upper to this one. The caller must hold
7073  * the RTNL lock.
7074  */
7075 void netdev_upper_dev_unlink(struct net_device *dev,
7076                              struct net_device *upper_dev)
7077 {
7078         struct netdev_notifier_changeupper_info changeupper_info = {
7079                 .info = {
7080                         .dev = dev,
7081                 },
7082                 .upper_dev = upper_dev,
7083                 .linking = false,
7084         };
7085 
7086         ASSERT_RTNL();
7087 
7088         changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
7089 
7090         call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
7091                                       &changeupper_info.info);
7092 
7093         __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
7094 
7095         call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
7096                                       &changeupper_info.info);
7097 }
7098 EXPORT_SYMBOL(netdev_upper_dev_unlink);
7099 
7100 /**
7101  * netdev_bonding_info_change - Dispatch event about slave change
7102  * @dev: device
7103  * @bonding_info: info to dispatch
7104  *
7105  * Send NETDEV_BONDING_INFO to netdev notifiers with info.
7106  * The caller must hold the RTNL lock.
7107  */
7108 void netdev_bonding_info_change(struct net_device *dev,
7109                                 struct netdev_bonding_info *bonding_info)
7110 {
7111         struct netdev_notifier_bonding_info info = {
7112                 .info.dev = dev,
7113         };
7114 
7115         memcpy(&info.bonding_info, bonding_info,
7116                sizeof(struct netdev_bonding_info));
7117         call_netdevice_notifiers_info(NETDEV_BONDING_INFO,
7118                                       &info.info);
7119 }
7120 EXPORT_SYMBOL(netdev_bonding_info_change);
7121 
7122 static void netdev_adjacent_add_links(struct net_device *dev)
7123 {
7124         struct netdev_adjacent *iter;
7125 
7126         struct net *net = dev_net(dev);
7127 
7128         list_for_each_entry(iter, &dev->adj_list.upper, list) {
7129                 if (!net_eq(net, dev_net(iter->dev)))
7130                         continue;
7131                 netdev_adjacent_sysfs_add(iter->dev, dev,
7132                                           &iter->dev->adj_list.lower);
7133                 netdev_adjacent_sysfs_add(dev, iter->dev,
7134                                           &dev->adj_list.upper);
7135         }
7136 
7137         list_for_each_entry(iter, &dev->adj_list.lower, list) {
7138                 if (!net_eq(net, dev_net(iter->dev)))
7139                         continue;
7140                 netdev_adjacent_sysfs_add(iter->dev, dev,
7141                                           &iter->dev->adj_list.upper);
7142                 netdev_adjacent_sysfs_add(dev, iter->dev,
7143                                           &dev->adj_list.lower);
7144         }
7145 }
7146 
7147 static void netdev_adjacent_del_links(struct net_device *dev)
7148 {
7149         struct netdev_adjacent *iter;
7150 
7151         struct net *net = dev_net(dev);
7152 
7153         list_for_each_entry(iter, &dev->adj_list.upper, list) {
7154                 if (!net_eq(net, dev_net(iter->dev)))
7155                         continue;
7156                 netdev_adjacent_sysfs_del(iter->dev, dev->name,
7157                                           &iter->dev->adj_list.lower);
7158                 netdev_adjacent_sysfs_del(dev, iter->dev->name,
7159                                           &dev->adj_list.upper);
7160         }
7161 
7162         list_for_each_entry(iter, &dev->adj_list.lower, list) {
7163                 if (!net_eq(net, dev_net(iter->dev)))
7164                         continue;
7165                 netdev_adjacent_sysfs_del(iter->dev, dev->name,
7166                                           &iter->dev->adj_list.upper);
7167                 netdev_adjacent_sysfs_del(dev, iter->dev->name,
7168                                           &dev->adj_list.lower);
7169         }
7170 }
7171 
7172 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
7173 {
7174         struct netdev_adjacent *iter;
7175 
7176         struct net *net = dev_net(dev);
7177 
7178         list_for_each_entry(iter, &dev->adj_list.upper, list) {
7179                 if (!net_eq(net, dev_net(iter->dev)))
7180                         continue;
7181                 netdev_adjacent_sysfs_del(iter->dev, oldname,
7182                                           &iter->dev->adj_list.lower);
7183                 netdev_adjacent_sysfs_add(iter->dev, dev,
7184                                           &iter->dev->adj_list.lower);
7185         }
7186 
7187         list_for_each_entry(iter, &dev->adj_list.lower, list) {
7188                 if (!net_eq(net, dev_net(iter->dev)))
7189                         continue;
7190                 netdev_adjacent_sysfs_del(iter->dev, oldname,
7191                                           &iter->dev->adj_list.upper);
7192                 netdev_adjacent_sysfs_add(iter->dev, dev,
7193                                           &iter->dev->adj_list.upper);
7194         }
7195 }
7196 
7197 void *netdev_lower_dev_get_private(struct net_device *dev,
7198                                    struct net_device *lower_dev)
7199 {
7200         struct netdev_adjacent *lower;
7201 
7202         if (!lower_dev)
7203                 return NULL;
7204         lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
7205         if (!lower)
7206                 return NULL;
7207 
7208         return lower->private;
7209 }
7210 EXPORT_SYMBOL(netdev_lower_dev_get_private);
7211 
7212 
7213 int dev_get_nest_level(struct net_device *dev)
7214 {
7215         struct net_device *lower = NULL;
7216         struct list_head *iter;
7217         int max_nest = -1;
7218         int nest;
7219 
7220         ASSERT_RTNL();
7221 
7222         netdev_for_each_lower_dev(dev, lower, iter) {
7223                 nest = dev_get_nest_level(lower);
7224                 if (max_nest < nest)
7225                         max_nest = nest;
7226         }
7227 
7228         return max_nest + 1;
7229 }
7230 EXPORT_SYMBOL(dev_get_nest_level);
7231 
7232 /**
7233  * netdev_lower_change - Dispatch event about lower device state change
7234  * @lower_dev: device
7235  * @lower_state_info: state to dispatch
7236  *
7237  * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
7238  * The caller must hold the RTNL lock.
7239  */
7240 void netdev_lower_state_changed(struct net_device *lower_dev,
7241                                 void *lower_state_info)
7242 {
7243         struct netdev_notifier_changelowerstate_info changelowerstate_info = {
7244                 .info.dev = lower_dev,
7245         };
7246 
7247         ASSERT_RTNL();
7248         changelowerstate_info.lower_state_info = lower_state_info;
7249         call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE,
7250                                       &changelowerstate_info.info);
7251 }
7252 EXPORT_SYMBOL(netdev_lower_state_changed);
7253 
7254 static void dev_change_rx_flags(struct net_device *dev, int flags)
7255 {
7256         const struct net_device_ops *ops = dev->netdev_ops;
7257 
7258         if (ops->ndo_change_rx_flags)
7259                 ops->ndo_change_rx_flags(dev, flags);
7260 }
7261 
7262 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
7263 {
7264         unsigned int old_flags = dev->flags;
7265         kuid_t uid;
7266         kgid_t gid;
7267 
7268         ASSERT_RTNL();
7269 
7270         dev->flags |= IFF_PROMISC;
7271         dev->promiscuity += inc;
7272         if (dev->promiscuity == 0) {
7273                 /*
7274                  * Avoid overflow.
7275                  * If inc causes overflow, untouch promisc and return error.
7276                  */
7277                 if (inc < 0)
7278                         dev->flags &= ~IFF_PROMISC;
7279                 else {
7280                         dev->promiscuity -= inc;
7281                         pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
7282                                 dev->name);
7283                         return -EOVERFLOW;
7284                 }
7285         }
7286         if (dev->flags != old_flags) {
7287                 pr_info("device %s %s promiscuous mode\n",
7288                         dev->name,
7289                         dev->flags & IFF_PROMISC ? "entered" : "left");
7290                 if (audit_enabled) {
7291                         current_uid_gid(&uid, &gid);
7292                         audit_log(audit_context(), GFP_ATOMIC,
7293                                   AUDIT_ANOM_PROMISCUOUS,
7294                                   "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
7295                                   dev->name, (dev->flags & IFF_PROMISC),
7296                                   (old_flags & IFF_PROMISC),
7297                                   from_kuid(&init_user_ns, audit_get_loginuid(current)),
7298                                   from_kuid(&init_user_ns, uid),
7299                                   from_kgid(&init_user_ns, gid),
7300                                   audit_get_sessionid(current));
7301                 }
7302 
7303                 dev_change_rx_flags(dev, IFF_PROMISC);
7304         }
7305         if (notify)
7306                 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
7307         return 0;
7308 }
7309 
7310 /**
7311  *      dev_set_promiscuity     - update promiscuity count on a device
7312  *      @dev: device
7313  *      @inc: modifier
7314  *
7315  *      Add or remove promiscuity from a device. While the count in the device
7316  *      remains above zero the interface remains promiscuous. Once it hits zero
7317  *      the device reverts back to normal filtering operation. A negative inc
7318  *      value is used to drop promiscuity on the device.
7319  *      Return 0 if successful or a negative errno code on error.
7320  */
7321 int dev_set_promiscuity(struct net_device *dev, int inc)
7322 {
7323         unsigned int old_flags = dev->flags;
7324         int err;
7325 
7326         err = __dev_set_promiscuity(dev, inc, true);
7327         if (err < 0)
7328                 return err;
7329         if (dev->flags != old_flags)
7330                 dev_set_rx_mode(dev);
7331         return err;
7332 }
7333 EXPORT_SYMBOL(dev_set_promiscuity);
7334 
7335 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
7336 {
7337         unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
7338 
7339         ASSERT_RTNL();
7340 
7341         dev->flags |= IFF_ALLMULTI;
7342         dev->allmulti += inc;
7343         if (dev->allmulti == 0) {
7344                 /*
7345                  * Avoid overflow.
7346                  * If inc causes overflow, untouch allmulti and return error.
7347                  */
7348                 if (inc < 0)
7349                         dev->flags &= ~IFF_ALLMULTI;
7350                 else {
7351                         dev->allmulti -= inc;
7352                         pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
7353                                 dev->name);
7354                         return -EOVERFLOW;
7355                 }
7356         }
7357         if (dev->flags ^ old_flags) {
7358                 dev_change_rx_flags(dev, IFF_ALLMULTI);
7359                 dev_set_rx_mode(dev);
7360                 if (notify)
7361                         __dev_notify_flags(dev, old_flags,
7362                                            dev->gflags ^ old_gflags);
7363         }
7364         return 0;
7365 }
7366 
7367 /**
7368  *      dev_set_allmulti        - update allmulti count on a device
7369  *      @dev: device
7370  *      @inc: modifier
7371  *
7372  *      Add or remove reception of all multicast frames to a device. While the
7373  *      count in the device remains above zero the interface remains listening
7374  *      to all interfaces. Once it hits zero the device reverts back to normal
7375  *      filtering operation. A negative @inc value is used to drop the counter
7376  *      when releasing a resource needing all multicasts.
7377  *      Return 0 if successful or a negative errno code on error.
7378  */
7379 
7380 int dev_set_allmulti(struct net_device *dev, int inc)
7381 {
7382         return __dev_set_allmulti(dev, inc, true);
7383 }
7384 EXPORT_SYMBOL(dev_set_allmulti);
7385 
7386 /*
7387  *      Upload unicast and multicast address lists to device and
7388  *      configure RX filtering. When the device doesn't support unicast
7389  *      filtering it is put in promiscuous mode while unicast addresses
7390  *      are present.
7391  */
7392 void __dev_set_rx_mode(struct net_device *dev)
7393 {
7394         const struct net_device_ops *ops = dev->netdev_ops;
7395 
7396         /* dev_open will call this function so the list will stay sane. */
7397         if (!(dev->flags&IFF_UP))
7398                 return;
7399 
7400         if (!netif_device_present(dev))
7401                 return;
7402 
7403         if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
7404                 /* Unicast addresses changes may only happen under the rtnl,
7405                  * therefore calling __dev_set_promiscuity here is safe.
7406                  */
7407                 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
7408                         __dev_set_promiscuity(dev, 1, false);
7409                         dev->uc_promisc = true;
7410                 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
7411                         __dev_set_promiscuity(dev, -1, false);
7412                         dev->uc_promisc = false;
7413                 }
7414         }
7415 
7416         if (ops->ndo_set_rx_mode)
7417                 ops->ndo_set_rx_mode(dev);
7418 }
7419 
7420 void dev_set_rx_mode(struct net_device *dev)
7421 {
7422         netif_addr_lock_bh(dev);
7423         __dev_set_rx_mode(dev);
7424         netif_addr_unlock_bh(dev);
7425 }
7426 
7427 /**
7428  *      dev_get_flags - get flags reported to userspace
7429  *      @dev: device
7430  *
7431  *      Get the combination of flag bits exported through APIs to userspace.
7432  */
7433 unsigned int dev_get_flags(const struct net_device *dev)
7434 {
7435         unsigned int flags;
7436 
7437         flags = (dev->flags & ~(IFF_PROMISC |
7438                                 IFF_ALLMULTI |
7439                                 IFF_RUNNING |
7440                                 IFF_LOWER_UP |
7441                                 IFF_DORMANT)) |
7442                 (dev->gflags & (IFF_PROMISC |
7443                                 IFF_ALLMULTI));
7444 
7445         if (netif_running(dev)) {
7446                 if (netif_oper_up(dev))
7447                         flags |= IFF_RUNNING;
7448                 if (netif_carrier_ok(dev))
7449                         flags |= IFF_LOWER_UP;
7450                 if (netif_dormant(dev))
7451                         flags |= IFF_DORMANT;
7452         }
7453 
7454         return flags;
7455 }
7456 EXPORT_SYMBOL(dev_get_flags);
7457 
7458 int __dev_change_flags(struct net_device *dev, unsigned int flags)
7459 {
7460         unsigned int old_flags = dev->flags;
7461         int ret;
7462 
7463         ASSERT_RTNL();
7464 
7465         /*
7466          *      Set the flags on our device.
7467          */
7468 
7469         dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
7470                                IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
7471                                IFF_AUTOMEDIA)) |
7472                      (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
7473                                     IFF_ALLMULTI));
7474 
7475         /*
7476          *      Load in the correct multicast list now the flags have changed.
7477          */
7478 
7479         if ((old_flags ^ flags) & IFF_MULTICAST)
7480                 dev_change_rx_flags(dev, IFF_MULTICAST);
7481 
7482         dev_set_rx_mode(dev);
7483 
7484         /*
7485          *      Have we downed the interface. We handle IFF_UP ourselves
7486          *      according to user attempts to set it, rather than blindly
7487          *      setting it.
7488          */
7489 
7490         ret = 0;
7491         if ((old_flags ^ flags) & IFF_UP) {
7492                 if (old_flags & IFF_UP)
7493                         __dev_close(dev);
7494                 else
7495                         ret = __dev_open(dev);
7496         }
7497 
7498         if ((flags ^ dev->gflags) & IFF_PROMISC) {
7499                 int inc = (flags & IFF_PROMISC) ? 1 : -1;
7500                 unsigned int old_flags = dev->flags;
7501 
7502                 dev->gflags ^= IFF_PROMISC;
7503 
7504                 if (__dev_set_promiscuity(dev, inc, false) >= 0)
7505                         if (dev->flags != old_flags)
7506                                 dev_set_rx_mode(dev);
7507         }
7508 
7509         /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
7510          * is important. Some (broken) drivers set IFF_PROMISC, when
7511          * IFF_ALLMULTI is requested not asking us and not reporting.
7512          */
7513         if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
7514                 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
7515 
7516                 dev->gflags ^= IFF_ALLMULTI;
7517                 __dev_set_allmulti(dev, inc, false);
7518         }
7519 
7520         return ret;
7521 }
7522 
7523 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
7524                         unsigned int gchanges)
7525 {
7526         unsigned int changes = dev->flags ^ old_flags;
7527 
7528         if (gchanges)
7529                 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
7530 
7531         if (changes & IFF_UP) {
7532                 if (dev->flags & IFF_UP)
7533                         call_netdevice_notifiers(NETDEV_UP, dev);
7534                 else
7535                         call_netdevice_notifiers(NETDEV_DOWN, dev);
7536         }
7537 
7538         if (dev->flags & IFF_UP &&
7539             (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
7540                 struct netdev_notifier_change_info change_info = {
7541                         .info = {
7542                                 .dev = dev,
7543                         },
7544                         .flags_changed = changes,
7545                 };
7546 
7547                 call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info);
7548         }
7549 }
7550 
7551 /**
7552  *      dev_change_flags - change device settings
7553  *      @dev: device
7554  *      @flags: device state flags
7555  *
7556  *      Change settings on device based state flags. The flags are
7557  *      in the userspace exported format.
7558  */
7559 int dev_change_flags(struct net_device *dev, unsigned int flags)
7560 {
7561         int ret;
7562         unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
7563 
7564         ret = __dev_change_flags(dev, flags);
7565         if (ret < 0)
7566                 return ret;
7567 
7568         changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
7569         __dev_notify_flags(dev, old_flags, changes);
7570         return ret;
7571 }
7572 EXPORT_SYMBOL(dev_change_flags);
7573 
7574 int __dev_set_mtu(struct net_device *dev, int new_mtu)
7575 {
7576         const struct net_device_ops *ops = dev->netdev_ops;
7577 
7578         if (ops->ndo_change_mtu)
7579                 return ops->ndo_change_mtu(dev, new_mtu);
7580 
7581         dev->mtu = new_mtu;
7582         return 0;
7583 }
7584 EXPORT_SYMBOL(__dev_set_mtu);
7585 
7586 /**
7587  *      dev_set_mtu_ext - Change maximum transfer unit
7588  *      @dev: device
7589  *      @new_mtu: new transfer unit
7590  *      @extack: netlink extended ack
7591  *
7592  *      Change the maximum transfer size of the network device.
7593  */
7594 int dev_set_mtu_ext(struct net_device *dev, int new_mtu,
7595                     struct netlink_ext_ack *extack)
7596 {
7597         int err, orig_mtu;
7598 
7599         if (new_mtu == dev->mtu)
7600                 return 0;
7601 
7602         /* MTU must be positive, and in range */
7603         if (new_mtu < 0 || new_mtu < dev->min_mtu) {
7604                 NL_SET_ERR_MSG(extack, "mtu less than device minimum");
7605                 return -EINVAL;
7606         }
7607 
7608         if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
7609                 NL_SET_ERR_MSG(extack, "mtu greater than device maximum");
7610                 return -EINVAL;
7611         }
7612 
7613         if (!netif_device_present(dev))
7614                 return -ENODEV;
7615 
7616         err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
7617         err = notifier_to_errno(err);
7618         if (err)
7619                 return err;
7620 
7621         orig_mtu = dev->mtu;
7622         err = __dev_set_mtu(dev, new_mtu);
7623 
7624         if (!err) {
7625                 err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
7626                                                    orig_mtu);
7627                 err = notifier_to_errno(err);
7628                 if (err) {
7629                         /* setting mtu back and notifying everyone again,
7630                          * so that they have a chance to revert changes.
7631                          */
7632                         __dev_set_mtu(dev, orig_mtu);
7633                         call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
7634                                                      new_mtu);
7635                 }
7636         }
7637         return err;
7638 }
7639 
7640 int dev_set_mtu(struct net_device *dev, int new_mtu)
7641 {
7642         struct netlink_ext_ack extack;
7643         int err;
7644 
7645         memset(&extack, 0, sizeof(extack));
7646         err = dev_set_mtu_ext(dev, new_mtu, &extack);
7647         if (err && extack._msg)
7648                 net_err_ratelimited("%s: %s\n", dev->name, extack._msg);
7649         return err;
7650 }
7651 EXPORT_SYMBOL(dev_set_mtu);
7652 
7653 /**
7654  *      dev_change_tx_queue_len - Change TX queue length of a netdevice
7655  *      @dev: device
7656  *      @new_len: new tx queue length
7657  */
7658 int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len)
7659 {
7660         unsigned int orig_len = dev->tx_queue_len;
7661         int res;
7662 
7663         if (new_len != (unsigned int)new_len)
7664                 return -ERANGE;
7665 
7666         if (new_len != orig_len) {
7667                 dev->tx_queue_len = new_len;
7668                 res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev);
7669                 res = notifier_to_errno(res);
7670                 if (res)
7671                         goto err_rollback;
7672                 res = dev_qdisc_change_tx_queue_len(dev);
7673                 if (res)
7674                         goto err_rollback;
7675         }
7676 
7677         return 0;
7678 
7679 err_rollback:
7680         netdev_err(dev, "refused to change device tx_queue_len\n");
7681         dev->tx_queue_len = orig_len;
7682         return res;
7683 }
7684 
7685 /**
7686  *      dev_set_group - Change group this device belongs to
7687  *      @dev: device
7688  *      @new_group: group this device should belong to
7689  */
7690 void dev_set_group(struct net_device *dev, int new_group)
7691 {
7692         dev->group = new_group;
7693 }
7694 EXPORT_SYMBOL(dev_set_group);
7695 
7696 /**
7697  *      dev_set_mac_address - Change Media Access Control Address
7698  *      @dev: device
7699  *      @sa: new address
7700  *
7701  *      Change the hardware (MAC) address of the device
7702  */
7703 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
7704 {
7705         const struct net_device_ops *ops = dev->netdev_ops;
7706         int err;
7707 
7708         if (!ops->ndo_set_mac_address)
7709                 return -EOPNOTSUPP;
7710         if (sa->sa_family != dev->type)
7711                 return -EINVAL;
7712         if (!netif_device_present(dev))
7713                 return -ENODEV;
7714         err = ops->ndo_set_mac_address(dev, sa);
7715         if (err)
7716                 return err;
7717         dev->addr_assign_type = NET_ADDR_SET;
7718         call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
7719         add_device_randomness(dev->dev_addr, dev->addr_len);
7720         return 0;
7721 }
7722 EXPORT_SYMBOL(dev_set_mac_address);
7723 
7724 /**
7725  *      dev_change_carrier - Change device carrier
7726  *      @dev: device
7727  *      @new_carrier: new value
7728  *
7729  *      Change device carrier
7730  */
7731 int dev_change_carrier(struct net_device *dev, bool new_carrier)
7732 {
7733         const struct net_device_ops *ops = dev->netdev_ops;
7734 
7735         if (!ops->ndo_change_carrier)
7736                 return -EOPNOTSUPP;
7737         if (!netif_device_present(dev))
7738                 return -ENODEV;
7739         return ops->ndo_change_carrier(dev, new_carrier);
7740 }
7741 EXPORT_SYMBOL(dev_change_carrier);
7742 
7743 /**
7744  *      dev_get_phys_port_id - Get device physical port ID
7745  *      @dev: device
7746  *      @ppid: port ID
7747  *
7748  *      Get device physical port ID
7749  */
7750 int dev_get_phys_port_id(struct net_device *dev,
7751                          struct netdev_phys_item_id *ppid)
7752 {
7753         const struct net_device_ops *ops = dev->netdev_ops;
7754 
7755         if (!ops->ndo_get_phys_port_id)
7756                 return -EOPNOTSUPP;
7757         return ops->ndo_get_phys_port_id(dev, ppid);
7758 }
7759 EXPORT_SYMBOL(dev_get_phys_port_id);
7760 
7761 /**
7762  *      dev_get_phys_port_name - Get device physical port name
7763  *      @dev: device
7764  *      @name: port name
7765  *      @len: limit of bytes to copy to name
7766  *
7767  *      Get device physical port name
7768  */
7769 int dev_get_phys_port_name(struct net_device *dev,
7770                            char *name, size_t len)
7771 {
7772         const struct net_device_ops *ops = dev->netdev_ops;
7773 
7774         if (!ops->ndo_get_phys_port_name)
7775                 return -EOPNOTSUPP;
7776         return ops->ndo_get_phys_port_name(dev, name, len);
7777 }
7778 EXPORT_SYMBOL(dev_get_phys_port_name);
7779 
7780 /**
7781  *      dev_change_proto_down - update protocol port state information
7782  *      @dev: device
7783  *      @proto_down: new value
7784  *
7785  *      This info can be used by switch drivers to set the phys state of the
7786  *      port.
7787  */
7788 int dev_change_proto_down(struct net_device *dev, bool proto_down)
7789 {
7790         const struct net_device_ops *ops = dev->netdev_ops;
7791 
7792         if (!ops->ndo_change_proto_down)
7793                 return -EOPNOTSUPP;
7794         if (!netif_device_present(dev))
7795                 return -ENODEV;
7796         return ops->ndo_change_proto_down(dev, proto_down);
7797 }
7798 EXPORT_SYMBOL(dev_change_proto_down);
7799 
7800 u32 __dev_xdp_query(struct net_device *dev, bpf_op_t bpf_op,
7801                     enum bpf_netdev_command cmd)
7802 {
7803         struct netdev_bpf xdp;
7804 
7805         if (!bpf_op)
7806                 return 0;
7807 
7808         memset(&xdp, 0, sizeof(xdp));
7809         xdp.command = cmd;
7810 
7811         /* Query must always succeed. */
7812         WARN_ON(bpf_op(dev, &xdp) < 0 && cmd == XDP_QUERY_PROG);
7813 
7814         return xdp.prog_id;
7815 }
7816 
7817 static int dev_xdp_install(struct net_device *dev, bpf_op_t bpf_op,
7818                            struct netlink_ext_ack *extack, u32 flags,
7819                            struct bpf_prog *prog)
7820 {
7821         struct netdev_bpf xdp;
7822 
7823         memset(&xdp, 0, sizeof(xdp));
7824         if (flags & XDP_FLAGS_HW_MODE)
7825                 xdp.command = XDP_SETUP_PROG_HW;
7826         else
7827                 xdp.command = XDP_SETUP_PROG;
7828         xdp.extack = extack;
7829         xdp.flags = flags;
7830         xdp.prog = prog;
7831 
7832         return bpf_op(dev, &xdp);
7833 }
7834 
7835 static void dev_xdp_uninstall(struct net_device *dev)
7836 {
7837         struct netdev_bpf xdp;
7838         bpf_op_t ndo_bpf;
7839 
7840         /* Remove generic XDP */
7841         WARN_ON(dev_xdp_install(dev, generic_xdp_install, NULL, 0, NULL));
7842 
7843         /* Remove from the driver */
7844         ndo_bpf = dev->netdev_ops->ndo_bpf;
7845         if (!ndo_bpf)
7846                 return;
7847 
7848         memset(&xdp, 0, sizeof(xdp));
7849         xdp.command = XDP_QUERY_PROG;
7850         WARN_ON(ndo_bpf(dev, &xdp));
7851         if (xdp.prog_id)
7852                 WARN_ON(dev_xdp_install(dev, ndo_bpf, NULL, xdp.prog_flags,
7853                                         NULL));
7854 
7855         /* Remove HW offload */
7856         memset(&xdp, 0, sizeof(xdp));
7857         xdp.command = XDP_QUERY_PROG_HW;
7858         if (!ndo_bpf(dev, &xdp) && xdp.prog_id)
7859                 WARN_ON(dev_xdp_install(dev, ndo_bpf, NULL, xdp.prog_flags,
7860                                         NULL));
7861 }
7862 
7863 /**
7864  *      dev_change_xdp_fd - set or clear a bpf program for a device rx path
7865  *      @dev: device
7866  *      @extack: netlink extended ack
7867  *      @fd: new program fd or negative value to clear
7868  *      @flags: xdp-related flags
7869  *
7870  *      Set or clear a bpf program for a device
7871  */
7872 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
7873                       int fd, u32 flags)
7874 {
7875         const struct net_device_ops *ops = dev->netdev_ops;
7876         enum bpf_netdev_command query;
7877         struct bpf_prog *prog = NULL;
7878         bpf_op_t bpf_op, bpf_chk;
7879         int err;
7880 
7881         ASSERT_RTNL();
7882 
7883         query = flags & XDP_FLAGS_HW_MODE ? XDP_QUERY_PROG_HW : XDP_QUERY_PROG;
7884 
7885         bpf_op = bpf_chk = ops->ndo_bpf;
7886         if (!bpf_op && (flags & (XDP_FLAGS_DRV_MODE | XDP_FLAGS_HW_MODE)))
7887                 return -EOPNOTSUPP;
7888         if (!bpf_op || (flags & XDP_FLAGS_SKB_MODE))
7889                 bpf_op = generic_xdp_install;
7890         if (bpf_op == bpf_chk)
7891                 bpf_chk = generic_xdp_install;
7892 
7893         if (fd >= 0) {
7894                 if (__dev_xdp_query(dev, bpf_chk, XDP_QUERY_PROG) ||
7895                     __dev_xdp_query(dev, bpf_chk, XDP_QUERY_PROG_HW))
7896                         return -EEXIST;
7897                 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) &&
7898                     __dev_xdp_query(dev, bpf_op, query))
7899                         return -EBUSY;
7900 
7901                 prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,
7902                                              bpf_op == ops->ndo_bpf);
7903                 if (IS_ERR(prog))
7904                         return PTR_ERR(prog);
7905 
7906                 if (!(flags & XDP_FLAGS_HW_MODE) &&
7907                     bpf_prog_is_dev_bound(prog->aux)) {
7908                         NL_SET_ERR_MSG(extack, "using device-bound program without HW_MODE flag is not supported");
7909                         bpf_prog_put(prog);
7910                         return -EINVAL;
7911                 }
7912         }
7913 
7914         err = dev_xdp_install(dev, bpf_op, extack, flags, prog);
7915         if (err < 0 && prog)
7916                 bpf_prog_put(prog);
7917 
7918         return err;
7919 }
7920 
7921 /**
7922  *      dev_new_index   -       allocate an ifindex
7923  *      @net: the applicable net namespace
7924  *
7925  *      Returns a suitable unique value for a new device interface
7926  *      number.  The caller must hold the rtnl semaphore or the
7927  *      dev_base_lock to be sure it remains unique.
7928  */
7929 static int dev_new_index(struct net *net)
7930 {
7931         int ifindex = net->ifindex;
7932 
7933         for (;;) {
7934                 if (++ifindex <= 0)
7935                         ifindex = 1;
7936                 if (!__dev_get_by_index(net, ifindex))
7937                         return net->ifindex = ifindex;
7938         }
7939 }
7940 
7941 /* Delayed registration/unregisteration */
7942 static LIST_HEAD(net_todo_list);
7943 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
7944 
7945 static void net_set_todo(struct net_device *dev)
7946 {
7947         list_add_tail(&dev->todo_list, &net_todo_list);
7948         dev_net(dev)->dev_unreg_count++;
7949 }
7950 
7951 static void rollback_registered_many(struct list_head *head)
7952 {
7953         struct net_device *dev, *tmp;
7954         LIST_HEAD(close_head);
7955 
7956         BUG_ON(dev_boot_phase);
7957         ASSERT_RTNL();
7958 
7959         list_for_each_entry_safe(dev, tmp, head, unreg_list) {
7960                 /* Some devices call without registering
7961                  * for initialization unwind. Remove those
7962                  * devices and proceed with the remaining.
7963                  */
7964                 if (dev->reg_state == NETREG_UNINITIALIZED) {
7965                         pr_debug("unregister_netdevice: device %s/%p never was registered\n",
7966                                  dev->name, dev);
7967 
7968                         WARN_ON(1);
7969                         list_del(&dev->unreg_list);
7970                         continue;
7971                 }
7972                 dev->dismantle = true;
7973                 BUG_ON(dev->reg_state != NETREG_REGISTERED);
7974         }
7975 
7976         /* If device is running, close it first. */
7977         list_for_each_entry(dev, head, unreg_list)
7978                 list_add_tail(&dev->close_list, &close_head);
7979         dev_close_many(&close_head, true);
7980 
7981         list_for_each_entry(dev, head, unreg_list) {
7982                 /* And unlink it from device chain. */
7983                 unlist_netdevice(dev);
7984 
7985                 dev->reg_state = NETREG_UNREGISTERING;
7986         }
7987         flush_all_backlogs();
7988 
7989         synchronize_net();
7990 
7991         list_for_each_entry(dev, head, unreg_list) {
7992                 struct sk_buff *skb = NULL;
7993 
7994                 /* Shutdown queueing discipline. */
7995                 dev_shutdown(dev);
7996 
7997                 dev_xdp_uninstall(dev);
7998 
7999                 /* Notify protocols, that we are about to destroy
8000                  * this device. They should clean all the things.
8001                  */
8002                 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
8003 
8004                 if (!dev->rtnl_link_ops ||
8005                     dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
8006                         skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
8007                                                      GFP_KERNEL, NULL, 0);
8008 
8009                 /*
8010                  *      Flush the unicast and multicast chains
8011                  */
8012                 dev_uc_flush(dev);
8013                 dev_mc_flush(dev);
8014 
8015                 if (dev->netdev_ops->ndo_uninit)
8016                         dev->netdev_ops->ndo_uninit(dev);
8017 
8018                 if (skb)
8019                         rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
8020 
8021                 /* Notifier chain MUST detach us all upper devices. */
8022                 WARN_ON(netdev_has_any_upper_dev(dev));
8023                 WARN_ON(netdev_has_any_lower_dev(dev));
8024 
8025                 /* Remove entries from kobject tree */
8026                 netdev_unregister_kobject(dev);
8027 #ifdef CONFIG_XPS
8028                 /* Remove XPS queueing entries */
8029                 netif_reset_xps_queues_gt(dev, 0);
8030 #endif
8031         }
8032 
8033         synchronize_net();
8034 
8035         list_for_each_entry(dev, head, unreg_list)
8036                 dev_put(dev);
8037 }
8038 
8039 static void rollback_registered(struct net_device *dev)
8040 {
8041         LIST_HEAD(single);
8042 
8043         list_add(&dev->unreg_list, &single);
8044         rollback_registered_many(&single);
8045         list_del(&single);
8046 }
8047 
8048 static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
8049         struct net_device *upper, netdev_features_t features)
8050 {
8051         netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
8052         netdev_features_t feature;
8053         int feature_bit;
8054 
8055         for_each_netdev_feature(&upper_disables, feature_bit) {
8056                 feature = __NETIF_F_BIT(feature_bit);
8057                 if (!(upper->wanted_features & feature)
8058                     && (features & feature)) {
8059                         netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
8060                                    &feature, upper->name);
8061                         features &= ~feature;
8062                 }
8063         }
8064 
8065         return features;
8066 }
8067 
8068 static void netdev_sync_lower_features(struct net_device *upper,
8069         struct net_device *lower, netdev_features_t features)
8070 {
8071         netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
8072         netdev_features_t feature;
8073         int feature_bit;
8074 
8075         for_each_netdev_feature(&upper_disables, feature_bit) {
8076                 feature = __NETIF_F_BIT(feature_bit);
8077                 if (!(features & feature) && (lower->features & feature)) {
8078                         netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
8079                                    &feature, lower->name);
8080                         lower->wanted_features &= ~feature;
8081                         netdev_update_features(lower);
8082 
8083                         if (unlikely(lower->features & feature))
8084                                 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
8085                                             &feature, lower->name);
8086                 }
8087         }
8088 }
8089 
8090 static netdev_features_t netdev_fix_features(struct net_device *dev,
8091         netdev_features_t features)
8092 {
8093         /* Fix illegal checksum combinations */
8094         if ((features & NETIF_F_HW_CSUM) &&
8095             (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
8096                 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
8097                 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
8098         }
8099 
8100         /* TSO requires that SG is present as well. */
8101         if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
8102                 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
8103                 features &= ~NETIF_F_ALL_TSO;
8104         }
8105 
8106         if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
8107                                         !(features & NETIF_F_IP_CSUM)) {
8108                 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
8109                 features &= ~NETIF_F_TSO;
8110                 features &= ~NETIF_F_TSO_ECN;
8111         }
8112 
8113         if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
8114                                          !(features & NETIF_F_IPV6_CSUM)) {
8115                 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
8116                 features &= ~NETIF_F_TSO6;
8117         }
8118 
8119         /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
8120         if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
8121                 features &= ~NETIF_F_TSO_MANGLEID;
8122 
8123         /* TSO ECN requires that TSO is present as well. */
8124         if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
8125                 features &= ~NETIF_F_TSO_ECN;
8126 
8127         /* Software GSO depends on SG. */
8128         if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
8129                 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
8130                 features &= ~NETIF_F_GSO;
8131         }
8132 
8133         /* GSO partial features require GSO partial be set */
8134         if ((features & dev->gso_partial_features) &&
8135             !(features & NETIF_F_GSO_PARTIAL)) {
8136                 netdev_dbg(dev,
8137                            "Dropping partially supported GSO features since no GSO partial.\n");
8138                 features &= ~dev->gso_partial_features;
8139         }
8140 
8141         if (!(features & NETIF_F_RXCSUM)) {
8142                 /* NETIF_F_GRO_HW implies doing RXCSUM since every packet
8143                  * successfully merged by hardware must also have the
8144                  * checksum verified by hardware.  If the user does not
8145                  * want to enable RXCSUM, logically, we should disable GRO_HW.
8146                  */
8147                 if (features & NETIF_F_GRO_HW) {
8148                         netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n");
8149                         features &= ~NETIF_F_GRO_HW;
8150                 }
8151         }
8152 
8153         /* LRO/HW-GRO features cannot be combined with RX-FCS */
8154         if (features & NETIF_F_RXFCS) {
8155                 if (features & NETIF_F_LRO) {
8156                         netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n");
8157                         features &= ~NETIF_F_LRO;
8158                 }
8159 
8160                 if (features & NETIF_F_GRO_HW) {
8161                         netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n");
8162                         features &= ~NETIF_F_GRO_HW;
8163                 }
8164         }
8165 
8166         return features;
8167 }
8168 
8169 int __netdev_update_features(struct net_device *dev)
8170 {
8171         struct net_device *upper, *lower;
8172         netdev_features_t features;
8173         struct list_head *iter;
8174         int err = -1;
8175 
8176         ASSERT_RTNL();
8177 
8178         features = netdev_get_wanted_features(dev);
8179 
8180         if (dev->netdev_ops->ndo_fix_features)
8181                 features = dev->netdev_ops->ndo_fix_features(dev, features);
8182 
8183         /* driver might be less strict about feature dependencies */
8184         features = netdev_fix_features(dev, features);
8185 
8186         /* some features can't be enabled if they're off an an upper device */
8187         netdev_for_each_upper_dev_rcu(dev, upper, iter)
8188                 features = netdev_sync_upper_features(dev, upper, features);
8189 
8190         if (dev->features == features)
8191                 goto sync_lower;
8192 
8193         netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
8194                 &dev->features, &features);
8195 
8196         if (dev->netdev_ops->ndo_set_features)
8197                 err = dev->netdev_ops->ndo_set_features(dev, features);
8198         else
8199                 err = 0;
8200 
8201         if (unlikely(err < 0)) {
8202                 netdev_err(dev,
8203                         "set_features() failed (%d); wanted %pNF, left %pNF\n",
8204                         err, &features, &dev->features);
8205                 /* return non-0 since some features might have changed and
8206                  * it's better to fire a spurious notification than miss it
8207                  */
8208                 return -1;
8209         }
8210 
8211 sync_lower:
8212         /* some features must be disabled on lower devices when disabled
8213          * on an upper device (think: bonding master or bridge)
8214          */
8215         netdev_for_each_lower_dev(dev, lower, iter)
8216                 netdev_sync_lower_features(dev, lower, features);
8217 
8218         if (!err) {
8219                 netdev_features_t diff = features ^ dev->features;
8220 
8221                 if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
8222                         /* udp_tunnel_{get,drop}_rx_info both need
8223                          * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the
8224                          * device, or they won't do anything.
8225                          * Thus we need to update dev->features
8226                          * *before* calling udp_tunnel_get_rx_info,
8227                          * but *after* calling udp_tunnel_drop_rx_info.
8228                          */
8229                         if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
8230                                 dev->features = features;
8231                                 udp_tunnel_get_rx_info(dev);
8232                         } else {
8233                                 udp_tunnel_drop_rx_info(dev);
8234                         }
8235                 }
8236 
8237                 if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) {
8238                         if (features & NETIF_F_HW_VLAN_CTAG_FILTER) {
8239                                 dev->features = features;
8240                                 err |= vlan_get_rx_ctag_filter_info(dev);
8241                         } else {
8242                                 vlan_drop_rx_ctag_filter_info(dev);
8243                         }
8244                 }
8245 
8246                 if (diff & NETIF_F_HW_VLAN_STAG_FILTER) {
8247                         if (features & NETIF_F_HW_VLAN_STAG_FILTER) {
8248                                 dev->features = features;
8249                                 err |= vlan_get_rx_stag_filter_info(dev);
8250                         } else {
8251                                 vlan_drop_rx_stag_filter_info(dev);
8252                         }
8253                 }
8254 
8255                 dev->features = features;
8256         }
8257 
8258         return err < 0 ? 0 : 1;
8259 }
8260 
8261 /**
8262  *      netdev_update_features - recalculate device features
8263  *      @dev: the device to check
8264  *
8265  *      Recalculate dev->features set and send notifications if it
8266  *      has changed. Should be called after driver or hardware dependent
8267  *      conditions might have changed that influence the features.
8268  */
8269 void netdev_update_features(struct net_device *dev)
8270 {
8271         if (__netdev_update_features(dev))
8272                 netdev_features_change(dev);
8273 }
8274 EXPORT_SYMBOL(netdev_update_features);
8275 
8276 /**
8277  *      netdev_change_features - recalculate device features
8278  *      @dev: the device to check
8279  *
8280  *      Recalculate dev->features set and send notifications even
8281  *      if they have not changed. Should be called instead of
8282  *      netdev_update_features() if also dev->vlan_features might
8283  *      have changed to allow the changes to be propagated to stacked
8284  *      VLAN devices.
8285  */
8286 void netdev_change_features(struct net_device *dev)
8287 {
8288         __netdev_update_features(dev);
8289         netdev_features_change(dev);
8290 }
8291 EXPORT_SYMBOL(netdev_change_features);
8292 
8293 /**
8294  *      netif_stacked_transfer_operstate -      transfer operstate
8295  *      @rootdev: the root or lower level device to transfer state from
8296  *      @dev: the device to transfer operstate to
8297  *
8298  *      Transfer operational state from root to device. This is normally
8299  *      called when a stacking relationship exists between the root
8300  *      device and the device(a leaf device).
8301  */
8302 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
8303                                         struct net_device *dev)
8304 {
8305         if (rootdev->operstate == IF_OPER_DORMANT)
8306                 netif_dormant_on(dev);
8307         else
8308                 netif_dormant_off(dev);
8309 
8310         if (netif_carrier_ok(rootdev))
8311                 netif_carrier_on(dev);
8312         else
8313                 netif_carrier_off(dev);
8314 }
8315 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
8316 
8317 static int netif_alloc_rx_queues(struct net_device *dev)
8318 {
8319         unsigned int i, count = dev->num_rx_queues;
8320         struct netdev_rx_queue *rx;
8321         size_t sz = count * sizeof(*rx);
8322         int err = 0;
8323 
8324         BUG_ON(count < 1);
8325 
8326         rx = kvzalloc(sz, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
8327         if (!rx)
8328                 return -ENOMEM;
8329 
8330         dev->_rx = rx;
8331 
8332         for (i = 0; i < count; i++) {
8333                 rx[i].dev = dev;
8334 
8335                 /* XDP RX-queue setup */
8336                 err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i);
8337                 if (err < 0)
8338                         goto err_rxq_info;
8339         }
8340         return 0;
8341 
8342 err_rxq_info:
8343         /* Rollback successful reg's and free other resources */
8344         while (i--)
8345                 xdp_rxq_info_unreg(&rx[i].xdp_rxq);
8346         kvfree(dev->_rx);
8347         dev->_rx = NULL;
8348         return err;
8349 }
8350 
8351 static void netif_free_rx_queues(struct net_device *dev)
8352 {
8353         unsigned int i, count = dev->num_rx_queues;