~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/include/linux/skbuff.h

Version: ~ [ linux-5.12-rc7 ] ~ [ linux-5.11.13 ] ~ [ linux-5.10.29 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.111 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.186 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.230 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.266 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.266 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* SPDX-License-Identifier: GPL-2.0-or-later */
  2 /*
  3  *      Definitions for the 'struct sk_buff' memory handlers.
  4  *
  5  *      Authors:
  6  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  7  *              Florian La Roche, <rzsfl@rz.uni-sb.de>
  8  */
  9 
 10 #ifndef _LINUX_SKBUFF_H
 11 #define _LINUX_SKBUFF_H
 12 
 13 #include <linux/kernel.h>
 14 #include <linux/compiler.h>
 15 #include <linux/time.h>
 16 #include <linux/bug.h>
 17 #include <linux/bvec.h>
 18 #include <linux/cache.h>
 19 #include <linux/rbtree.h>
 20 #include <linux/socket.h>
 21 #include <linux/refcount.h>
 22 
 23 #include <linux/atomic.h>
 24 #include <asm/types.h>
 25 #include <linux/spinlock.h>
 26 #include <linux/net.h>
 27 #include <linux/textsearch.h>
 28 #include <net/checksum.h>
 29 #include <linux/rcupdate.h>
 30 #include <linux/hrtimer.h>
 31 #include <linux/dma-mapping.h>
 32 #include <linux/netdev_features.h>
 33 #include <linux/sched.h>
 34 #include <linux/sched/clock.h>
 35 #include <net/flow_dissector.h>
 36 #include <linux/splice.h>
 37 #include <linux/in6.h>
 38 #include <linux/if_packet.h>
 39 #include <net/flow.h>
 40 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
 41 #include <linux/netfilter/nf_conntrack_common.h>
 42 #endif
 43 
 44 /* The interface for checksum offload between the stack and networking drivers
 45  * is as follows...
 46  *
 47  * A. IP checksum related features
 48  *
 49  * Drivers advertise checksum offload capabilities in the features of a device.
 50  * From the stack's point of view these are capabilities offered by the driver.
 51  * A driver typically only advertises features that it is capable of offloading
 52  * to its device.
 53  *
 54  * The checksum related features are:
 55  *
 56  *      NETIF_F_HW_CSUM - The driver (or its device) is able to compute one
 57  *                        IP (one's complement) checksum for any combination
 58  *                        of protocols or protocol layering. The checksum is
 59  *                        computed and set in a packet per the CHECKSUM_PARTIAL
 60  *                        interface (see below).
 61  *
 62  *      NETIF_F_IP_CSUM - Driver (device) is only able to checksum plain
 63  *                        TCP or UDP packets over IPv4. These are specifically
 64  *                        unencapsulated packets of the form IPv4|TCP or
 65  *                        IPv4|UDP where the Protocol field in the IPv4 header
 66  *                        is TCP or UDP. The IPv4 header may contain IP options.
 67  *                        This feature cannot be set in features for a device
 68  *                        with NETIF_F_HW_CSUM also set. This feature is being
 69  *                        DEPRECATED (see below).
 70  *
 71  *      NETIF_F_IPV6_CSUM - Driver (device) is only able to checksum plain
 72  *                        TCP or UDP packets over IPv6. These are specifically
 73  *                        unencapsulated packets of the form IPv6|TCP or
 74  *                        IPv6|UDP where the Next Header field in the IPv6
 75  *                        header is either TCP or UDP. IPv6 extension headers
 76  *                        are not supported with this feature. This feature
 77  *                        cannot be set in features for a device with
 78  *                        NETIF_F_HW_CSUM also set. This feature is being
 79  *                        DEPRECATED (see below).
 80  *
 81  *      NETIF_F_RXCSUM - Driver (device) performs receive checksum offload.
 82  *                       This flag is only used to disable the RX checksum
 83  *                       feature for a device. The stack will accept receive
 84  *                       checksum indication in packets received on a device
 85  *                       regardless of whether NETIF_F_RXCSUM is set.
 86  *
 87  * B. Checksumming of received packets by device. Indication of checksum
 88  *    verification is set in skb->ip_summed. Possible values are:
 89  *
 90  * CHECKSUM_NONE:
 91  *
 92  *   Device did not checksum this packet e.g. due to lack of capabilities.
 93  *   The packet contains full (though not verified) checksum in packet but
 94  *   not in skb->csum. Thus, skb->csum is undefined in this case.
 95  *
 96  * CHECKSUM_UNNECESSARY:
 97  *
 98  *   The hardware you're dealing with doesn't calculate the full checksum
 99  *   (as in CHECKSUM_COMPLETE), but it does parse headers and verify checksums
100  *   for specific protocols. For such packets it will set CHECKSUM_UNNECESSARY
101  *   if their checksums are okay. skb->csum is still undefined in this case
102  *   though. A driver or device must never modify the checksum field in the
103  *   packet even if checksum is verified.
104  *
105  *   CHECKSUM_UNNECESSARY is applicable to following protocols:
106  *     TCP: IPv6 and IPv4.
107  *     UDP: IPv4 and IPv6. A device may apply CHECKSUM_UNNECESSARY to a
108  *       zero UDP checksum for either IPv4 or IPv6, the networking stack
109  *       may perform further validation in this case.
110  *     GRE: only if the checksum is present in the header.
111  *     SCTP: indicates the CRC in SCTP header has been validated.
112  *     FCOE: indicates the CRC in FC frame has been validated.
113  *
114  *   skb->csum_level indicates the number of consecutive checksums found in
115  *   the packet minus one that have been verified as CHECKSUM_UNNECESSARY.
116  *   For instance if a device receives an IPv6->UDP->GRE->IPv4->TCP packet
117  *   and a device is able to verify the checksums for UDP (possibly zero),
118  *   GRE (checksum flag is set) and TCP, skb->csum_level would be set to
119  *   two. If the device were only able to verify the UDP checksum and not
120  *   GRE, either because it doesn't support GRE checksum or because GRE
121  *   checksum is bad, skb->csum_level would be set to zero (TCP checksum is
122  *   not considered in this case).
123  *
124  * CHECKSUM_COMPLETE:
125  *
126  *   This is the most generic way. The device supplied checksum of the _whole_
127  *   packet as seen by netif_rx() and fills in skb->csum. This means the
128  *   hardware doesn't need to parse L3/L4 headers to implement this.
129  *
130  *   Notes:
131  *   - Even if device supports only some protocols, but is able to produce
132  *     skb->csum, it MUST use CHECKSUM_COMPLETE, not CHECKSUM_UNNECESSARY.
133  *   - CHECKSUM_COMPLETE is not applicable to SCTP and FCoE protocols.
134  *
135  * CHECKSUM_PARTIAL:
136  *
137  *   A checksum is set up to be offloaded to a device as described in the
138  *   output description for CHECKSUM_PARTIAL. This may occur on a packet
139  *   received directly from another Linux OS, e.g., a virtualized Linux kernel
140  *   on the same host, or it may be set in the input path in GRO or remote
141  *   checksum offload. For the purposes of checksum verification, the checksum
142  *   referred to by skb->csum_start + skb->csum_offset and any preceding
143  *   checksums in the packet are considered verified. Any checksums in the
144  *   packet that are after the checksum being offloaded are not considered to
145  *   be verified.
146  *
147  * C. Checksumming on transmit for non-GSO. The stack requests checksum offload
148  *    in the skb->ip_summed for a packet. Values are:
149  *
150  * CHECKSUM_PARTIAL:
151  *
152  *   The driver is required to checksum the packet as seen by hard_start_xmit()
153  *   from skb->csum_start up to the end, and to record/write the checksum at
154  *   offset skb->csum_start + skb->csum_offset. A driver may verify that the
155  *   csum_start and csum_offset values are valid values given the length and
156  *   offset of the packet, but it should not attempt to validate that the
157  *   checksum refers to a legitimate transport layer checksum -- it is the
158  *   purview of the stack to validate that csum_start and csum_offset are set
159  *   correctly.
160  *
161  *   When the stack requests checksum offload for a packet, the driver MUST
162  *   ensure that the checksum is set correctly. A driver can either offload the
163  *   checksum calculation to the device, or call skb_checksum_help (in the case
164  *   that the device does not support offload for a particular checksum).
165  *
166  *   NETIF_F_IP_CSUM and NETIF_F_IPV6_CSUM are being deprecated in favor of
167  *   NETIF_F_HW_CSUM. New devices should use NETIF_F_HW_CSUM to indicate
168  *   checksum offload capability.
169  *   skb_csum_hwoffload_help() can be called to resolve CHECKSUM_PARTIAL based
170  *   on network device checksumming capabilities: if a packet does not match
171  *   them, skb_checksum_help or skb_crc32c_help (depending on the value of
172  *   csum_not_inet, see item D.) is called to resolve the checksum.
173  *
174  * CHECKSUM_NONE:
175  *
176  *   The skb was already checksummed by the protocol, or a checksum is not
177  *   required.
178  *
179  * CHECKSUM_UNNECESSARY:
180  *
181  *   This has the same meaning as CHECKSUM_NONE for checksum offload on
182  *   output.
183  *
184  * CHECKSUM_COMPLETE:
185  *   Not used in checksum output. If a driver observes a packet with this value
186  *   set in skbuff, it should treat the packet as if CHECKSUM_NONE were set.
187  *
188  * D. Non-IP checksum (CRC) offloads
189  *
190  *   NETIF_F_SCTP_CRC - This feature indicates that a device is capable of
191  *     offloading the SCTP CRC in a packet. To perform this offload the stack
192  *     will set csum_start and csum_offset accordingly, set ip_summed to
193  *     CHECKSUM_PARTIAL and set csum_not_inet to 1, to provide an indication in
194  *     the skbuff that the CHECKSUM_PARTIAL refers to CRC32c.
195  *     A driver that supports both IP checksum offload and SCTP CRC32c offload
196  *     must verify which offload is configured for a packet by testing the
197  *     value of skb->csum_not_inet; skb_crc32c_csum_help is provided to resolve
198  *     CHECKSUM_PARTIAL on skbs where csum_not_inet is set to 1.
199  *
200  *   NETIF_F_FCOE_CRC - This feature indicates that a device is capable of
201  *     offloading the FCOE CRC in a packet. To perform this offload the stack
202  *     will set ip_summed to CHECKSUM_PARTIAL and set csum_start and csum_offset
203  *     accordingly. Note that there is no indication in the skbuff that the
204  *     CHECKSUM_PARTIAL refers to an FCOE checksum, so a driver that supports
205  *     both IP checksum offload and FCOE CRC offload must verify which offload
206  *     is configured for a packet, presumably by inspecting packet headers.
207  *
208  * E. Checksumming on output with GSO.
209  *
210  * In the case of a GSO packet (skb_is_gso(skb) is true), checksum offload
211  * is implied by the SKB_GSO_* flags in gso_type. Most obviously, if the
212  * gso_type is SKB_GSO_TCPV4 or SKB_GSO_TCPV6, TCP checksum offload as
213  * part of the GSO operation is implied. If a checksum is being offloaded
214  * with GSO then ip_summed is CHECKSUM_PARTIAL, and both csum_start and
215  * csum_offset are set to refer to the outermost checksum being offloaded
216  * (two offloaded checksums are possible with UDP encapsulation).
217  */
218 
219 /* Don't change this without changing skb_csum_unnecessary! */
220 #define CHECKSUM_NONE           0
221 #define CHECKSUM_UNNECESSARY    1
222 #define CHECKSUM_COMPLETE       2
223 #define CHECKSUM_PARTIAL        3
224 
225 /* Maximum value in skb->csum_level */
226 #define SKB_MAX_CSUM_LEVEL      3
227 
228 #define SKB_DATA_ALIGN(X)       ALIGN(X, SMP_CACHE_BYTES)
229 #define SKB_WITH_OVERHEAD(X)    \
230         ((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
231 #define SKB_MAX_ORDER(X, ORDER) \
232         SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X))
233 #define SKB_MAX_HEAD(X)         (SKB_MAX_ORDER((X), 0))
234 #define SKB_MAX_ALLOC           (SKB_MAX_ORDER(0, 2))
235 
236 /* return minimum truesize of one skb containing X bytes of data */
237 #define SKB_TRUESIZE(X) ((X) +                                          \
238                          SKB_DATA_ALIGN(sizeof(struct sk_buff)) +       \
239                          SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
240 
241 struct ahash_request;
242 struct net_device;
243 struct scatterlist;
244 struct pipe_inode_info;
245 struct iov_iter;
246 struct napi_struct;
247 struct bpf_prog;
248 union bpf_attr;
249 struct skb_ext;
250 
251 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
252 struct nf_bridge_info {
253         enum {
254                 BRNF_PROTO_UNCHANGED,
255                 BRNF_PROTO_8021Q,
256                 BRNF_PROTO_PPPOE
257         } orig_proto:8;
258         u8                      pkt_otherhost:1;
259         u8                      in_prerouting:1;
260         u8                      bridged_dnat:1;
261         __u16                   frag_max_size;
262         struct net_device       *physindev;
263 
264         /* always valid & non-NULL from FORWARD on, for physdev match */
265         struct net_device       *physoutdev;
266         union {
267                 /* prerouting: detect dnat in orig/reply direction */
268                 __be32          ipv4_daddr;
269                 struct in6_addr ipv6_daddr;
270 
271                 /* after prerouting + nat detected: store original source
272                  * mac since neigh resolution overwrites it, only used while
273                  * skb is out in neigh layer.
274                  */
275                 char neigh_header[8];
276         };
277 };
278 #endif
279 
280 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
281 /* Chain in tc_skb_ext will be used to share the tc chain with
282  * ovs recirc_id. It will be set to the current chain by tc
283  * and read by ovs to recirc_id.
284  */
285 struct tc_skb_ext {
286         __u32 chain;
287         __u16 mru;
288 };
289 #endif
290 
291 struct sk_buff_head {
292         /* These two members must be first. */
293         struct sk_buff  *next;
294         struct sk_buff  *prev;
295 
296         __u32           qlen;
297         spinlock_t      lock;
298 };
299 
300 struct sk_buff;
301 
302 /* To allow 64K frame to be packed as single skb without frag_list we
303  * require 64K/PAGE_SIZE pages plus 1 additional page to allow for
304  * buffers which do not start on a page boundary.
305  *
306  * Since GRO uses frags we allocate at least 16 regardless of page
307  * size.
308  */
309 #if (65536/PAGE_SIZE + 1) < 16
310 #define MAX_SKB_FRAGS 16UL
311 #else
312 #define MAX_SKB_FRAGS (65536/PAGE_SIZE + 1)
313 #endif
314 extern int sysctl_max_skb_frags;
315 
316 /* Set skb_shinfo(skb)->gso_size to this in case you want skb_segment to
317  * segment using its current segmentation instead.
318  */
319 #define GSO_BY_FRAGS    0xFFFF
320 
321 typedef struct bio_vec skb_frag_t;
322 
323 /**
324  * skb_frag_size() - Returns the size of a skb fragment
325  * @frag: skb fragment
326  */
327 static inline unsigned int skb_frag_size(const skb_frag_t *frag)
328 {
329         return frag->bv_len;
330 }
331 
332 /**
333  * skb_frag_size_set() - Sets the size of a skb fragment
334  * @frag: skb fragment
335  * @size: size of fragment
336  */
337 static inline void skb_frag_size_set(skb_frag_t *frag, unsigned int size)
338 {
339         frag->bv_len = size;
340 }
341 
342 /**
343  * skb_frag_size_add() - Increments the size of a skb fragment by @delta
344  * @frag: skb fragment
345  * @delta: value to add
346  */
347 static inline void skb_frag_size_add(skb_frag_t *frag, int delta)
348 {
349         frag->bv_len += delta;
350 }
351 
352 /**
353  * skb_frag_size_sub() - Decrements the size of a skb fragment by @delta
354  * @frag: skb fragment
355  * @delta: value to subtract
356  */
357 static inline void skb_frag_size_sub(skb_frag_t *frag, int delta)
358 {
359         frag->bv_len -= delta;
360 }
361 
362 /**
363  * skb_frag_must_loop - Test if %p is a high memory page
364  * @p: fragment's page
365  */
366 static inline bool skb_frag_must_loop(struct page *p)
367 {
368 #if defined(CONFIG_HIGHMEM)
369         if (PageHighMem(p))
370                 return true;
371 #endif
372         return false;
373 }
374 
375 /**
376  *      skb_frag_foreach_page - loop over pages in a fragment
377  *
378  *      @f:             skb frag to operate on
379  *      @f_off:         offset from start of f->bv_page
380  *      @f_len:         length from f_off to loop over
381  *      @p:             (temp var) current page
382  *      @p_off:         (temp var) offset from start of current page,
383  *                                 non-zero only on first page.
384  *      @p_len:         (temp var) length in current page,
385  *                                 < PAGE_SIZE only on first and last page.
386  *      @copied:        (temp var) length so far, excluding current p_len.
387  *
388  *      A fragment can hold a compound page, in which case per-page
389  *      operations, notably kmap_atomic, must be called for each
390  *      regular page.
391  */
392 #define skb_frag_foreach_page(f, f_off, f_len, p, p_off, p_len, copied) \
393         for (p = skb_frag_page(f) + ((f_off) >> PAGE_SHIFT),            \
394              p_off = (f_off) & (PAGE_SIZE - 1),                         \
395              p_len = skb_frag_must_loop(p) ?                            \
396              min_t(u32, f_len, PAGE_SIZE - p_off) : f_len,              \
397              copied = 0;                                                \
398              copied < f_len;                                            \
399              copied += p_len, p++, p_off = 0,                           \
400              p_len = min_t(u32, f_len - copied, PAGE_SIZE))             \
401 
402 #define HAVE_HW_TIME_STAMP
403 
404 /**
405  * struct skb_shared_hwtstamps - hardware time stamps
406  * @hwtstamp:   hardware time stamp transformed into duration
407  *              since arbitrary point in time
408  *
409  * Software time stamps generated by ktime_get_real() are stored in
410  * skb->tstamp.
411  *
412  * hwtstamps can only be compared against other hwtstamps from
413  * the same device.
414  *
415  * This structure is attached to packets as part of the
416  * &skb_shared_info. Use skb_hwtstamps() to get a pointer.
417  */
418 struct skb_shared_hwtstamps {
419         ktime_t hwtstamp;
420 };
421 
422 /* Definitions for tx_flags in struct skb_shared_info */
423 enum {
424         /* generate hardware time stamp */
425         SKBTX_HW_TSTAMP = 1 << 0,
426 
427         /* generate software time stamp when queueing packet to NIC */
428         SKBTX_SW_TSTAMP = 1 << 1,
429 
430         /* device driver is going to provide hardware time stamp */
431         SKBTX_IN_PROGRESS = 1 << 2,
432 
433         /* device driver supports TX zero-copy buffers */
434         SKBTX_DEV_ZEROCOPY = 1 << 3,
435 
436         /* generate wifi status information (where possible) */
437         SKBTX_WIFI_STATUS = 1 << 4,
438 
439         /* This indicates at least one fragment might be overwritten
440          * (as in vmsplice(), sendfile() ...)
441          * If we need to compute a TX checksum, we'll need to copy
442          * all frags to avoid possible bad checksum
443          */
444         SKBTX_SHARED_FRAG = 1 << 5,
445 
446         /* generate software time stamp when entering packet scheduling */
447         SKBTX_SCHED_TSTAMP = 1 << 6,
448 };
449 
450 #define SKBTX_ZEROCOPY_FRAG     (SKBTX_DEV_ZEROCOPY | SKBTX_SHARED_FRAG)
451 #define SKBTX_ANY_SW_TSTAMP     (SKBTX_SW_TSTAMP    | \
452                                  SKBTX_SCHED_TSTAMP)
453 #define SKBTX_ANY_TSTAMP        (SKBTX_HW_TSTAMP | SKBTX_ANY_SW_TSTAMP)
454 
455 /*
456  * The callback notifies userspace to release buffers when skb DMA is done in
457  * lower device, the skb last reference should be 0 when calling this.
458  * The zerocopy_success argument is true if zero copy transmit occurred,
459  * false on data copy or out of memory error caused by data copy attempt.
460  * The ctx field is used to track device context.
461  * The desc field is used to track userspace buffer index.
462  */
463 struct ubuf_info {
464         void (*callback)(struct ubuf_info *, bool zerocopy_success);
465         union {
466                 struct {
467                         unsigned long desc;
468                         void *ctx;
469                 };
470                 struct {
471                         u32 id;
472                         u16 len;
473                         u16 zerocopy:1;
474                         u32 bytelen;
475                 };
476         };
477         refcount_t refcnt;
478 
479         struct mmpin {
480                 struct user_struct *user;
481                 unsigned int num_pg;
482         } mmp;
483 };
484 
485 #define skb_uarg(SKB)   ((struct ubuf_info *)(skb_shinfo(SKB)->destructor_arg))
486 
487 int mm_account_pinned_pages(struct mmpin *mmp, size_t size);
488 void mm_unaccount_pinned_pages(struct mmpin *mmp);
489 
490 struct ubuf_info *sock_zerocopy_alloc(struct sock *sk, size_t size);
491 struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size,
492                                         struct ubuf_info *uarg);
493 
494 static inline void sock_zerocopy_get(struct ubuf_info *uarg)
495 {
496         refcount_inc(&uarg->refcnt);
497 }
498 
499 void sock_zerocopy_put(struct ubuf_info *uarg);
500 void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref);
501 
502 void sock_zerocopy_callback(struct ubuf_info *uarg, bool success);
503 
504 int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len);
505 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
506                              struct msghdr *msg, int len,
507                              struct ubuf_info *uarg);
508 
509 /* This data is invariant across clones and lives at
510  * the end of the header data, ie. at skb->end.
511  */
512 struct skb_shared_info {
513         __u8            __unused;
514         __u8            meta_len;
515         __u8            nr_frags;
516         __u8            tx_flags;
517         unsigned short  gso_size;
518         /* Warning: this field is not always filled in (UFO)! */
519         unsigned short  gso_segs;
520         struct sk_buff  *frag_list;
521         struct skb_shared_hwtstamps hwtstamps;
522         unsigned int    gso_type;
523         u32             tskey;
524 
525         /*
526          * Warning : all fields before dataref are cleared in __alloc_skb()
527          */
528         atomic_t        dataref;
529 
530         /* Intermediate layers must ensure that destructor_arg
531          * remains valid until skb destructor */
532         void *          destructor_arg;
533 
534         /* must be last field, see pskb_expand_head() */
535         skb_frag_t      frags[MAX_SKB_FRAGS];
536 };
537 
538 /* We divide dataref into two halves.  The higher 16 bits hold references
539  * to the payload part of skb->data.  The lower 16 bits hold references to
540  * the entire skb->data.  A clone of a headerless skb holds the length of
541  * the header in skb->hdr_len.
542  *
543  * All users must obey the rule that the skb->data reference count must be
544  * greater than or equal to the payload reference count.
545  *
546  * Holding a reference to the payload part means that the user does not
547  * care about modifications to the header part of skb->data.
548  */
549 #define SKB_DATAREF_SHIFT 16
550 #define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1)
551 
552 
553 enum {
554         SKB_FCLONE_UNAVAILABLE, /* skb has no fclone (from head_cache) */
555         SKB_FCLONE_ORIG,        /* orig skb (from fclone_cache) */
556         SKB_FCLONE_CLONE,       /* companion fclone skb (from fclone_cache) */
557 };
558 
559 enum {
560         SKB_GSO_TCPV4 = 1 << 0,
561 
562         /* This indicates the skb is from an untrusted source. */
563         SKB_GSO_DODGY = 1 << 1,
564 
565         /* This indicates the tcp segment has CWR set. */
566         SKB_GSO_TCP_ECN = 1 << 2,
567 
568         SKB_GSO_TCP_FIXEDID = 1 << 3,
569 
570         SKB_GSO_TCPV6 = 1 << 4,
571 
572         SKB_GSO_FCOE = 1 << 5,
573 
574         SKB_GSO_GRE = 1 << 6,
575 
576         SKB_GSO_GRE_CSUM = 1 << 7,
577 
578         SKB_GSO_IPXIP4 = 1 << 8,
579 
580         SKB_GSO_IPXIP6 = 1 << 9,
581 
582         SKB_GSO_UDP_TUNNEL = 1 << 10,
583 
584         SKB_GSO_UDP_TUNNEL_CSUM = 1 << 11,
585 
586         SKB_GSO_PARTIAL = 1 << 12,
587 
588         SKB_GSO_TUNNEL_REMCSUM = 1 << 13,
589 
590         SKB_GSO_SCTP = 1 << 14,
591 
592         SKB_GSO_ESP = 1 << 15,
593 
594         SKB_GSO_UDP = 1 << 16,
595 
596         SKB_GSO_UDP_L4 = 1 << 17,
597 
598         SKB_GSO_FRAGLIST = 1 << 18,
599 };
600 
601 #if BITS_PER_LONG > 32
602 #define NET_SKBUFF_DATA_USES_OFFSET 1
603 #endif
604 
605 #ifdef NET_SKBUFF_DATA_USES_OFFSET
606 typedef unsigned int sk_buff_data_t;
607 #else
608 typedef unsigned char *sk_buff_data_t;
609 #endif
610 
611 /**
612  *      struct sk_buff - socket buffer
613  *      @next: Next buffer in list
614  *      @prev: Previous buffer in list
615  *      @tstamp: Time we arrived/left
616  *      @skb_mstamp_ns: (aka @tstamp) earliest departure time; start point
617  *              for retransmit timer
618  *      @rbnode: RB tree node, alternative to next/prev for netem/tcp
619  *      @list: queue head
620  *      @sk: Socket we are owned by
621  *      @ip_defrag_offset: (aka @sk) alternate use of @sk, used in
622  *              fragmentation management
623  *      @dev: Device we arrived on/are leaving by
624  *      @dev_scratch: (aka @dev) alternate use of @dev when @dev would be %NULL
625  *      @cb: Control buffer. Free for use by every layer. Put private vars here
626  *      @_skb_refdst: destination entry (with norefcount bit)
627  *      @sp: the security path, used for xfrm
628  *      @len: Length of actual data
629  *      @data_len: Data length
630  *      @mac_len: Length of link layer header
631  *      @hdr_len: writable header length of cloned skb
632  *      @csum: Checksum (must include start/offset pair)
633  *      @csum_start: Offset from skb->head where checksumming should start
634  *      @csum_offset: Offset from csum_start where checksum should be stored
635  *      @priority: Packet queueing priority
636  *      @ignore_df: allow local fragmentation
637  *      @cloned: Head may be cloned (check refcnt to be sure)
638  *      @ip_summed: Driver fed us an IP checksum
639  *      @nohdr: Payload reference only, must not modify header
640  *      @pkt_type: Packet class
641  *      @fclone: skbuff clone status
642  *      @ipvs_property: skbuff is owned by ipvs
643  *      @inner_protocol_type: whether the inner protocol is
644  *              ENCAP_TYPE_ETHER or ENCAP_TYPE_IPPROTO
645  *      @remcsum_offload: remote checksum offload is enabled
646  *      @offload_fwd_mark: Packet was L2-forwarded in hardware
647  *      @offload_l3_fwd_mark: Packet was L3-forwarded in hardware
648  *      @tc_skip_classify: do not classify packet. set by IFB device
649  *      @tc_at_ingress: used within tc_classify to distinguish in/egress
650  *      @redirected: packet was redirected by packet classifier
651  *      @from_ingress: packet was redirected from the ingress path
652  *      @peeked: this packet has been seen already, so stats have been
653  *              done for it, don't do them again
654  *      @nf_trace: netfilter packet trace flag
655  *      @protocol: Packet protocol from driver
656  *      @destructor: Destruct function
657  *      @tcp_tsorted_anchor: list structure for TCP (tp->tsorted_sent_queue)
658  *      @_nfct: Associated connection, if any (with nfctinfo bits)
659  *      @nf_bridge: Saved data about a bridged frame - see br_netfilter.c
660  *      @skb_iif: ifindex of device we arrived on
661  *      @tc_index: Traffic control index
662  *      @hash: the packet hash
663  *      @queue_mapping: Queue mapping for multiqueue devices
664  *      @head_frag: skb was allocated from page fragments,
665  *              not allocated by kmalloc() or vmalloc().
666  *      @pfmemalloc: skbuff was allocated from PFMEMALLOC reserves
667  *      @active_extensions: active extensions (skb_ext_id types)
668  *      @ndisc_nodetype: router type (from link layer)
669  *      @ooo_okay: allow the mapping of a socket to a queue to be changed
670  *      @l4_hash: indicate hash is a canonical 4-tuple hash over transport
671  *              ports.
672  *      @sw_hash: indicates hash was computed in software stack
673  *      @wifi_acked_valid: wifi_acked was set
674  *      @wifi_acked: whether frame was acked on wifi or not
675  *      @no_fcs:  Request NIC to treat last 4 bytes as Ethernet FCS
676  *      @encapsulation: indicates the inner headers in the skbuff are valid
677  *      @encap_hdr_csum: software checksum is needed
678  *      @csum_valid: checksum is already valid
679  *      @csum_not_inet: use CRC32c to resolve CHECKSUM_PARTIAL
680  *      @csum_complete_sw: checksum was completed by software
681  *      @csum_level: indicates the number of consecutive checksums found in
682  *              the packet minus one that have been verified as
683  *              CHECKSUM_UNNECESSARY (max 3)
684  *      @dst_pending_confirm: need to confirm neighbour
685  *      @decrypted: Decrypted SKB
686  *      @napi_id: id of the NAPI struct this skb came from
687  *      @sender_cpu: (aka @napi_id) source CPU in XPS
688  *      @secmark: security marking
689  *      @mark: Generic packet mark
690  *      @reserved_tailroom: (aka @mark) number of bytes of free space available
691  *              at the tail of an sk_buff
692  *      @vlan_present: VLAN tag is present
693  *      @vlan_proto: vlan encapsulation protocol
694  *      @vlan_tci: vlan tag control information
695  *      @inner_protocol: Protocol (encapsulation)
696  *      @inner_ipproto: (aka @inner_protocol) stores ipproto when
697  *              skb->inner_protocol_type == ENCAP_TYPE_IPPROTO;
698  *      @inner_transport_header: Inner transport layer header (encapsulation)
699  *      @inner_network_header: Network layer header (encapsulation)
700  *      @inner_mac_header: Link layer header (encapsulation)
701  *      @transport_header: Transport layer header
702  *      @network_header: Network layer header
703  *      @mac_header: Link layer header
704  *      @tail: Tail pointer
705  *      @end: End pointer
706  *      @head: Head of buffer
707  *      @data: Data head pointer
708  *      @truesize: Buffer size
709  *      @users: User count - see {datagram,tcp}.c
710  *      @extensions: allocated extensions, valid if active_extensions is nonzero
711  */
712 
713 struct sk_buff {
714         union {
715                 struct {
716                         /* These two members must be first. */
717                         struct sk_buff          *next;
718                         struct sk_buff          *prev;
719 
720                         union {
721                                 struct net_device       *dev;
722                                 /* Some protocols might use this space to store information,
723                                  * while device pointer would be NULL.
724                                  * UDP receive path is one user.
725                                  */
726                                 unsigned long           dev_scratch;
727                         };
728                 };
729                 struct rb_node          rbnode; /* used in netem, ip4 defrag, and tcp stack */
730                 struct list_head        list;
731         };
732 
733         union {
734                 struct sock             *sk;
735                 int                     ip_defrag_offset;
736         };
737 
738         union {
739                 ktime_t         tstamp;
740                 u64             skb_mstamp_ns; /* earliest departure time */
741         };
742         /*
743          * This is the control buffer. It is free to use for every
744          * layer. Please put your private variables there. If you
745          * want to keep them across layers you have to do a skb_clone()
746          * first. This is owned by whoever has the skb queued ATM.
747          */
748         char                    cb[48] __aligned(8);
749 
750         union {
751                 struct {
752                         unsigned long   _skb_refdst;
753                         void            (*destructor)(struct sk_buff *skb);
754                 };
755                 struct list_head        tcp_tsorted_anchor;
756         };
757 
758 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
759         unsigned long            _nfct;
760 #endif
761         unsigned int            len,
762                                 data_len;
763         __u16                   mac_len,
764                                 hdr_len;
765 
766         /* Following fields are _not_ copied in __copy_skb_header()
767          * Note that queue_mapping is here mostly to fill a hole.
768          */
769         __u16                   queue_mapping;
770 
771 /* if you move cloned around you also must adapt those constants */
772 #ifdef __BIG_ENDIAN_BITFIELD
773 #define CLONED_MASK     (1 << 7)
774 #else
775 #define CLONED_MASK     1
776 #endif
777 #define CLONED_OFFSET()         offsetof(struct sk_buff, __cloned_offset)
778 
779         /* private: */
780         __u8                    __cloned_offset[0];
781         /* public: */
782         __u8                    cloned:1,
783                                 nohdr:1,
784                                 fclone:2,
785                                 peeked:1,
786                                 head_frag:1,
787                                 pfmemalloc:1;
788 #ifdef CONFIG_SKB_EXTENSIONS
789         __u8                    active_extensions;
790 #endif
791         /* fields enclosed in headers_start/headers_end are copied
792          * using a single memcpy() in __copy_skb_header()
793          */
794         /* private: */
795         __u32                   headers_start[0];
796         /* public: */
797 
798 /* if you move pkt_type around you also must adapt those constants */
799 #ifdef __BIG_ENDIAN_BITFIELD
800 #define PKT_TYPE_MAX    (7 << 5)
801 #else
802 #define PKT_TYPE_MAX    7
803 #endif
804 #define PKT_TYPE_OFFSET()       offsetof(struct sk_buff, __pkt_type_offset)
805 
806         /* private: */
807         __u8                    __pkt_type_offset[0];
808         /* public: */
809         __u8                    pkt_type:3;
810         __u8                    ignore_df:1;
811         __u8                    nf_trace:1;
812         __u8                    ip_summed:2;
813         __u8                    ooo_okay:1;
814 
815         __u8                    l4_hash:1;
816         __u8                    sw_hash:1;
817         __u8                    wifi_acked_valid:1;
818         __u8                    wifi_acked:1;
819         __u8                    no_fcs:1;
820         /* Indicates the inner headers are valid in the skbuff. */
821         __u8                    encapsulation:1;
822         __u8                    encap_hdr_csum:1;
823         __u8                    csum_valid:1;
824 
825 #ifdef __BIG_ENDIAN_BITFIELD
826 #define PKT_VLAN_PRESENT_BIT    7
827 #else
828 #define PKT_VLAN_PRESENT_BIT    0
829 #endif
830 #define PKT_VLAN_PRESENT_OFFSET()       offsetof(struct sk_buff, __pkt_vlan_present_offset)
831         /* private: */
832         __u8                    __pkt_vlan_present_offset[0];
833         /* public: */
834         __u8                    vlan_present:1;
835         __u8                    csum_complete_sw:1;
836         __u8                    csum_level:2;
837         __u8                    csum_not_inet:1;
838         __u8                    dst_pending_confirm:1;
839 #ifdef CONFIG_IPV6_NDISC_NODETYPE
840         __u8                    ndisc_nodetype:2;
841 #endif
842 
843         __u8                    ipvs_property:1;
844         __u8                    inner_protocol_type:1;
845         __u8                    remcsum_offload:1;
846 #ifdef CONFIG_NET_SWITCHDEV
847         __u8                    offload_fwd_mark:1;
848         __u8                    offload_l3_fwd_mark:1;
849 #endif
850 #ifdef CONFIG_NET_CLS_ACT
851         __u8                    tc_skip_classify:1;
852         __u8                    tc_at_ingress:1;
853 #endif
854 #ifdef CONFIG_NET_REDIRECT
855         __u8                    redirected:1;
856         __u8                    from_ingress:1;
857 #endif
858 #ifdef CONFIG_TLS_DEVICE
859         __u8                    decrypted:1;
860 #endif
861 
862 #ifdef CONFIG_NET_SCHED
863         __u16                   tc_index;       /* traffic control index */
864 #endif
865 
866         union {
867                 __wsum          csum;
868                 struct {
869                         __u16   csum_start;
870                         __u16   csum_offset;
871                 };
872         };
873         __u32                   priority;
874         int                     skb_iif;
875         __u32                   hash;
876         __be16                  vlan_proto;
877         __u16                   vlan_tci;
878 #if defined(CONFIG_NET_RX_BUSY_POLL) || defined(CONFIG_XPS)
879         union {
880                 unsigned int    napi_id;
881                 unsigned int    sender_cpu;
882         };
883 #endif
884 #ifdef CONFIG_NETWORK_SECMARK
885         __u32           secmark;
886 #endif
887 
888         union {
889                 __u32           mark;
890                 __u32           reserved_tailroom;
891         };
892 
893         union {
894                 __be16          inner_protocol;
895                 __u8            inner_ipproto;
896         };
897 
898         __u16                   inner_transport_header;
899         __u16                   inner_network_header;
900         __u16                   inner_mac_header;
901 
902         __be16                  protocol;
903         __u16                   transport_header;
904         __u16                   network_header;
905         __u16                   mac_header;
906 
907         /* private: */
908         __u32                   headers_end[0];
909         /* public: */
910 
911         /* These elements must be at the end, see alloc_skb() for details.  */
912         sk_buff_data_t          tail;
913         sk_buff_data_t          end;
914         unsigned char           *head,
915                                 *data;
916         unsigned int            truesize;
917         refcount_t              users;
918 
919 #ifdef CONFIG_SKB_EXTENSIONS
920         /* only useable after checking ->active_extensions != 0 */
921         struct skb_ext          *extensions;
922 #endif
923 };
924 
925 #ifdef __KERNEL__
926 /*
927  *      Handling routines are only of interest to the kernel
928  */
929 
930 #define SKB_ALLOC_FCLONE        0x01
931 #define SKB_ALLOC_RX            0x02
932 #define SKB_ALLOC_NAPI          0x04
933 
934 /**
935  * skb_pfmemalloc - Test if the skb was allocated from PFMEMALLOC reserves
936  * @skb: buffer
937  */
938 static inline bool skb_pfmemalloc(const struct sk_buff *skb)
939 {
940         return unlikely(skb->pfmemalloc);
941 }
942 
943 /*
944  * skb might have a dst pointer attached, refcounted or not.
945  * _skb_refdst low order bit is set if refcount was _not_ taken
946  */
947 #define SKB_DST_NOREF   1UL
948 #define SKB_DST_PTRMASK ~(SKB_DST_NOREF)
949 
950 /**
951  * skb_dst - returns skb dst_entry
952  * @skb: buffer
953  *
954  * Returns skb dst_entry, regardless of reference taken or not.
955  */
956 static inline struct dst_entry *skb_dst(const struct sk_buff *skb)
957 {
958         /* If refdst was not refcounted, check we still are in a
959          * rcu_read_lock section
960          */
961         WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) &&
962                 !rcu_read_lock_held() &&
963                 !rcu_read_lock_bh_held());
964         return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK);
965 }
966 
967 /**
968  * skb_dst_set - sets skb dst
969  * @skb: buffer
970  * @dst: dst entry
971  *
972  * Sets skb dst, assuming a reference was taken on dst and should
973  * be released by skb_dst_drop()
974  */
975 static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst)
976 {
977         skb->_skb_refdst = (unsigned long)dst;
978 }
979 
980 /**
981  * skb_dst_set_noref - sets skb dst, hopefully, without taking reference
982  * @skb: buffer
983  * @dst: dst entry
984  *
985  * Sets skb dst, assuming a reference was not taken on dst.
986  * If dst entry is cached, we do not take reference and dst_release
987  * will be avoided by refdst_drop. If dst entry is not cached, we take
988  * reference, so that last dst_release can destroy the dst immediately.
989  */
990 static inline void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst)
991 {
992         WARN_ON(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
993         skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF;
994 }
995 
996 /**
997  * skb_dst_is_noref - Test if skb dst isn't refcounted
998  * @skb: buffer
999  */
1000 static inline bool skb_dst_is_noref(const struct sk_buff *skb)
1001 {
1002         return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb);
1003 }
1004 
1005 /**
1006  * skb_rtable - Returns the skb &rtable
1007  * @skb: buffer
1008  */
1009 static inline struct rtable *skb_rtable(const struct sk_buff *skb)
1010 {
1011         return (struct rtable *)skb_dst(skb);
1012 }
1013 
1014 /* For mangling skb->pkt_type from user space side from applications
1015  * such as nft, tc, etc, we only allow a conservative subset of
1016  * possible pkt_types to be set.
1017 */
1018 static inline bool skb_pkt_type_ok(u32 ptype)
1019 {
1020         return ptype <= PACKET_OTHERHOST;
1021 }
1022 
1023 /**
1024  * skb_napi_id - Returns the skb's NAPI id
1025  * @skb: buffer
1026  */
1027 static inline unsigned int skb_napi_id(const struct sk_buff *skb)
1028 {
1029 #ifdef CONFIG_NET_RX_BUSY_POLL
1030         return skb->napi_id;
1031 #else
1032         return 0;
1033 #endif
1034 }
1035 
1036 /**
1037  * skb_unref - decrement the skb's reference count
1038  * @skb: buffer
1039  *
1040  * Returns true if we can free the skb.
1041  */
1042 static inline bool skb_unref(struct sk_buff *skb)
1043 {
1044         if (unlikely(!skb))
1045                 return false;
1046         if (likely(refcount_read(&skb->users) == 1))
1047                 smp_rmb();
1048         else if (likely(!refcount_dec_and_test(&skb->users)))
1049                 return false;
1050 
1051         return true;
1052 }
1053 
1054 void skb_release_head_state(struct sk_buff *skb);
1055 void kfree_skb(struct sk_buff *skb);
1056 void kfree_skb_list(struct sk_buff *segs);
1057 void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt);
1058 void skb_tx_error(struct sk_buff *skb);
1059 
1060 #ifdef CONFIG_TRACEPOINTS
1061 void consume_skb(struct sk_buff *skb);
1062 #else
1063 static inline void consume_skb(struct sk_buff *skb)
1064 {
1065         return kfree_skb(skb);
1066 }
1067 #endif
1068 
1069 void __consume_stateless_skb(struct sk_buff *skb);
1070 void  __kfree_skb(struct sk_buff *skb);
1071 extern struct kmem_cache *skbuff_head_cache;
1072 
1073 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
1074 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
1075                       bool *fragstolen, int *delta_truesize);
1076 
1077 struct sk_buff *__alloc_skb(unsigned int size, gfp_t priority, int flags,
1078                             int node);
1079 struct sk_buff *__build_skb(void *data, unsigned int frag_size);
1080 struct sk_buff *build_skb(void *data, unsigned int frag_size);
1081 struct sk_buff *build_skb_around(struct sk_buff *skb,
1082                                  void *data, unsigned int frag_size);
1083 
1084 /**
1085  * alloc_skb - allocate a network buffer
1086  * @size: size to allocate
1087  * @priority: allocation mask
1088  *
1089  * This function is a convenient wrapper around __alloc_skb().
1090  */
1091 static inline struct sk_buff *alloc_skb(unsigned int size,
1092                                         gfp_t priority)
1093 {
1094         return __alloc_skb(size, priority, 0, NUMA_NO_NODE);
1095 }
1096 
1097 struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
1098                                      unsigned long data_len,
1099                                      int max_page_order,
1100                                      int *errcode,
1101                                      gfp_t gfp_mask);
1102 struct sk_buff *alloc_skb_for_msg(struct sk_buff *first);
1103 
1104 /* Layout of fast clones : [skb1][skb2][fclone_ref] */
1105 struct sk_buff_fclones {
1106         struct sk_buff  skb1;
1107 
1108         struct sk_buff  skb2;
1109 
1110         refcount_t      fclone_ref;
1111 };
1112 
1113 /**
1114  *      skb_fclone_busy - check if fclone is busy
1115  *      @sk: socket
1116  *      @skb: buffer
1117  *
1118  * Returns true if skb is a fast clone, and its clone is not freed.
1119  * Some drivers call skb_orphan() in their ndo_start_xmit(),
1120  * so we also check that this didnt happen.
1121  */
1122 static inline bool skb_fclone_busy(const struct sock *sk,
1123                                    const struct sk_buff *skb)
1124 {
1125         const struct sk_buff_fclones *fclones;
1126 
1127         fclones = container_of(skb, struct sk_buff_fclones, skb1);
1128 
1129         return skb->fclone == SKB_FCLONE_ORIG &&
1130                refcount_read(&fclones->fclone_ref) > 1 &&
1131                fclones->skb2.sk == sk;
1132 }
1133 
1134 /**
1135  * alloc_skb_fclone - allocate a network buffer from fclone cache
1136  * @size: size to allocate
1137  * @priority: allocation mask
1138  *
1139  * This function is a convenient wrapper around __alloc_skb().
1140  */
1141 static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
1142                                                gfp_t priority)
1143 {
1144         return __alloc_skb(size, priority, SKB_ALLOC_FCLONE, NUMA_NO_NODE);
1145 }
1146 
1147 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src);
1148 void skb_headers_offset_update(struct sk_buff *skb, int off);
1149 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask);
1150 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t priority);
1151 void skb_copy_header(struct sk_buff *new, const struct sk_buff *old);
1152 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t priority);
1153 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
1154                                    gfp_t gfp_mask, bool fclone);
1155 static inline struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom,
1156                                           gfp_t gfp_mask)
1157 {
1158         return __pskb_copy_fclone(skb, headroom, gfp_mask, false);
1159 }
1160 
1161 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, gfp_t gfp_mask);
1162 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb,
1163                                      unsigned int headroom);
1164 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, int newheadroom,
1165                                 int newtailroom, gfp_t priority);
1166 int __must_check skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
1167                                      int offset, int len);
1168 int __must_check skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg,
1169                               int offset, int len);
1170 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer);
1171 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error);
1172 
1173 /**
1174  *      skb_pad                 -       zero pad the tail of an skb
1175  *      @skb: buffer to pad
1176  *      @pad: space to pad
1177  *
1178  *      Ensure that a buffer is followed by a padding area that is zero
1179  *      filled. Used by network drivers which may DMA or transfer data
1180  *      beyond the buffer end onto the wire.
1181  *
1182  *      May return error in out of memory cases. The skb is freed on error.
1183  */
1184 static inline int skb_pad(struct sk_buff *skb, int pad)
1185 {
1186         return __skb_pad(skb, pad, true);
1187 }
1188 #define dev_kfree_skb(a)        consume_skb(a)
1189 
1190 int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
1191                          int offset, size_t size);
1192 
1193 struct skb_seq_state {
1194         __u32           lower_offset;
1195         __u32           upper_offset;
1196         __u32           frag_idx;
1197         __u32           stepped_offset;
1198         struct sk_buff  *root_skb;
1199         struct sk_buff  *cur_skb;
1200         __u8            *frag_data;
1201 };
1202 
1203 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
1204                           unsigned int to, struct skb_seq_state *st);
1205 unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
1206                           struct skb_seq_state *st);
1207 void skb_abort_seq_read(struct skb_seq_state *st);
1208 
1209 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
1210                            unsigned int to, struct ts_config *config);
1211 
1212 /*
1213  * Packet hash types specify the type of hash in skb_set_hash.
1214  *
1215  * Hash types refer to the protocol layer addresses which are used to
1216  * construct a packet's hash. The hashes are used to differentiate or identify
1217  * flows of the protocol layer for the hash type. Hash types are either
1218  * layer-2 (L2), layer-3 (L3), or layer-4 (L4).
1219  *
1220  * Properties of hashes:
1221  *
1222  * 1) Two packets in different flows have different hash values
1223  * 2) Two packets in the same flow should have the same hash value
1224  *
1225  * A hash at a higher layer is considered to be more specific. A driver should
1226  * set the most specific hash possible.
1227  *
1228  * A driver cannot indicate a more specific hash than the layer at which a hash
1229  * was computed. For instance an L3 hash cannot be set as an L4 hash.
1230  *
1231  * A driver may indicate a hash level which is less specific than the
1232  * actual layer the hash was computed on. For instance, a hash computed
1233  * at L4 may be considered an L3 hash. This should only be done if the
1234  * driver can't unambiguously determine that the HW computed the hash at
1235  * the higher layer. Note that the "should" in the second property above
1236  * permits this.
1237  */
1238 enum pkt_hash_types {
1239         PKT_HASH_TYPE_NONE,     /* Undefined type */
1240         PKT_HASH_TYPE_L2,       /* Input: src_MAC, dest_MAC */
1241         PKT_HASH_TYPE_L3,       /* Input: src_IP, dst_IP */
1242         PKT_HASH_TYPE_L4,       /* Input: src_IP, dst_IP, src_port, dst_port */
1243 };
1244 
1245 static inline void skb_clear_hash(struct sk_buff *skb)
1246 {
1247         skb->hash = 0;
1248         skb->sw_hash = 0;
1249         skb->l4_hash = 0;
1250 }
1251 
1252 static inline void skb_clear_hash_if_not_l4(struct sk_buff *skb)
1253 {
1254         if (!skb->l4_hash)
1255                 skb_clear_hash(skb);
1256 }
1257 
1258 static inline void
1259 __skb_set_hash(struct sk_buff *skb, __u32 hash, bool is_sw, bool is_l4)
1260 {
1261         skb->l4_hash = is_l4;
1262         skb->sw_hash = is_sw;
1263         skb->hash = hash;
1264 }
1265 
1266 static inline void
1267 skb_set_hash(struct sk_buff *skb, __u32 hash, enum pkt_hash_types type)
1268 {
1269         /* Used by drivers to set hash from HW */
1270         __skb_set_hash(skb, hash, false, type == PKT_HASH_TYPE_L4);
1271 }
1272 
1273 static inline void
1274 __skb_set_sw_hash(struct sk_buff *skb, __u32 hash, bool is_l4)
1275 {
1276         __skb_set_hash(skb, hash, true, is_l4);
1277 }
1278 
1279 void __skb_get_hash(struct sk_buff *skb);
1280 u32 __skb_get_hash_symmetric(const struct sk_buff *skb);
1281 u32 skb_get_poff(const struct sk_buff *skb);
1282 u32 __skb_get_poff(const struct sk_buff *skb, void *data,
1283                    const struct flow_keys_basic *keys, int hlen);
1284 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
1285                             void *data, int hlen_proto);
1286 
1287 static inline __be32 skb_flow_get_ports(const struct sk_buff *skb,
1288                                         int thoff, u8 ip_proto)
1289 {
1290         return __skb_flow_get_ports(skb, thoff, ip_proto, NULL, 0);
1291 }
1292 
1293 void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
1294                              const struct flow_dissector_key *key,
1295                              unsigned int key_count);
1296 
1297 struct bpf_flow_dissector;
1298 bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
1299                       __be16 proto, int nhoff, int hlen, unsigned int flags);
1300 
1301 bool __skb_flow_dissect(const struct net *net,
1302                         const struct sk_buff *skb,
1303                         struct flow_dissector *flow_dissector,
1304                         void *target_container,
1305                         void *data, __be16 proto, int nhoff, int hlen,
1306                         unsigned int flags);
1307 
1308 static inline bool skb_flow_dissect(const struct sk_buff *skb,
1309                                     struct flow_dissector *flow_dissector,
1310                                     void *target_container, unsigned int flags)
1311 {
1312         return __skb_flow_dissect(NULL, skb, flow_dissector,
1313                                   target_container, NULL, 0, 0, 0, flags);
1314 }
1315 
1316 static inline bool skb_flow_dissect_flow_keys(const struct sk_buff *skb,
1317                                               struct flow_keys *flow,
1318                                               unsigned int flags)
1319 {
1320         memset(flow, 0, sizeof(*flow));
1321         return __skb_flow_dissect(NULL, skb, &flow_keys_dissector,
1322                                   flow, NULL, 0, 0, 0, flags);
1323 }
1324 
1325 static inline bool
1326 skb_flow_dissect_flow_keys_basic(const struct net *net,
1327                                  const struct sk_buff *skb,
1328                                  struct flow_keys_basic *flow, void *data,
1329                                  __be16 proto, int nhoff, int hlen,
1330                                  unsigned int flags)
1331 {
1332         memset(flow, 0, sizeof(*flow));
1333         return __skb_flow_dissect(net, skb, &flow_keys_basic_dissector, flow,
1334                                   data, proto, nhoff, hlen, flags);
1335 }
1336 
1337 void skb_flow_dissect_meta(const struct sk_buff *skb,
1338                            struct flow_dissector *flow_dissector,
1339                            void *target_container);
1340 
1341 /* Gets a skb connection tracking info, ctinfo map should be a
1342  * map of mapsize to translate enum ip_conntrack_info states
1343  * to user states.
1344  */
1345 void
1346 skb_flow_dissect_ct(const struct sk_buff *skb,
1347                     struct flow_dissector *flow_dissector,
1348                     void *target_container,
1349                     u16 *ctinfo_map,
1350                     size_t mapsize);
1351 void
1352 skb_flow_dissect_tunnel_info(const struct sk_buff *skb,
1353                              struct flow_dissector *flow_dissector,
1354                              void *target_container);
1355 
1356 void skb_flow_dissect_hash(const struct sk_buff *skb,
1357                            struct flow_dissector *flow_dissector,
1358                            void *target_container);
1359 
1360 static inline __u32 skb_get_hash(struct sk_buff *skb)
1361 {
1362         if (!skb->l4_hash && !skb->sw_hash)
1363                 __skb_get_hash(skb);
1364 
1365         return skb->hash;
1366 }
1367 
1368 static inline __u32 skb_get_hash_flowi6(struct sk_buff *skb, const struct flowi6 *fl6)
1369 {
1370         if (!skb->l4_hash && !skb->sw_hash) {
1371                 struct flow_keys keys;
1372                 __u32 hash = __get_hash_from_flowi6(fl6, &keys);
1373 
1374                 __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys));
1375         }
1376 
1377         return skb->hash;
1378 }
1379 
1380 __u32 skb_get_hash_perturb(const struct sk_buff *skb,
1381                            const siphash_key_t *perturb);
1382 
1383 static inline __u32 skb_get_hash_raw(const struct sk_buff *skb)
1384 {
1385         return skb->hash;
1386 }
1387 
1388 static inline void skb_copy_hash(struct sk_buff *to, const struct sk_buff *from)
1389 {
1390         to->hash = from->hash;
1391         to->sw_hash = from->sw_hash;
1392         to->l4_hash = from->l4_hash;
1393 };
1394 
1395 static inline void skb_copy_decrypted(struct sk_buff *to,
1396                                       const struct sk_buff *from)
1397 {
1398 #ifdef CONFIG_TLS_DEVICE
1399         to->decrypted = from->decrypted;
1400 #endif
1401 }
1402 
1403 #ifdef NET_SKBUFF_DATA_USES_OFFSET
1404 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
1405 {
1406         return skb->head + skb->end;
1407 }
1408 
1409 static inline unsigned int skb_end_offset(const struct sk_buff *skb)
1410 {
1411         return skb->end;
1412 }
1413 #else
1414 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
1415 {
1416         return skb->end;
1417 }
1418 
1419 static inline unsigned int skb_end_offset(const struct sk_buff *skb)
1420 {
1421         return skb->end - skb->head;
1422 }
1423 #endif
1424 
1425 /* Internal */
1426 #define skb_shinfo(SKB) ((struct skb_shared_info *)(skb_end_pointer(SKB)))
1427 
1428 static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb)
1429 {
1430         return &skb_shinfo(skb)->hwtstamps;
1431 }
1432 
1433 static inline struct ubuf_info *skb_zcopy(struct sk_buff *skb)
1434 {
1435         bool is_zcopy = skb && skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY;
1436 
1437         return is_zcopy ? skb_uarg(skb) : NULL;
1438 }
1439 
1440 static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg,
1441                                  bool *have_ref)
1442 {
1443         if (skb && uarg && !skb_zcopy(skb)) {
1444                 if (unlikely(have_ref && *have_ref))
1445                         *have_ref = false;
1446                 else
1447                         sock_zerocopy_get(uarg);
1448                 skb_shinfo(skb)->destructor_arg = uarg;
1449                 skb_shinfo(skb)->tx_flags |= SKBTX_ZEROCOPY_FRAG;
1450         }
1451 }
1452 
1453 static inline void skb_zcopy_set_nouarg(struct sk_buff *skb, void *val)
1454 {
1455         skb_shinfo(skb)->destructor_arg = (void *)((uintptr_t) val | 0x1UL);
1456         skb_shinfo(skb)->tx_flags |= SKBTX_ZEROCOPY_FRAG;
1457 }
1458 
1459 static inline bool skb_zcopy_is_nouarg(struct sk_buff *skb)
1460 {
1461         return (uintptr_t) skb_shinfo(skb)->destructor_arg & 0x1UL;
1462 }
1463 
1464 static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb)
1465 {
1466         return (void *)((uintptr_t) skb_shinfo(skb)->destructor_arg & ~0x1UL);
1467 }
1468 
1469 /* Release a reference on a zerocopy structure */
1470 static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy)
1471 {
1472         struct ubuf_info *uarg = skb_zcopy(skb);
1473 
1474         if (uarg) {
1475                 if (skb_zcopy_is_nouarg(skb)) {
1476                         /* no notification callback */
1477                 } else if (uarg->callback == sock_zerocopy_callback) {
1478                         uarg->zerocopy = uarg->zerocopy && zerocopy;
1479                         sock_zerocopy_put(uarg);
1480                 } else {
1481                         uarg->callback(uarg, zerocopy);
1482                 }
1483 
1484                 skb_shinfo(skb)->tx_flags &= ~SKBTX_ZEROCOPY_FRAG;
1485         }
1486 }
1487 
1488 /* Abort a zerocopy operation and revert zckey on error in send syscall */
1489 static inline void skb_zcopy_abort(struct sk_buff *skb)
1490 {
1491         struct ubuf_info *uarg = skb_zcopy(skb);
1492 
1493         if (uarg) {
1494                 sock_zerocopy_put_abort(uarg, false);
1495                 skb_shinfo(skb)->tx_flags &= ~SKBTX_ZEROCOPY_FRAG;
1496         }
1497 }
1498 
1499 static inline void skb_mark_not_on_list(struct sk_buff *skb)
1500 {
1501         skb->next = NULL;
1502 }
1503 
1504 /* Iterate through singly-linked GSO fragments of an skb. */
1505 #define skb_list_walk_safe(first, skb, next_skb)                               \
1506         for ((skb) = (first), (next_skb) = (skb) ? (skb)->next : NULL; (skb);  \
1507              (skb) = (next_skb), (next_skb) = (skb) ? (skb)->next : NULL)
1508 
1509 static inline void skb_list_del_init(struct sk_buff *skb)
1510 {
1511         __list_del_entry(&skb->list);
1512         skb_mark_not_on_list(skb);
1513 }
1514 
1515 /**
1516  *      skb_queue_empty - check if a queue is empty
1517  *      @list: queue head
1518  *
1519  *      Returns true if the queue is empty, false otherwise.
1520  */
1521 static inline int skb_queue_empty(const struct sk_buff_head *list)
1522 {
1523         return list->next == (const struct sk_buff *) list;
1524 }
1525 
1526 /**
1527  *      skb_queue_empty_lockless - check if a queue is empty
1528  *      @list: queue head
1529  *
1530  *      Returns true if the queue is empty, false otherwise.
1531  *      This variant can be used in lockless contexts.
1532  */
1533 static inline bool skb_queue_empty_lockless(const struct sk_buff_head *list)
1534 {
1535         return READ_ONCE(list->next) == (const struct sk_buff *) list;
1536 }
1537 
1538 
1539 /**
1540  *      skb_queue_is_last - check if skb is the last entry in the queue
1541  *      @list: queue head
1542  *      @skb: buffer
1543  *
1544  *      Returns true if @skb is the last buffer on the list.
1545  */
1546 static inline bool skb_queue_is_last(const struct sk_buff_head *list,
1547                                      const struct sk_buff *skb)
1548 {
1549         return skb->next == (const struct sk_buff *) list;
1550 }
1551 
1552 /**
1553  *      skb_queue_is_first - check if skb is the first entry in the queue
1554  *      @list: queue head
1555  *      @skb: buffer
1556  *
1557  *      Returns true if @skb is the first buffer on the list.
1558  */
1559 static inline bool skb_queue_is_first(const struct sk_buff_head *list,
1560                                       const struct sk_buff *skb)
1561 {
1562         return skb->prev == (const struct sk_buff *) list;
1563 }
1564 
1565 /**
1566  *      skb_queue_next - return the next packet in the queue
1567  *      @list: queue head
1568  *      @skb: current buffer
1569  *
1570  *      Return the next packet in @list after @skb.  It is only valid to
1571  *      call this if skb_queue_is_last() evaluates to false.
1572  */
1573 static inline struct sk_buff *skb_queue_next(const struct sk_buff_head *list,
1574                                              const struct sk_buff *skb)
1575 {
1576         /* This BUG_ON may seem severe, but if we just return then we
1577          * are going to dereference garbage.
1578          */
1579         BUG_ON(skb_queue_is_last(list, skb));
1580         return skb->next;
1581 }
1582 
1583 /**
1584  *      skb_queue_prev - return the prev packet in the queue
1585  *      @list: queue head
1586  *      @skb: current buffer
1587  *
1588  *      Return the prev packet in @list before @skb.  It is only valid to
1589  *      call this if skb_queue_is_first() evaluates to false.
1590  */
1591 static inline struct sk_buff *skb_queue_prev(const struct sk_buff_head *list,
1592                                              const struct sk_buff *skb)
1593 {
1594         /* This BUG_ON may seem severe, but if we just return then we
1595          * are going to dereference garbage.
1596          */
1597         BUG_ON(skb_queue_is_first(list, skb));
1598         return skb->prev;
1599 }
1600 
1601 /**
1602  *      skb_get - reference buffer
1603  *      @skb: buffer to reference
1604  *
1605  *      Makes another reference to a socket buffer and returns a pointer
1606  *      to the buffer.
1607  */
1608 static inline struct sk_buff *skb_get(struct sk_buff *skb)
1609 {
1610         refcount_inc(&skb->users);
1611         return skb;
1612 }
1613 
1614 /*
1615  * If users == 1, we are the only owner and can avoid redundant atomic changes.
1616  */
1617 
1618 /**
1619  *      skb_cloned - is the buffer a clone
1620  *      @skb: buffer to check
1621  *
1622  *      Returns true if the buffer was generated with skb_clone() and is
1623  *      one of multiple shared copies of the buffer. Cloned buffers are
1624  *      shared data so must not be written to under normal circumstances.
1625  */
1626 static inline int skb_cloned(const struct sk_buff *skb)
1627 {
1628         return skb->cloned &&
1629                (atomic_read(&skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1;
1630 }
1631 
1632 static inline int skb_unclone(struct sk_buff *skb, gfp_t pri)
1633 {
1634         might_sleep_if(gfpflags_allow_blocking(pri));
1635 
1636         if (skb_cloned(skb))
1637                 return pskb_expand_head(skb, 0, 0, pri);
1638 
1639         return 0;
1640 }
1641 
1642 /**
1643  *      skb_header_cloned - is the header a clone
1644  *      @skb: buffer to check
1645  *
1646  *      Returns true if modifying the header part of the buffer requires
1647  *      the data to be copied.
1648  */
1649 static inline int skb_header_cloned(const struct sk_buff *skb)
1650 {
1651         int dataref;
1652 
1653         if (!skb->cloned)
1654                 return 0;
1655 
1656         dataref = atomic_read(&skb_shinfo(skb)->dataref);
1657         dataref = (dataref & SKB_DATAREF_MASK) - (dataref >> SKB_DATAREF_SHIFT);
1658         return dataref != 1;
1659 }
1660 
1661 static inline int skb_header_unclone(struct sk_buff *skb, gfp_t pri)
1662 {
1663         might_sleep_if(gfpflags_allow_blocking(pri));
1664 
1665         if (skb_header_cloned(skb))
1666                 return pskb_expand_head(skb, 0, 0, pri);
1667 
1668         return 0;
1669 }
1670 
1671 /**
1672  *      __skb_header_release - release reference to header
1673  *      @skb: buffer to operate on
1674  */
1675 static inline void __skb_header_release(struct sk_buff *skb)
1676 {
1677         skb->nohdr = 1;
1678         atomic_set(&skb_shinfo(skb)->dataref, 1 + (1 << SKB_DATAREF_SHIFT));
1679 }
1680 
1681 
1682 /**
1683  *      skb_shared - is the buffer shared
1684  *      @skb: buffer to check
1685  *
1686  *      Returns true if more than one person has a reference to this
1687  *      buffer.
1688  */
1689 static inline int skb_shared(const struct sk_buff *skb)
1690 {
1691         return refcount_read(&skb->users) != 1;
1692 }
1693 
1694 /**
1695  *      skb_share_check - check if buffer is shared and if so clone it
1696  *      @skb: buffer to check
1697  *      @pri: priority for memory allocation
1698  *
1699  *      If the buffer is shared the buffer is cloned and the old copy
1700  *      drops a reference. A new clone with a single reference is returned.
1701  *      If the buffer is not shared the original buffer is returned. When
1702  *      being called from interrupt status or with spinlocks held pri must
1703  *      be GFP_ATOMIC.
1704  *
1705  *      NULL is returned on a memory allocation failure.
1706  */
1707 static inline struct sk_buff *skb_share_check(struct sk_buff *skb, gfp_t pri)
1708 {
1709         might_sleep_if(gfpflags_allow_blocking(pri));
1710         if (skb_shared(skb)) {
1711                 struct sk_buff *nskb = skb_clone(skb, pri);
1712 
1713                 if (likely(nskb))
1714                         consume_skb(skb);
1715                 else
1716                         kfree_skb(skb);
1717                 skb = nskb;
1718         }
1719         return skb;
1720 }
1721 
1722 /*
1723  *      Copy shared buffers into a new sk_buff. We effectively do COW on
1724  *      packets to handle cases where we have a local reader and forward
1725  *      and a couple of other messy ones. The normal one is tcpdumping
1726  *      a packet thats being forwarded.
1727  */
1728 
1729 /**
1730  *      skb_unshare - make a copy of a shared buffer
1731  *      @skb: buffer to check
1732  *      @pri: priority for memory allocation
1733  *
1734  *      If the socket buffer is a clone then this function creates a new
1735  *      copy of the data, drops a reference count on the old copy and returns
1736  *      the new copy with the reference count at 1. If the buffer is not a clone
1737  *      the original buffer is returned. When called with a spinlock held or
1738  *      from interrupt state @pri must be %GFP_ATOMIC
1739  *
1740  *      %NULL is returned on a memory allocation failure.
1741  */
1742 static inline struct sk_buff *skb_unshare(struct sk_buff *skb,
1743                                           gfp_t pri)
1744 {
1745         might_sleep_if(gfpflags_allow_blocking(pri));
1746         if (skb_cloned(skb)) {
1747                 struct sk_buff *nskb = skb_copy(skb, pri);
1748 
1749                 /* Free our shared copy */
1750                 if (likely(nskb))
1751                         consume_skb(skb);
1752                 else
1753                         kfree_skb(skb);
1754                 skb = nskb;
1755         }
1756         return skb;
1757 }
1758 
1759 /**
1760  *      skb_peek - peek at the head of an &sk_buff_head
1761  *      @list_: list to peek at
1762  *
1763  *      Peek an &sk_buff. Unlike most other operations you _MUST_
1764  *      be careful with this one. A peek leaves the buffer on the
1765  *      list and someone else may run off with it. You must hold
1766  *      the appropriate locks or have a private queue to do this.
1767  *
1768  *      Returns %NULL for an empty list or a pointer to the head element.
1769  *      The reference count is not incremented and the reference is therefore
1770  *      volatile. Use with caution.
1771  */
1772 static inline struct sk_buff *skb_peek(const struct sk_buff_head *list_)
1773 {
1774         struct sk_buff *skb = list_->next;
1775 
1776         if (skb == (struct sk_buff *)list_)
1777                 skb = NULL;
1778         return skb;
1779 }
1780 
1781 /**
1782  *      __skb_peek - peek at the head of a non-empty &sk_buff_head
1783  *      @list_: list to peek at
1784  *
1785  *      Like skb_peek(), but the caller knows that the list is not empty.
1786  */
1787 static inline struct sk_buff *__skb_peek(const struct sk_buff_head *list_)
1788 {
1789         return list_->next;
1790 }
1791 
1792 /**
1793  *      skb_peek_next - peek skb following the given one from a queue
1794  *      @skb: skb to start from
1795  *      @list_: list to peek at
1796  *
1797  *      Returns %NULL when the end of the list is met or a pointer to the
1798  *      next element. The reference count is not incremented and the
1799  *      reference is therefore volatile. Use with caution.
1800  */
1801 static inline struct sk_buff *skb_peek_next(struct sk_buff *skb,
1802                 const struct sk_buff_head *list_)
1803 {
1804         struct sk_buff *next = skb->next;
1805 
1806         if (next == (struct sk_buff *)list_)
1807                 next = NULL;
1808         return next;
1809 }
1810 
1811 /**
1812  *      skb_peek_tail - peek at the tail of an &sk_buff_head
1813  *      @list_: list to peek at
1814  *
1815  *      Peek an &sk_buff. Unlike most other operations you _MUST_
1816  *      be careful with this one. A peek leaves the buffer on the
1817  *      list and someone else may run off with it. You must hold
1818  *      the appropriate locks or have a private queue to do this.
1819  *
1820  *      Returns %NULL for an empty list or a pointer to the tail element.
1821  *      The reference count is not incremented and the reference is therefore
1822  *      volatile. Use with caution.
1823  */
1824 static inline struct sk_buff *skb_peek_tail(const struct sk_buff_head *list_)
1825 {
1826         struct sk_buff *skb = READ_ONCE(list_->prev);
1827 
1828         if (skb == (struct sk_buff *)list_)
1829                 skb = NULL;
1830         return skb;
1831 
1832 }
1833 
1834 /**
1835  *      skb_queue_len   - get queue length
1836  *      @list_: list to measure
1837  *
1838  *      Return the length of an &sk_buff queue.
1839  */
1840 static inline __u32 skb_queue_len(const struct sk_buff_head *list_)
1841 {
1842         return list_->qlen;
1843 }
1844 
1845 /**
1846  *      skb_queue_len_lockless  - get queue length
1847  *      @list_: list to measure
1848  *
1849  *      Return the length of an &sk_buff queue.
1850  *      This variant can be used in lockless contexts.
1851  */
1852 static inline __u32 skb_queue_len_lockless(const struct sk_buff_head *list_)
1853 {
1854         return READ_ONCE(list_->qlen);
1855 }
1856 
1857 /**
1858  *      __skb_queue_head_init - initialize non-spinlock portions of sk_buff_head
1859  *      @list: queue to initialize
1860  *
1861  *      This initializes only the list and queue length aspects of
1862  *      an sk_buff_head object.  This allows to initialize the list
1863  *      aspects of an sk_buff_head without reinitializing things like
1864  *      the spinlock.  It can also be used for on-stack sk_buff_head
1865  *      objects where the spinlock is known to not be used.
1866  */
1867 static inline void __skb_queue_head_init(struct sk_buff_head *list)
1868 {
1869         list->prev = list->next = (struct sk_buff *)list;
1870         list->qlen = 0;
1871 }
1872 
1873 /*
1874  * This function creates a split out lock class for each invocation;
1875  * this is needed for now since a whole lot of users of the skb-queue
1876  * infrastructure in drivers have different locking usage (in hardirq)
1877  * than the networking core (in softirq only). In the long run either the
1878  * network layer or drivers should need annotation to consolidate the
1879  * main types of usage into 3 classes.
1880  */
1881 static inline void skb_queue_head_init(struct sk_buff_head *list)
1882 {
1883         spin_lock_init(&list->lock);
1884         __skb_queue_head_init(list);
1885 }
1886 
1887 static inline void skb_queue_head_init_class(struct sk_buff_head *list,
1888                 struct lock_class_key *class)
1889 {
1890         skb_queue_head_init(list);
1891         lockdep_set_class(&list->lock, class);
1892 }
1893 
1894 /*
1895  *      Insert an sk_buff on a list.
1896  *
1897  *      The "__skb_xxxx()" functions are the non-atomic ones that
1898  *      can only be called with interrupts disabled.
1899  */
1900 static inline void __skb_insert(struct sk_buff *newsk,
1901                                 struct sk_buff *prev, struct sk_buff *next,
1902                                 struct sk_buff_head *list)
1903 {
1904         /* See skb_queue_empty_lockless() and skb_peek_tail()
1905          * for the opposite READ_ONCE()
1906          */
1907         WRITE_ONCE(newsk->next, next);
1908         WRITE_ONCE(newsk->prev, prev);
1909         WRITE_ONCE(next->prev, newsk);
1910         WRITE_ONCE(prev->next, newsk);
1911         list->qlen++;
1912 }
1913 
1914 static inline void __skb_queue_splice(const struct sk_buff_head *list,
1915                                       struct sk_buff *prev,
1916                                       struct sk_buff *next)
1917 {
1918         struct sk_buff *first = list->next;
1919         struct sk_buff *last = list->prev;
1920 
1921         WRITE_ONCE(first->prev, prev);
1922         WRITE_ONCE(prev->next, first);
1923 
1924         WRITE_ONCE(last->next, next);
1925         WRITE_ONCE(next->prev, last);
1926 }
1927 
1928 /**
1929  *      skb_queue_splice - join two skb lists, this is designed for stacks
1930  *      @list: the new list to add
1931  *      @head: the place to add it in the first list
1932  */
1933 static inline void skb_queue_splice(const struct sk_buff_head *list,
1934                                     struct sk_buff_head *head)
1935 {
1936         if (!skb_queue_empty(list)) {
1937                 __skb_queue_splice(list, (struct sk_buff *) head, head->next);
1938                 head->qlen += list->qlen;
1939         }
1940 }
1941 
1942 /**
1943  *      skb_queue_splice_init - join two skb lists and reinitialise the emptied list
1944  *      @list: the new list to add
1945  *      @head: the place to add it in the first list
1946  *
1947  *      The list at @list is reinitialised
1948  */
1949 static inline void skb_queue_splice_init(struct sk_buff_head *list,
1950                                          struct sk_buff_head *head)
1951 {
1952         if (!skb_queue_empty(list)) {
1953                 __skb_queue_splice(list, (struct sk_buff *) head, head->next);
1954                 head->qlen += list->qlen;
1955                 __skb_queue_head_init(list);
1956         }
1957 }
1958 
1959 /**
1960  *      skb_queue_splice_tail - join two skb lists, each list being a queue
1961  *      @list: the new list to add
1962  *      @head: the place to add it in the first list
1963  */
1964 static inline void skb_queue_splice_tail(const struct sk_buff_head *list,
1965                                          struct sk_buff_head *head)
1966 {
1967         if (!skb_queue_empty(list)) {
1968                 __skb_queue_splice(list, head->prev, (struct sk_buff *) head);
1969                 head->qlen += list->qlen;
1970         }
1971 }
1972 
1973 /**
1974  *      skb_queue_splice_tail_init - join two skb lists and reinitialise the emptied list
1975  *      @list: the new list to add
1976  *      @head: the place to add it in the first list
1977  *
1978  *      Each of the lists is a queue.
1979  *      The list at @list is reinitialised
1980  */
1981 static inline void skb_queue_splice_tail_init(struct sk_buff_head *list,
1982                                               struct sk_buff_head *head)
1983 {
1984         if (!skb_queue_empty(list)) {
1985                 __skb_queue_splice(list, head->prev, (struct sk_buff *) head);
1986                 head->qlen += list->qlen;
1987                 __skb_queue_head_init(list);
1988         }
1989 }
1990 
1991 /**
1992  *      __skb_queue_after - queue a buffer at the list head
1993  *      @list: list to use
1994  *      @prev: place after this buffer
1995  *      @newsk: buffer to queue
1996  *
1997  *      Queue a buffer int the middle of a list. This function takes no locks
1998  *      and you must therefore hold required locks before calling it.
1999  *
2000  *      A buffer cannot be placed on two lists at the same time.
2001  */
2002 static inline void __skb_queue_after(struct sk_buff_head *list,
2003                                      struct sk_buff *prev,
2004                                      struct sk_buff *newsk)
2005 {
2006         __skb_insert(newsk, prev, prev->next, list);
2007 }
2008 
2009 void skb_append(struct sk_buff *old, struct sk_buff *newsk,
2010                 struct sk_buff_head *list);
2011 
2012 static inline void __skb_queue_before(struct sk_buff_head *list,
2013                                       struct sk_buff *next,
2014                                       struct sk_buff *newsk)
2015 {
2016         __skb_insert(newsk, next->prev, next, list);
2017 }
2018 
2019 /**
2020  *      __skb_queue_head - queue a buffer at the list head
2021  *      @list: list to use
2022  *      @newsk: buffer to queue
2023  *
2024  *      Queue a buffer at the start of a list. This function takes no locks
2025  *      and you must therefore hold required locks before calling it.
2026  *
2027  *      A buffer cannot be placed on two lists at the same time.
2028  */
2029 static inline void __skb_queue_head(struct sk_buff_head *list,
2030                                     struct sk_buff *newsk)
2031 {
2032         __skb_queue_after(list, (struct sk_buff *)list, newsk);
2033 }
2034 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk);
2035 
2036 /**
2037  *      __skb_queue_tail - queue a buffer at the list tail
2038  *      @list: list to use
2039  *      @newsk: buffer to queue
2040  *
2041  *      Queue a buffer at the end of a list. This function takes no locks
2042  *      and you must therefore hold required locks before calling it.
2043  *
2044  *      A buffer cannot be placed on two lists at the same time.
2045  */
2046 static inline void __skb_queue_tail(struct sk_buff_head *list,
2047                                    struct sk_buff *newsk)
2048 {
2049         __skb_queue_before(list, (struct sk_buff *)list, newsk);
2050 }
2051 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk);
2052 
2053 /*
2054  * remove sk_buff from list. _Must_ be called atomically, and with
2055  * the list known..
2056  */
2057 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list);
2058 static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
2059 {
2060         struct sk_buff *next, *prev;
2061 
2062         WRITE_ONCE(list->qlen, list->qlen - 1);
2063         next       = skb->next;
2064         prev       = skb->prev;
2065         skb->next  = skb->prev = NULL;
2066         WRITE_ONCE(next->prev, prev);
2067         WRITE_ONCE(prev->next, next);
2068 }
2069 
2070 /**
2071  *      __skb_dequeue - remove from the head of the queue
2072  *      @list: list to dequeue from
2073  *
2074  *      Remove the head of the list. This function does not take any locks
2075  *      so must be used with appropriate locks held only. The head item is
2076  *      returned or %NULL if the list is empty.
2077  */
2078 static inline struct sk_buff *__skb_dequeue(struct sk_buff_head *list)
2079 {
2080         struct sk_buff *skb = skb_peek(list);
2081         if (skb)
2082                 __skb_unlink(skb, list);
2083         return skb;
2084 }
2085 struct sk_buff *skb_dequeue(struct sk_buff_head *list);
2086 
2087 /**
2088  *      __skb_dequeue_tail - remove from the tail of the queue
2089  *      @list: list to dequeue from
2090  *
2091  *      Remove the tail of the list. This function does not take any locks
2092  *      so must be used with appropriate locks held only. The tail item is
2093  *      returned or %NULL if the list is empty.
2094  */
2095 static inline struct sk_buff *__skb_dequeue_tail(struct sk_buff_head *list)
2096 {
2097         struct sk_buff *skb = skb_peek_tail(list);
2098         if (skb)
2099                 __skb_unlink(skb, list);
2100         return skb;
2101 }
2102 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list);
2103 
2104 
2105 static inline bool skb_is_nonlinear(const struct sk_buff *skb)
2106 {
2107         return skb->data_len;
2108 }
2109 
2110 static inline unsigned int skb_headlen(const struct sk_buff *skb)
2111 {
2112         return skb->len - skb->data_len;
2113 }
2114 
2115 static inline unsigned int __skb_pagelen(const struct sk_buff *skb)
2116 {
2117         unsigned int i, len = 0;
2118 
2119         for (i = skb_shinfo(skb)->nr_frags - 1; (int)i >= 0; i--)
2120                 len += skb_frag_size(&skb_shinfo(skb)->frags[i]);
2121         return len;
2122 }
2123 
2124 static inline unsigned int skb_pagelen(const struct sk_buff *skb)
2125 {
2126         return skb_headlen(skb) + __skb_pagelen(skb);
2127 }
2128 
2129 /**
2130  * __skb_fill_page_desc - initialise a paged fragment in an skb
2131  * @skb: buffer containing fragment to be initialised
2132  * @i: paged fragment index to initialise
2133  * @page: the page to use for this fragment
2134  * @off: the offset to the data with @page
2135  * @size: the length of the data
2136  *
2137  * Initialises the @i'th fragment of @skb to point to &size bytes at
2138  * offset @off within @page.
2139  *
2140  * Does not take any additional reference on the fragment.
2141  */
2142 static inline void __skb_fill_page_desc(struct sk_buff *skb, int i,
2143                                         struct page *page, int off, int size)
2144 {
2145         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2146 
2147         /*
2148          * Propagate page pfmemalloc to the skb if we can. The problem is
2149          * that not all callers have unique ownership of the page but rely
2150          * on page_is_pfmemalloc doing the right thing(tm).
2151          */
2152         frag->bv_page             = page;
2153         frag->bv_offset           = off;
2154         skb_frag_size_set(frag, size);
2155 
2156         page = compound_head(page);
2157         if (page_is_pfmemalloc(page))
2158                 skb->pfmemalloc = true;
2159 }
2160 
2161 /**
2162  * skb_fill_page_desc - initialise a paged fragment in an skb
2163  * @skb: buffer containing fragment to be initialised
2164  * @i: paged fragment index to initialise
2165  * @page: the page to use for this fragment
2166  * @off: the offset to the data with @page
2167  * @size: the length of the data
2168  *
2169  * As per __skb_fill_page_desc() -- initialises the @i'th fragment of
2170  * @skb to point to @size bytes at offset @off within @page. In
2171  * addition updates @skb such that @i is the last fragment.
2172  *
2173  * Does not take any additional reference on the fragment.
2174  */
2175 static inline void skb_fill_page_desc(struct sk_buff *skb, int i,
2176                                       struct page *page, int off, int size)
2177 {
2178         __skb_fill_page_desc(skb, i, page, off, size);
2179         skb_shinfo(skb)->nr_frags = i + 1;
2180 }
2181 
2182 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
2183                      int size, unsigned int truesize);
2184 
2185 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
2186                           unsigned int truesize);
2187 
2188 #define SKB_LINEAR_ASSERT(skb)  BUG_ON(skb_is_nonlinear(skb))
2189 
2190 #ifdef NET_SKBUFF_DATA_USES_OFFSET
2191 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
2192 {
2193         return skb->head + skb->tail;
2194 }
2195 
2196 static inline void skb_reset_tail_pointer(struct sk_buff *skb)
2197 {
2198         skb->tail = skb->data - skb->head;
2199 }
2200 
2201 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
2202 {
2203         skb_reset_tail_pointer(skb);
2204         skb->tail += offset;
2205 }
2206 
2207 #else /* NET_SKBUFF_DATA_USES_OFFSET */
2208 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
2209 {
2210         return skb->tail;
2211 }
2212 
2213 static inline void skb_reset_tail_pointer(struct sk_buff *skb)
2214 {
2215         skb->tail = skb->data;
2216 }
2217 
2218 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
2219 {
2220         skb->tail = skb->data + offset;
2221 }
2222 
2223 #endif /* NET_SKBUFF_DATA_USES_OFFSET */
2224 
2225 /*
2226  *      Add data to an sk_buff
2227  */
2228 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len);
2229 void *skb_put(struct sk_buff *skb, unsigned int len);
2230 static inline void *__skb_put(struct sk_buff *skb, unsigned int len)
2231 {
2232         void *tmp = skb_tail_pointer(skb);
2233         SKB_LINEAR_ASSERT(skb);
2234         skb->tail += len;
2235         skb->len  += len;
2236         return tmp;
2237 }
2238 
2239 static inline void *__skb_put_zero(struct sk_buff *skb, unsigned int len)
2240 {
2241         void *tmp = __skb_put(skb, len);
2242 
2243         memset(tmp, 0, len);
2244         return tmp;
2245 }
2246 
2247 static inline void *__skb_put_data(struct sk_buff *skb, const void *data,
2248                                    unsigned int len)
2249 {
2250         void *tmp = __skb_put(skb, len);
2251 
2252         memcpy(tmp, data, len);
2253         return tmp;
2254 }
2255 
2256 static inline void __skb_put_u8(struct sk_buff *skb, u8 val)
2257 {
2258         *(u8 *)__skb_put(skb, 1) = val;
2259 }
2260 
2261 static inline void *skb_put_zero(struct sk_buff *skb, unsigned int len)
2262 {
2263         void *tmp = skb_put(skb, len);
2264 
2265         memset(tmp, 0, len);
2266 
2267         return tmp;
2268 }
2269 
2270 static inline void *skb_put_data(struct sk_buff *skb, const void *data,
2271                                  unsigned int len)
2272 {
2273         void *tmp = skb_put(skb, len);
2274 
2275         memcpy(tmp, data, len);
2276 
2277         return tmp;
2278 }
2279 
2280 static inline void skb_put_u8(struct sk_buff *skb, u8 val)
2281 {
2282         *(u8 *)skb_put(skb, 1) = val;
2283 }
2284 
2285 void *skb_push(struct sk_buff *skb, unsigned int len);
2286 static inline void *__skb_push(struct sk_buff *skb, unsigned int len)
2287 {
2288         skb->data -= len;
2289         skb->len  += len;
2290         return skb->data;
2291 }
2292 
2293 void *skb_pull(struct sk_buff *skb, unsigned int len);
2294 static inline void *__skb_pull(struct sk_buff *skb, unsigned int len)
2295 {
2296         skb->len -= len;
2297         BUG_ON(skb->len < skb->data_len);
2298         return skb->data += len;
2299 }
2300 
2301 static inline void *skb_pull_inline(struct sk_buff *skb, unsigned int len)
2302 {
2303         return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
2304 }
2305 
2306 void *__pskb_pull_tail(struct sk_buff *skb, int delta);
2307 
2308 static inline void *__pskb_pull(struct sk_buff *skb, unsigned int len)
2309 {
2310         if (len > skb_headlen(skb) &&
2311             !__pskb_pull_tail(skb, len - skb_headlen(skb)))
2312                 return NULL;
2313         skb->len -= len;
2314         return skb->data += len;
2315 }
2316 
2317 static inline void *pskb_pull(struct sk_buff *skb, unsigned int len)
2318 {
2319         return unlikely(len > skb->len) ? NULL : __pskb_pull(skb, len);
2320 }
2321 
2322 static inline bool pskb_may_pull(struct sk_buff *skb, unsigned int len)
2323 {
2324         if (likely(len <= skb_headlen(skb)))
2325                 return true;
2326         if (unlikely(len > skb->len))
2327                 return false;
2328         return __pskb_pull_tail(skb, len - skb_headlen(skb)) != NULL;
2329 }
2330 
2331 void skb_condense(struct sk_buff *skb);
2332 
2333 /**
2334  *      skb_headroom - bytes at buffer head
2335  *      @skb: buffer to check
2336  *
2337  *      Return the number of bytes of free space at the head of an &sk_buff.
2338  */
2339 static inline unsigned int skb_headroom(const struct sk_buff *skb)
2340 {
2341         return skb->data - skb->head;
2342 }
2343 
2344 /**
2345  *      skb_tailroom - bytes at buffer end
2346  *      @skb: buffer to check
2347  *
2348  *      Return the number of bytes of free space at the tail of an sk_buff
2349  */
2350 static inline int skb_tailroom(const struct sk_buff *skb)
2351 {
2352         return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail;
2353 }
2354 
2355 /**
2356  *      skb_availroom - bytes at buffer end
2357  *      @skb: buffer to check
2358  *
2359  *      Return the number of bytes of free space at the tail of an sk_buff
2360  *      allocated by sk_stream_alloc()
2361  */
2362 static inline int skb_availroom(const struct sk_buff *skb)
2363 {
2364         if (skb_is_nonlinear(skb))
2365                 return 0;
2366 
2367         return skb->end - skb->tail - skb->reserved_tailroom;
2368 }
2369 
2370 /**
2371  *      skb_reserve - adjust headroom
2372  *      @skb: buffer to alter
2373  *      @len: bytes to move
2374  *
2375  *      Increase the headroom of an empty &sk_buff by reducing the tail
2376  *      room. This is only allowed for an empty buffer.
2377  */
2378 static inline void skb_reserve(struct sk_buff *skb, int len)
2379 {
2380         skb->data += len;
2381         skb->tail += len;
2382 }
2383 
2384 /**
2385  *      skb_tailroom_reserve - adjust reserved_tailroom
2386  *      @skb: buffer to alter
2387  *      @mtu: maximum amount of headlen permitted
2388  *      @needed_tailroom: minimum amount of reserved_tailroom
2389  *
2390  *      Set reserved_tailroom so that headlen can be as large as possible but
2391  *      not larger than mtu and tailroom cannot be smaller than
2392  *      needed_tailroom.
2393  *      The required headroom should already have been reserved before using
2394  *      this function.
2395  */
2396 static inline void skb_tailroom_reserve(struct sk_buff *skb, unsigned int mtu,
2397                                         unsigned int needed_tailroom)
2398 {
2399         SKB_LINEAR_ASSERT(skb);
2400         if (mtu < skb_tailroom(skb) - needed_tailroom)
2401                 /* use at most mtu */
2402                 skb->reserved_tailroom = skb_tailroom(skb) - mtu;
2403         else
2404                 /* use up to all available space */
2405                 skb->reserved_tailroom = needed_tailroom;
2406 }
2407 
2408 #define ENCAP_TYPE_ETHER        0
2409 #define ENCAP_TYPE_IPPROTO      1
2410 
2411 static inline void skb_set_inner_protocol(struct sk_buff *skb,
2412                                           __be16 protocol)
2413 {
2414         skb->inner_protocol = protocol;
2415         skb->inner_protocol_type = ENCAP_TYPE_ETHER;
2416 }
2417 
2418 static inline void skb_set_inner_ipproto(struct sk_buff *skb,
2419                                          __u8 ipproto)
2420 {
2421         skb->inner_ipproto = ipproto;
2422         skb->inner_protocol_type = ENCAP_TYPE_IPPROTO;
2423 }
2424 
2425 static inline void skb_reset_inner_headers(struct sk_buff *skb)
2426 {
2427         skb->inner_mac_header = skb->mac_header;
2428         skb->inner_network_header = skb->network_header;
2429         skb->inner_transport_header = skb->transport_header;
2430 }
2431 
2432 static inline void skb_reset_mac_len(struct sk_buff *skb)
2433 {
2434         skb->mac_len = skb->network_header - skb->mac_header;
2435 }
2436 
2437 static inline unsigned char *skb_inner_transport_header(const struct sk_buff
2438                                                         *skb)
2439 {
2440         return skb->head + skb->inner_transport_header;
2441 }
2442 
2443 static inline int skb_inner_transport_offset(const struct sk_buff *skb)
2444 {
2445         return skb_inner_transport_header(skb) - skb->data;
2446 }
2447 
2448 static inline void skb_reset_inner_transport_header(struct sk_buff *skb)
2449 {
2450         skb->inner_transport_header = skb->data - skb->head;
2451 }
2452 
2453 static inline void skb_set_inner_transport_header(struct sk_buff *skb,
2454                                                    const int offset)
2455 {
2456         skb_reset_inner_transport_header(skb);
2457         skb->inner_transport_header += offset;
2458 }
2459 
2460 static inline unsigned char *skb_inner_network_header(const struct sk_buff *skb)
2461 {
2462         return skb->head + skb->inner_network_header;
2463 }
2464 
2465 static inline void skb_reset_inner_network_header(struct sk_buff *skb)
2466 {
2467         skb->inner_network_header = skb->data - skb->head;
2468 }
2469 
2470 static inline void skb_set_inner_network_header(struct sk_buff *skb,
2471                                                 const int offset)
2472 {
2473         skb_reset_inner_network_header(skb);
2474         skb->inner_network_header += offset;
2475 }
2476 
2477 static inline unsigned char *skb_inner_mac_header(const struct sk_buff *skb)
2478 {
2479         return skb->head + skb->inner_mac_header;
2480 }
2481 
2482 static inline void skb_reset_inner_mac_header(struct sk_buff *skb)
2483 {
2484         skb->inner_mac_header = skb->data - skb->head;
2485 }
2486 
2487 static inline void skb_set_inner_mac_header(struct sk_buff *skb,
2488                                             const int offset)
2489 {
2490         skb_reset_inner_mac_header(skb);
2491         skb->inner_mac_header += offset;
2492 }
2493 static inline bool skb_transport_header_was_set(const struct sk_buff *skb)
2494 {
2495         return skb->transport_header != (typeof(skb->transport_header))~0U;
2496 }
2497 
2498 static inline unsigned char *skb_transport_header(const struct sk_buff *skb)
2499 {
2500         return skb->head + skb->transport_header;
2501 }
2502 
2503 static inline void skb_reset_transport_header(struct sk_buff *skb)
2504 {
2505         skb->transport_header = skb->data - skb->head;
2506 }
2507 
2508 static inline void skb_set_transport_header(struct sk_buff *skb,
2509                                             const int offset)
2510 {
2511         skb_reset_transport_header(skb);
2512         skb->transport_header += offset;
2513 }
2514 
2515 static inline unsigned char *skb_network_header(const struct sk_buff *skb)
2516 {
2517         return skb->head + skb->network_header;
2518 }
2519 
2520 static inline void skb_reset_network_header(struct sk_buff *skb)
2521 {
2522         skb->network_header = skb->data - skb->head;
2523 }
2524 
2525 static inline void skb_set_network_header(struct sk_buff *skb, const int offset)
2526 {
2527         skb_reset_network_header(skb);
2528         skb->network_header += offset;
2529 }
2530 
2531 static inline unsigned char *skb_mac_header(const struct sk_buff *skb)
2532 {
2533         return skb->head + skb->mac_header;
2534 }
2535 
2536 static inline int skb_mac_offset(const struct sk_buff *skb)
2537 {
2538         return skb_mac_header(skb) - skb->data;
2539 }
2540 
2541 static inline u32 skb_mac_header_len(const struct sk_buff *skb)
2542 {
2543         return skb->network_header - skb->mac_header;
2544 }
2545 
2546 static inline int skb_mac_header_was_set(const struct sk_buff *skb)
2547 {
2548         return skb->mac_header != (typeof(skb->mac_header))~0U;
2549 }
2550 
2551 static inline void skb_reset_mac_header(struct sk_buff *skb)
2552 {
2553         skb->mac_header = skb->data - skb->head;
2554 }
2555 
2556 static inline void skb_set_mac_header(struct sk_buff *skb, const int offset)
2557 {
2558         skb_reset_mac_header(skb);
2559         skb->mac_header += offset;
2560 }
2561 
2562 static inline void skb_pop_mac_header(struct sk_buff *skb)
2563 {
2564         skb->mac_header = skb->network_header;
2565 }
2566 
2567 static inline void skb_probe_transport_header(struct sk_buff *skb)
2568 {
2569         struct flow_keys_basic keys;
2570 
2571         if (skb_transport_header_was_set(skb))
2572                 return;
2573 
2574         if (skb_flow_dissect_flow_keys_basic(NULL, skb, &keys,
2575                                              NULL, 0, 0, 0, 0))
2576                 skb_set_transport_header(skb, keys.control.thoff);
2577 }
2578 
2579 static inline void skb_mac_header_rebuild(struct sk_buff *skb)
2580 {
2581         if (skb_mac_header_was_set(skb)) {
2582                 const unsigned char *old_mac = skb_mac_header(skb);
2583 
2584                 skb_set_mac_header(skb, -skb->mac_len);
2585                 memmove(skb_mac_header(skb), old_mac, skb->mac_len);
2586         }
2587 }
2588 
2589 static inline int skb_checksum_start_offset(const struct sk_buff *skb)
2590 {
2591         return skb->csum_start - skb_headroom(skb);
2592 }
2593 
2594 static inline unsigned char *skb_checksum_start(const struct sk_buff *skb)
2595 {
2596         return skb->head + skb->csum_start;
2597 }
2598 
2599 static inline int skb_transport_offset(const struct sk_buff *skb)
2600 {
2601         return skb_transport_header(skb) - skb->data;
2602 }
2603 
2604 static inline u32 skb_network_header_len(const struct sk_buff *skb)
2605 {
2606         return skb->transport_header - skb->network_header;
2607 }
2608 
2609 static inline u32 skb_inner_network_header_len(const struct sk_buff *skb)
2610 {
2611         return skb->inner_transport_header - skb->inner_network_header;
2612 }
2613 
2614 static inline int skb_network_offset(const struct sk_buff *skb)
2615 {
2616         return skb_network_header(skb) - skb->data;
2617 }
2618 
2619 static inline int skb_inner_network_offset(const struct sk_buff *skb)
2620 {
2621         return skb_inner_network_header(skb) - skb->data;
2622 }
2623 
2624 static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len)
2625 {
2626         return pskb_may_pull(skb, skb_network_offset(skb) + len);
2627 }
2628 
2629 /*
2630  * CPUs often take a performance hit when accessing unaligned memory
2631  * locations. The actual performance hit varies, it can be small if the
2632  * hardware handles it or large if we have to take an exception and fix it
2633  * in software.
2634  *
2635  * Since an ethernet header is 14 bytes network drivers often end up with
2636  * the IP header at an unaligned offset. The IP header can be aligned by
2637  * shifting the start of the packet by 2 bytes. Drivers should do this
2638  * with:
2639  *
2640  * skb_reserve(skb, NET_IP_ALIGN);
2641  *
2642  * The downside to this alignment of the IP header is that the DMA is now
2643  * unaligned. On some architectures the cost of an unaligned DMA is high
2644  * and this cost outweighs the gains made by aligning the IP header.
2645  *
2646  * Since this trade off varies between architectures, we allow NET_IP_ALIGN
2647  * to be overridden.
2648  */
2649 #ifndef NET_IP_ALIGN
2650 #define NET_IP_ALIGN    2
2651 #endif
2652 
2653 /*
2654  * The networking layer reserves some headroom in skb data (via
2655  * dev_alloc_skb). This is used to avoid having to reallocate skb data when
2656  * the header has to grow. In the default case, if the header has to grow
2657  * 32 bytes or less we avoid the reallocation.
2658  *
2659  * Unfortunately this headroom changes the DMA alignment of the resulting
2660  * network packet. As for NET_IP_ALIGN, this unaligned DMA is expensive
2661  * on some architectures. An architecture can override this value,
2662  * perhaps setting it to a cacheline in size (since that will maintain
2663  * cacheline alignment of the DMA). It must be a power of 2.
2664  *
2665  * Various parts of the networking layer expect at least 32 bytes of
2666  * headroom, you should not reduce this.
2667  *
2668  * Using max(32, L1_CACHE_BYTES) makes sense (especially with RPS)
2669  * to reduce average number of cache lines per packet.
2670  * get_rps_cpu() for example only access one 64 bytes aligned block :
2671  * NET_IP_ALIGN(2) + ethernet_header(14) + IP_header(20/40) + ports(8)
2672  */
2673 #ifndef NET_SKB_PAD
2674 #define NET_SKB_PAD     max(32, L1_CACHE_BYTES)
2675 #endif
2676 
2677 int ___pskb_trim(struct sk_buff *skb, unsigned int len);
2678 
2679 static inline void __skb_set_length(struct sk_buff *skb, unsigned int len)
2680 {
2681         if (WARN_ON(skb_is_nonlinear(skb)))
2682                 return;
2683         skb->len = len;
2684         skb_set_tail_pointer(skb, len);
2685 }
2686 
2687 static inline void __skb_trim(struct sk_buff *skb, unsigned int len)
2688 {
2689         __skb_set_length(skb, len);
2690 }
2691 
2692 void skb_trim(struct sk_buff *skb, unsigned int len);
2693 
2694 static inline int __pskb_trim(struct sk_buff *skb, unsigned int len)
2695 {
2696         if (skb->data_len)
2697                 return ___pskb_trim(skb, len);
2698         __skb_trim(skb, len);
2699         return 0;
2700 }
2701 
2702 static inline int pskb_trim(struct sk_buff *skb, unsigned int len)
2703 {
2704         return (len < skb->len) ? __pskb_trim(skb, len) : 0;
2705 }
2706 
2707 /**
2708  *      pskb_trim_unique - remove end from a paged unique (not cloned) buffer
2709  *      @skb: buffer to alter
2710  *      @len: new length
2711  *
2712  *      This is identical to pskb_trim except that the caller knows that
2713  *      the skb is not cloned so we should never get an error due to out-
2714  *      of-memory.
2715  */
2716 static inline void pskb_trim_unique(struct sk_buff *skb, unsigned int len)
2717 {
2718         int err = pskb_trim(skb, len);
2719         BUG_ON(err);
2720 }
2721 
2722 static inline int __skb_grow(struct sk_buff *skb, unsigned int len)
2723 {
2724         unsigned int diff = len - skb->len;
2725 
2726         if (skb_tailroom(skb) < diff) {
2727                 int ret = pskb_expand_head(skb, 0, diff - skb_tailroom(skb),
2728                                            GFP_ATOMIC);
2729                 if (ret)
2730                         return ret;
2731         }
2732         __skb_set_length(skb, len);
2733         return 0;
2734 }
2735 
2736 /**
2737  *      skb_orphan - orphan a buffer
2738  *      @skb: buffer to orphan
2739  *
2740  *      If a buffer currently has an owner then we call the owner's
2741  *      destructor function and make the @skb unowned. The buffer continues
2742  *      to exist but is no longer charged to its former owner.
2743  */
2744 static inline void skb_orphan(struct sk_buff *skb)
2745 {
2746         if (skb->destructor) {
2747                 skb->destructor(skb);
2748                 skb->destructor = NULL;
2749                 skb->sk         = NULL;
2750         } else {
2751                 BUG_ON(skb->sk);
2752         }
2753 }
2754 
2755 /**
2756  *      skb_orphan_frags - orphan the frags contained in a buffer
2757  *      @skb: buffer to orphan frags from
2758  *      @gfp_mask: allocation mask for replacement pages
2759  *
2760  *      For each frag in the SKB which needs a destructor (i.e. has an
2761  *      owner) create a copy of that frag and release the original
2762  *      page by calling the destructor.
2763  */
2764 static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask)
2765 {
2766         if (likely(!skb_zcopy(skb)))
2767                 return 0;
2768         if (!skb_zcopy_is_nouarg(skb) &&
2769             skb_uarg(skb)->callback == sock_zerocopy_callback)
2770                 return 0;
2771         return skb_copy_ubufs(skb, gfp_mask);
2772 }
2773 
2774 /* Frags must be orphaned, even if refcounted, if skb might loop to rx path */
2775 static inline int skb_orphan_frags_rx(struct sk_buff *skb, gfp_t gfp_mask)
2776 {
2777         if (likely(!skb_zcopy(skb)))
2778                 return 0;
2779         return skb_copy_ubufs(skb, gfp_mask);
2780 }
2781 
2782 /**
2783  *      __skb_queue_purge - empty a list
2784  *      @list: list to empty
2785  *
2786  *      Delete all buffers on an &sk_buff list. Each buffer is removed from
2787  *      the list and one reference dropped. This function does not take the
2788  *      list lock and the caller must hold the relevant locks to use it.
2789  */
2790 static inline void __skb_queue_purge(struct sk_buff_head *list)
2791 {
2792         struct sk_buff *skb;
2793         while ((skb = __skb_dequeue(list)) != NULL)
2794                 kfree_skb(skb);
2795 }
2796 void skb_queue_purge(struct sk_buff_head *list);
2797 
2798 unsigned int skb_rbtree_purge(struct rb_root *root);
2799 
2800 void *netdev_alloc_frag(unsigned int fragsz);
2801 
2802 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int length,
2803                                    gfp_t gfp_mask);
2804 
2805 /**
2806  *      netdev_alloc_skb - allocate an skbuff for rx on a specific device
2807  *      @dev: network device to receive on
2808  *      @length: length to allocate
2809  *
2810  *      Allocate a new &sk_buff and assign it a usage count of one. The
2811  *      buffer has unspecified headroom built in. Users should allocate
2812  *      the headroom they think they need without accounting for the
2813  *      built in space. The built in space is used for optimisations.
2814  *
2815  *      %NULL is returned if there is no free memory. Although this function
2816  *      allocates memory it can be called from an interrupt.
2817  */
2818 static inline struct sk_buff *netdev_alloc_skb(struct net_device *dev,
2819                                                unsigned int length)
2820 {
2821         return __netdev_alloc_skb(dev, length, GFP_ATOMIC);
2822 }
2823 
2824 /* legacy helper around __netdev_alloc_skb() */
2825 static inline struct sk_buff *__dev_alloc_skb(unsigned int length,
2826                                               gfp_t gfp_mask)
2827 {
2828         return __netdev_alloc_skb(NULL, length, gfp_mask);
2829 }
2830 
2831 /* legacy helper around netdev_alloc_skb() */
2832 static inline struct sk_buff *dev_alloc_skb(unsigned int length)
2833 {
2834         return netdev_alloc_skb(NULL, length);
2835 }
2836 
2837 
2838 static inline struct sk_buff *__netdev_alloc_skb_ip_align(struct net_device *dev,
2839                 unsigned int length, gfp_t gfp)
2840 {
2841         struct sk_buff *skb = __netdev_alloc_skb(dev, length + NET_IP_ALIGN, gfp);
2842 
2843         if (NET_IP_ALIGN && skb)
2844                 skb_reserve(skb, NET_IP_ALIGN);
2845         return skb;
2846 }
2847 
2848 static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev,
2849                 unsigned int length)
2850 {
2851         return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC);
2852 }
2853 
2854 static inline void skb_free_frag(void *addr)
2855 {
2856         page_frag_free(addr);
2857 }
2858 
2859 void *napi_alloc_frag(unsigned int fragsz);
2860 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi,
2861                                  unsigned int length, gfp_t gfp_mask);
2862 static inline struct sk_buff *napi_alloc_skb(struct napi_struct *napi,
2863                                              unsigned int length)
2864 {
2865         return __napi_alloc_skb(napi, length, GFP_ATOMIC);
2866 }
2867 void napi_consume_skb(struct sk_buff *skb, int budget);
2868 
2869 void __kfree_skb_flush(void);
2870 void __kfree_skb_defer(struct sk_buff *skb);
2871 
2872 /**
2873  * __dev_alloc_pages - allocate page for network Rx
2874  * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx
2875  * @order: size of the allocation
2876  *
2877  * Allocate a new page.
2878  *
2879  * %NULL is returned if there is no free memory.
2880 */
2881 static inline struct page *__dev_alloc_pages(gfp_t gfp_mask,
2882                                              unsigned int order)
2883 {
2884         /* This piece of code contains several assumptions.
2885          * 1.  This is for device Rx, therefor a cold page is preferred.
2886          * 2.  The expectation is the user wants a compound page.
2887          * 3.  If requesting a order 0 page it will not be compound
2888          *     due to the check to see if order has a value in prep_new_page
2889          * 4.  __GFP_MEMALLOC is ignored if __GFP_NOMEMALLOC is set due to
2890          *     code in gfp_to_alloc_flags that should be enforcing this.
2891          */
2892         gfp_mask |= __GFP_COMP | __GFP_MEMALLOC;
2893 
2894         return alloc_pages_node(NUMA_NO_NODE, gfp_mask, order);
2895 }
2896 
2897 static inline struct page *dev_alloc_pages(unsigned int order)
2898 {
2899         return __dev_alloc_pages(GFP_ATOMIC | __GFP_NOWARN, order);
2900 }
2901 
2902 /**
2903  * __dev_alloc_page - allocate a page for network Rx
2904  * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx
2905  *
2906  * Allocate a new page.
2907  *
2908  * %NULL is returned if there is no free memory.
2909  */
2910 static inline struct page *__dev_alloc_page(gfp_t gfp_mask)
2911 {
2912         return __dev_alloc_pages(gfp_mask, 0);
2913 }
2914 
2915 static inline struct page *dev_alloc_page(void)
2916 {
2917         return dev_alloc_pages(0);
2918 }
2919 
2920 /**
2921  *      skb_propagate_pfmemalloc - Propagate pfmemalloc if skb is allocated after RX page
2922  *      @page: The page that was allocated from skb_alloc_page
2923  *      @skb: The skb that may need pfmemalloc set
2924  */
2925 static inline void skb_propagate_pfmemalloc(struct page *page,
2926                                              struct sk_buff *skb)
2927 {
2928         if (page_is_pfmemalloc(page))
2929                 skb->pfmemalloc = true;
2930 }
2931 
2932 /**
2933  * skb_frag_off() - Returns the offset of a skb fragment
2934  * @frag: the paged fragment
2935  */
2936 static inline unsigned int skb_frag_off(const skb_frag_t *frag)
2937 {
2938         return frag->bv_offset;
2939 }
2940 
2941 /**
2942  * skb_frag_off_add() - Increments the offset of a skb fragment by @delta
2943  * @frag: skb fragment
2944  * @delta: value to add
2945  */
2946 static inline void skb_frag_off_add(skb_frag_t *frag, int delta)
2947 {
2948         frag->bv_offset += delta;
2949 }
2950 
2951 /**
2952  * skb_frag_off_set() - Sets the offset of a skb fragment
2953  * @frag: skb fragment
2954  * @offset: offset of fragment
2955  */
2956 static inline void skb_frag_off_set(skb_frag_t *frag, unsigned int offset)
2957 {
2958         frag->bv_offset = offset;
2959 }
2960 
2961 /**
2962  * skb_frag_off_copy() - Sets the offset of a skb fragment from another fragment
2963  * @fragto: skb fragment where offset is set
2964  * @fragfrom: skb fragment offset is copied from
2965  */
2966 static inline void skb_frag_off_copy(skb_frag_t *fragto,
2967                                      const skb_frag_t *fragfrom)
2968 {
2969         fragto->bv_offset = fragfrom->bv_offset;
2970 }
2971 
2972 /**
2973  * skb_frag_page - retrieve the page referred to by a paged fragment
2974  * @frag: the paged fragment
2975  *
2976  * Returns the &struct page associated with @frag.
2977  */
2978 static inline struct page *skb_frag_page(const skb_frag_t *frag)
2979 {
2980         return frag->bv_page;
2981 }
2982 
2983 /**
2984  * __skb_frag_ref - take an addition reference on a paged fragment.
2985  * @frag: the paged fragment
2986  *
2987  * Takes an additional reference on the paged fragment @frag.
2988  */
2989 static inline void __skb_frag_ref(skb_frag_t *frag)
2990 {
2991         get_page(skb_frag_page(frag));
2992 }
2993 
2994 /**
2995  * skb_frag_ref - take an addition reference on a paged fragment of an skb.
2996  * @skb: the buffer
2997  * @f: the fragment offset.
2998  *
2999  * Takes an additional reference on the @f'th paged fragment of @skb.
3000  */
3001 static inline void skb_frag_ref(struct sk_buff *skb, int f)
3002 {
3003         __skb_frag_ref(&skb_shinfo(skb)->frags[f]);
3004 }
3005 
3006 /**
3007  * __skb_frag_unref - release a reference on a paged fragment.
3008  * @frag: the paged fragment
3009  *
3010  * Releases a reference on the paged fragment @frag.
3011  */
3012 static inline void __skb_frag_unref(skb_frag_t *frag)
3013 {
3014         put_page(skb_frag_page(frag));
3015 }
3016 
3017 /**
3018  * skb_frag_unref - release a reference on a paged fragment of an skb.
3019  * @skb: the buffer
3020  * @f: the fragment offset
3021  *
3022  * Releases a reference on the @f'th paged fragment of @skb.
3023  */
3024 static inline void skb_frag_unref(struct sk_buff *skb, int f)
3025 {
3026         __skb_frag_unref(&skb_shinfo(skb)->frags[f]);
3027 }
3028 
3029 /**
3030  * skb_frag_address - gets the address of the data contained in a paged fragment
3031  * @frag: the paged fragment buffer
3032  *
3033  * Returns the address of the data within @frag. The page must already
3034  * be mapped.
3035  */
3036 static inline void *skb_frag_address(const skb_frag_t *frag)
3037 {
3038         return page_address(skb_frag_page(frag)) + skb_frag_off(frag);
3039 }
3040 
3041 /**
3042  * skb_frag_address_safe - gets the address of the data contained in a paged fragment
3043  * @frag: the paged fragment buffer
3044  *
3045  * Returns the address of the data within @frag. Checks that the page
3046  * is mapped and returns %NULL otherwise.
3047  */
3048 static inline void *skb_frag_address_safe(const skb_frag_t *frag)
3049 {
3050         void *ptr = page_address(skb_frag_page(frag));
3051         if (unlikely(!ptr))
3052                 return NULL;
3053 
3054         return ptr + skb_frag_off(frag);
3055 }
3056 
3057 /**
3058  * skb_frag_page_copy() - sets the page in a fragment from another fragment
3059  * @fragto: skb fragment where page is set
3060  * @fragfrom: skb fragment page is copied from
3061  */
3062 static inline void skb_frag_page_copy(skb_frag_t *fragto,
3063                                       const skb_frag_t *fragfrom)
3064 {
3065         fragto->bv_page = fragfrom->bv_page;
3066 }
3067 
3068 /**
3069  * __skb_frag_set_page - sets the page contained in a paged fragment
3070  * @frag: the paged fragment
3071  * @page: the page to set
3072  *
3073  * Sets the fragment @frag to contain @page.
3074  */
3075 static inline void __skb_frag_set_page(skb_frag_t *frag, struct page *page)
3076 {
3077         frag->bv_page = page;
3078 }
3079 
3080 /**
3081  * skb_frag_set_page - sets the page contained in a paged fragment of an skb
3082  * @skb: the buffer
3083  * @f: the fragment offset
3084  * @page: the page to set
3085  *
3086  * Sets the @f'th fragment of @skb to contain @page.
3087  */
3088 static inline void skb_frag_set_page(struct sk_buff *skb, int f,
3089                                      struct page *page)
3090 {
3091         __skb_frag_set_page(&skb_shinfo(skb)->frags[f], page);
3092 }
3093 
3094 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio);
3095 
3096 /**
3097  * skb_frag_dma_map - maps a paged fragment via the DMA API
3098  * @dev: the device to map the fragment to
3099  * @frag: the paged fragment to map
3100  * @offset: the offset within the fragment (starting at the
3101  *          fragment's own offset)
3102  * @size: the number of bytes to map
3103  * @dir: the direction of the mapping (``PCI_DMA_*``)
3104  *
3105  * Maps the page associated with @frag to @device.
3106  */
3107 static inline dma_addr_t skb_frag_dma_map(struct device *dev,
3108                                           const skb_frag_t *frag,
3109                                           size_t offset, size_t size,
3110                                           enum dma_data_direction dir)
3111 {
3112         return dma_map_page(dev, skb_frag_page(frag),
3113                             skb_frag_off(frag) + offset, size, dir);
3114 }
3115 
3116 static inline struct sk_buff *pskb_copy(struct sk_buff *skb,
3117                                         gfp_t gfp_mask)
3118 {
3119         return __pskb_copy(skb, skb_headroom(skb), gfp_mask);
3120 }
3121 
3122 
3123 static inline struct sk_buff *pskb_copy_for_clone(struct sk_buff *skb,
3124                                                   gfp_t gfp_mask)
3125 {
3126         return __pskb_copy_fclone(skb, skb_headroom(skb), gfp_mask, true);
3127 }
3128 
3129 
3130 /**
3131  *      skb_clone_writable - is the header of a clone writable
3132  *      @skb: buffer to check
3133  *      @len: length up to which to write
3134  *
3135  *      Returns true if modifying the header part of the cloned buffer
3136  *      does not requires the data to be copied.
3137  */
3138 static inline int skb_clone_writable(const struct sk_buff *skb, unsigned int len)
3139 {
3140         return !skb_header_cloned(skb) &&
3141                skb_headroom(skb) + len <= skb->hdr_len;
3142 }
3143 
3144 static inline int skb_try_make_writable(struct sk_buff *skb,
3145                                         unsigned int write_len)
3146 {
3147         return skb_cloned(skb) && !skb_clone_writable(skb, write_len) &&
3148                pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3149 }
3150 
3151 static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom,
3152                             int cloned)
3153 {
3154         int delta = 0;
3155 
3156         if (headroom > skb_headroom(skb))
3157                 delta = headroom - skb_headroom(skb);
3158 
3159         if (delta || cloned)
3160                 return pskb_expand_head(skb, ALIGN(delta, NET_SKB_PAD), 0,
3161                                         GFP_ATOMIC);
3162         return 0;
3163 }
3164 
3165 /**
3166  *      skb_cow - copy header of skb when it is required
3167  *      @skb: buffer to cow
3168  *      @headroom: needed headroom
3169  *
3170  *      If the skb passed lacks sufficient headroom or its data part
3171  *      is shared, data is reallocated. If reallocation fails, an error
3172  *      is returned and original skb is not changed.
3173  *
3174  *      The result is skb with writable area skb->head...skb->tail
3175  *      and at least @headroom of space at head.
3176  */
3177 static inline int skb_cow(struct sk_buff *skb, unsigned int headroom)
3178 {
3179         return __skb_cow(skb, headroom, skb_cloned(skb));
3180 }
3181 
3182 /**
3183  *      skb_cow_head - skb_cow but only making the head writable
3184  *      @skb: buffer to cow
3185  *      @headroom: needed headroom
3186  *
3187  *      This function is identical to skb_cow except that we replace the
3188  *      skb_cloned check by skb_header_cloned.  It should be used when
3189  *      you only need to push on some header and do not need to modify
3190  *      the data.
3191  */
3192 static inline int skb_cow_head(struct sk_buff *skb, unsigned int headroom)
3193 {
3194         return __skb_cow(skb, headroom, skb_header_cloned(skb));
3195 }
3196 
3197 /**
3198  *      skb_padto       - pad an skbuff up to a minimal size
3199  *      @skb: buffer to pad
3200  *      @len: minimal length
3201  *
3202  *      Pads up a buffer to ensure the trailing bytes exist and are
3203  *      blanked. If the buffer already contains sufficient data it
3204  *      is untouched. Otherwise it is extended. Returns zero on
3205  *      success. The skb is freed on error.
3206  */
3207 static inline int skb_padto(struct sk_buff *skb, unsigned int len)
3208 {
3209         unsigned int size = skb->len;
3210         if (likely(size >= len))
3211                 return 0;
3212         return skb_pad(skb, len - size);
3213 }
3214 
3215 /**
3216  *      __skb_put_padto - increase size and pad an skbuff up to a minimal size
3217  *      @skb: buffer to pad
3218  *      @len: minimal length
3219  *      @free_on_error: free buffer on error
3220  *
3221  *      Pads up a buffer to ensure the trailing bytes exist and are
3222  *      blanked. If the buffer already contains sufficient data it
3223  *      is untouched. Otherwise it is extended. Returns zero on
3224  *      success. The skb is freed on error if @free_on_error is true.
3225  */
3226 static inline int __must_check __skb_put_padto(struct sk_buff *skb,
3227                                                unsigned int len,
3228                                                bool free_on_error)
3229 {
3230         unsigned int size = skb->len;
3231 
3232         if (unlikely(size < len)) {
3233                 len -= size;
3234                 if (__skb_pad(skb, len, free_on_error))
3235                         return -ENOMEM;
3236                 __skb_put(skb, len);
3237         }
3238         return 0;
3239 }
3240 
3241 /**
3242  *      skb_put_padto - increase size and pad an skbuff up to a minimal size
3243  *      @skb: buffer to pad
3244  *      @len: minimal length
3245  *
3246  *      Pads up a buffer to ensure the trailing bytes exist and are
3247  *      blanked. If the buffer already contains sufficient data it
3248  *      is untouched. Otherwise it is extended. Returns zero on
3249  *      success. The skb is freed on error.
3250  */
3251 static inline int __must_check skb_put_padto(struct sk_buff *skb, unsigned int len)
3252 {
3253         return __skb_put_padto(skb, len, true);
3254 }
3255 
3256 static inline int skb_add_data(struct sk_buff *skb,
3257                                struct iov_iter *from, int copy)
3258 {
3259         const int off = skb->len;
3260 
3261         if (skb->ip_summed == CHECKSUM_NONE) {
3262                 __wsum csum = 0;
3263                 if (csum_and_copy_from_iter_full(skb_put(skb, copy), copy,
3264                                                  &csum, from)) {
3265                         skb->csum = csum_block_add(skb->csum, csum, off);
3266                         return 0;
3267                 }
3268         } else if (copy_from_iter_full(skb_put(skb, copy), copy, from))
3269                 return 0;
3270 
3271         __skb_trim(skb, off);
3272         return -EFAULT;
3273 }
3274 
3275 static inline bool skb_can_coalesce(struct sk_buff *skb, int i,
3276                                     const struct page *page, int off)
3277 {
3278         if (skb_zcopy(skb))
3279                 return false;
3280         if (i) {
3281                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
3282 
3283                 return page == skb_frag_page(frag) &&
3284                        off == skb_frag_off(frag) + skb_frag_size(frag);
3285         }
3286         return false;
3287 }
3288 
3289 static inline int __skb_linearize(struct sk_buff *skb)
3290 {
3291         return __pskb_pull_tail(skb, skb->data_len) ? 0 : -ENOMEM;
3292 }
3293 
3294 /**
3295  *      skb_linearize - convert paged skb to linear one
3296  *      @skb: buffer to linarize
3297  *
3298  *      If there is no free memory -ENOMEM is returned, otherwise zero
3299  *      is returned and the old skb data released.
3300  */
3301 static inline int skb_linearize(struct sk_buff *skb)
3302 {
3303         return skb_is_nonlinear(skb) ? __skb_linearize(skb) : 0;
3304 }
3305 
3306 /**
3307  * skb_has_shared_frag - can any frag be overwritten
3308  * @skb: buffer to test
3309  *
3310  * Return true if the skb has at least one frag that might be modified
3311  * by an external entity (as in vmsplice()/sendfile())
3312  */
3313 static inline bool skb_has_shared_frag(const struct sk_buff *skb)
3314 {
3315         return skb_is_nonlinear(skb) &&
3316                skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG;
3317 }
3318 
3319 /**
3320  *      skb_linearize_cow - make sure skb is linear and writable
3321  *      @skb: buffer to process
3322  *
3323  *      If there is no free memory -ENOMEM is returned, otherwise zero
3324  *      is returned and the old skb data released.
3325  */
3326 static inline int skb_linearize_cow(struct sk_buff *skb)
3327 {
3328         return skb_is_nonlinear(skb) || skb_cloned(skb) ?
3329                __skb_linearize(skb) : 0;
3330 }
3331 
3332 static __always_inline void
3333 __skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len,
3334                      unsigned int off)
3335 {
3336         if (skb->ip_summed == CHECKSUM_COMPLETE)
3337                 skb->csum = csum_block_sub(skb->csum,
3338                                            csum_partial(start, len, 0), off);
3339         else if (skb->ip_summed == CHECKSUM_PARTIAL &&
3340                  skb_checksum_start_offset(skb) < 0)
3341                 skb->ip_summed = CHECKSUM_NONE;
3342 }
3343 
3344 /**
3345  *      skb_postpull_rcsum - update checksum for received skb after pull
3346  *      @skb: buffer to update
3347  *      @start: start of data before pull
3348  *      @len: length of data pulled
3349  *
3350  *      After doing a pull on a received packet, you need to call this to
3351  *      update the CHECKSUM_COMPLETE checksum, or set ip_summed to
3352  *      CHECKSUM_NONE so that it can be recomputed from scratch.
3353  */
3354 static inline void skb_postpull_rcsum(struct sk_buff *skb,
3355                                       const void *start, unsigned int len)
3356 {
3357         __skb_postpull_rcsum(skb, start, len, 0);
3358 }
3359 
3360 static __always_inline void
3361 __skb_postpush_rcsum(struct sk_buff *skb, const void *start, unsigned int len,
3362                      unsigned int off)
3363 {
3364         if (skb->ip_summed == CHECKSUM_COMPLETE)
3365                 skb->csum = csum_block_add(skb->csum,
3366                                            csum_partial(start, len, 0), off);
3367 }
3368 
3369 /**
3370  *      skb_postpush_rcsum - update checksum for received skb after push
3371  *      @skb: buffer to update
3372  *      @start: start of data after push
3373  *      @len: length of data pushed
3374  *
3375  *      After doing a push on a received packet, you need to call this to
3376  *      update the CHECKSUM_COMPLETE checksum.
3377  */
3378 static inline void skb_postpush_rcsum(struct sk_buff *skb,
3379                                       const void *start, unsigned int len)
3380 {
3381         __skb_postpush_rcsum(skb, start, len, 0);
3382 }
3383 
3384 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len);
3385 
3386 /**
3387  *      skb_push_rcsum - push skb and update receive checksum
3388  *      @skb: buffer to update
3389  *      @len: length of data pulled
3390  *
3391  *      This function performs an skb_push on the packet and updates
3392  *      the CHECKSUM_COMPLETE checksum.  It should be used on
3393  *      receive path processing instead of skb_push unless you know
3394  *      that the checksum difference is zero (e.g., a valid IP header)
3395  *      or you are setting ip_summed to CHECKSUM_NONE.
3396  */
3397 static inline void *skb_push_rcsum(struct sk_buff *skb, unsigned int len)
3398 {
3399         skb_push(skb, len);
3400         skb_postpush_rcsum(skb, skb->data, len);
3401         return skb->data;
3402 }
3403 
3404 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len);
3405 /**
3406  *      pskb_trim_rcsum - trim received skb and update checksum
3407  *      @skb: buffer to trim
3408  *      @len: new length
3409  *
3410  *      This is exactly the same as pskb_trim except that it ensures the
3411  *      checksum of received packets are still valid after the operation.
3412  *      It can change skb pointers.
3413  */
3414 
3415 static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len)
3416 {
3417         if (likely(len >= skb->len))
3418                 return 0;
3419         return pskb_trim_rcsum_slow(skb, len);
3420 }
3421 
3422 static inline int __skb_trim_rcsum(struct sk_buff *skb, unsigned int len)
3423 {
3424         if (skb->ip_summed == CHECKSUM_COMPLETE)
3425                 skb->ip_summed = CHECKSUM_NONE;
3426         __skb_trim(skb, len);
3427         return 0;
3428 }
3429 
3430 static inline int __skb_grow_rcsum(struct sk_buff *skb, unsigned int len)
3431 {
3432         if (skb->ip_summed == CHECKSUM_COMPLETE)
3433                 skb->ip_summed = CHECKSUM_NONE;
3434         return __skb_grow(skb, len);
3435 }
3436 
3437 #define rb_to_skb(rb) rb_entry_safe(rb, struct sk_buff, rbnode)
3438 #define skb_rb_first(root) rb_to_skb(rb_first(root))
3439 #define skb_rb_last(root)  rb_to_skb(rb_last(root))
3440 #define skb_rb_next(skb)   rb_to_skb(rb_next(&(skb)->rbnode))
3441 #define skb_rb_prev(skb)   rb_to_skb(rb_prev(&(skb)->rbnode))
3442 
3443 #define skb_queue_walk(queue, skb) \
3444                 for (skb = (queue)->next;                                       \
3445                      skb != (struct sk_buff *)(queue);                          \
3446                      skb = skb->next)
3447 
3448 #define skb_queue_walk_safe(queue, skb, tmp)                                    \
3449                 for (skb = (queue)->next, tmp = skb->next;                      \
3450                      skb != (struct sk_buff *)(queue);                          \
3451                      skb = tmp, tmp = skb->next)
3452 
3453 #define skb_queue_walk_from(queue, skb)                                         \
3454                 for (; skb != (struct sk_buff *)(queue);                        \
3455                      skb = skb->next)
3456 
3457 #define skb_rbtree_walk(skb, root)                                              \
3458                 for (skb = skb_rb_first(root); skb != NULL;                     \
3459                      skb = skb_rb_next(skb))
3460 
3461 #define skb_rbtree_walk_from(skb)                                               \
3462                 for (; skb != NULL;                                             \
3463                      skb = skb_rb_next(skb))
3464 
3465 #define skb_rbtree_walk_from_safe(skb, tmp)                                     \
3466                 for (; tmp = skb ? skb_rb_next(skb) : NULL, (skb != NULL);      \
3467                      skb = tmp)
3468 
3469 #define skb_queue_walk_from_safe(queue, skb, tmp)                               \
3470                 for (tmp = skb->next;                                           \
3471                      skb != (struct sk_buff *)(queue);                          \
3472                      skb = tmp, tmp = skb->next)
3473 
3474 #define skb_queue_reverse_walk(queue, skb) \
3475                 for (skb = (queue)->prev;                                       \
3476                      skb != (struct sk_buff *)(queue);                          \
3477                      skb = skb->prev)
3478 
3479 #define skb_queue_reverse_walk_safe(queue, skb, tmp)                            \
3480                 for (skb = (queue)->prev, tmp = skb->prev;                      \
3481                      skb != (struct sk_buff *)(queue);                          \
3482                      skb = tmp, tmp = skb->prev)
3483 
3484 #define skb_queue_reverse_walk_from_safe(queue, skb, tmp)                       \
3485                 for (tmp = skb->prev;                                           \
3486                      skb != (struct sk_buff *)(queue);                          \
3487                      skb = tmp, tmp = skb->prev)
3488 
3489 static inline bool skb_has_frag_list(const struct sk_buff *skb)
3490 {
3491         return skb_shinfo(skb)->frag_list != NULL;
3492 }
3493 
3494 static inline void skb_frag_list_init(struct sk_buff *skb)
3495 {
3496         skb_shinfo(skb)->frag_list = NULL;
3497 }
3498 
3499 #define skb_walk_frags(skb, iter)       \
3500         for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next)
3501 
3502 
3503 int __skb_wait_for_more_packets(struct sock *sk, struct sk_buff_head *queue,
3504                                 int *err, long *timeo_p,
3505                                 const struct sk_buff *skb);
3506 struct sk_buff *__skb_try_recv_from_queue(struct sock *sk,
3507                                           struct sk_buff_head *queue,
3508                                           unsigned int flags,
3509                                           int *off, int *err,
3510                                           struct sk_buff **last);
3511 struct sk_buff *__skb_try_recv_datagram(struct sock *sk,
3512                                         struct sk_buff_head *queue,
3513                                         unsigned int flags, int *off, int *err,
3514                                         struct sk_buff **last);
3515 struct sk_buff *__skb_recv_datagram(struct sock *sk,
3516                                     struct sk_buff_head *sk_queue,
3517                                     unsigned int flags, int *off, int *err);
3518 struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned flags, int noblock,
3519                                   int *err);
3520 __poll_t datagram_poll(struct file *file, struct socket *sock,
3521                            struct poll_table_struct *wait);
3522 int skb_copy_datagram_iter(const struct sk_buff *from, int offset,
3523                            struct iov_iter *to, int size);
3524 static inline int skb_copy_datagram_msg(const struct sk_buff *from, int offset,
3525                                         struct msghdr *msg, int size)
3526 {
3527         return skb_copy_datagram_iter(from, offset, &msg->msg_iter, size);
3528 }
3529 int skb_copy_and_csum_datagram_msg(struct sk_buff *skb, int hlen,
3530                                    struct msghdr *msg);
3531 int skb_copy_and_hash_datagram_iter(const struct sk_buff *skb, int offset,
3532                            struct iov_iter *to, int len,
3533                            struct ahash_request *hash);
3534 int skb_copy_datagram_from_iter(struct sk_buff *skb, int offset,
3535                                  struct iov_iter *from, int len);
3536 int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *frm);
3537 void skb_free_datagram(struct sock *sk, struct sk_buff *skb);
3538 void __skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb, int len);
3539 static inline void skb_free_datagram_locked(struct sock *sk,
3540                                             struct sk_buff *skb)
3541 {
3542         __skb_free_datagram_locked(sk, skb, 0);
3543 }
3544 int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags);
3545 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len);
3546 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len);
3547 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, u8 *to,
3548                               int len, __wsum csum);
3549 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
3550                     struct pipe_inode_info *pipe, unsigned int len,
3551                     unsigned int flags);
3552 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
3553                          int len);
3554 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
3555 unsigned int skb_zerocopy_headlen(const struct sk_buff *from);
3556 int skb_zerocopy(struct sk_buff *to, struct sk_buff *from,
3557                  int len, int hlen);
3558 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len);
3559 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen);
3560 void skb_scrub_packet(struct sk_buff *skb, bool xnet);
3561 bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu);
3562 bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len);
3563 struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features);
3564 struct sk_buff *skb_segment_list(struct sk_buff *skb, netdev_features_t features,
3565                                  unsigned int offset);
3566 struct sk_buff *skb_vlan_untag(struct sk_buff *skb);
3567 int skb_ensure_writable(struct sk_buff *skb, int write_len);
3568 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci);
3569 int skb_vlan_pop(struct sk_buff *skb);
3570 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci);
3571 int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
3572                   int mac_len, bool ethernet);
3573 int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
3574                  bool ethernet);
3575 int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse);
3576 int skb_mpls_dec_ttl(struct sk_buff *skb);
3577 struct sk_buff *pskb_extract(struct sk_buff *skb, int off, int to_copy,
3578                              gfp_t gfp);
3579 
3580 static inline int memcpy_from_msg(void *data, struct msghdr *msg, int len)
3581 {
3582         return copy_from_iter_full(data, len, &msg->msg_iter) ? 0 : -EFAULT;
3583 }
3584 
3585 static inline int memcpy_to_msg(struct msghdr *msg, void *data, int len)
3586 {
3587         return copy_to_iter(data, len, &msg->msg_iter) == len ? 0 : -EFAULT;
3588 }
3589 
3590 struct skb_checksum_ops {
3591         __wsum (*update)(const void *mem, int len, __wsum wsum);
3592         __wsum (*combine)(__wsum csum, __wsum csum2, int offset, int len);
3593 };
3594 
3595 extern const struct skb_checksum_ops *crc32c_csum_stub __read_mostly;
3596 
3597 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
3598                       __wsum csum, const struct skb_checksum_ops *ops);
3599 __wsum skb_checksum(const struct sk_buff *skb, int offset, int len,
3600                     __wsum csum);
3601 
3602 static inline void * __must_check
3603 __skb_header_pointer(const struct sk_buff *skb, int offset,
3604                      int len, void *data, int hlen, void *buffer)
3605 {
3606         if (hlen - offset >= len)
3607                 return data + offset;
3608 
3609         if (!skb ||
3610             skb_copy_bits(skb, offset, buffer, len) < 0)
3611                 return NULL;
3612 
3613         return buffer;
3614 }
3615 
3616 static inline void * __must_check
3617 skb_header_pointer(const struct sk_buff *skb, int offset, int len, void *buffer)
3618 {
3619         return __skb_header_pointer(skb, offset, len, skb->data,
3620                                     skb_headlen(skb), buffer);
3621 }
3622 
3623 /**
3624  *      skb_needs_linearize - check if we need to linearize a given skb
3625  *                            depending on the given device features.
3626  *      @skb: socket buffer to check
3627  *      @features: net device features
3628  *
3629  *      Returns true if either:
3630  *      1. skb has frag_list and the device doesn't support FRAGLIST, or
3631  *      2. skb is fragmented and the device does not support SG.
3632  */
3633 static inline bool skb_needs_linearize(struct sk_buff *skb,
3634                                        netdev_features_t features)
3635 {
3636         return skb_is_nonlinear(skb) &&
3637                ((skb_has_frag_list(skb) && !(features & NETIF_F_FRAGLIST)) ||
3638                 (skb_shinfo(skb)->nr_frags && !(features & NETIF_F_SG)));
3639 }
3640 
3641 static inline void skb_copy_from_linear_data(const struct sk_buff *skb,
3642                                              void *to,
3643                                              const unsigned int len)
3644 {
3645         memcpy(to, skb->data, len);
3646 }
3647 
3648 static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb,
3649                                                     const int offset, void *to,
3650                                                     const unsigned int len)
3651 {
3652         memcpy(to, skb->data + offset, len);
3653 }
3654 
3655 static inline void skb_copy_to_linear_data(struct sk_buff *skb,
3656                                            const void *from,
3657                                            const unsigned int len)
3658 {
3659         memcpy(skb->data, from, len);
3660 }
3661 
3662 static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb,
3663                                                   const int offset,
3664                                                   const void *from,
3665                                                   const unsigned int len)
3666 {
3667         memcpy(skb->data + offset, from, len);
3668 }
3669 
3670 void skb_init(void);
3671 
3672 static inline ktime_t skb_get_ktime(const struct sk_buff *skb)
3673 {
3674         return skb->tstamp;
3675 }
3676 
3677 /**
3678  *      skb_get_timestamp - get timestamp from a skb
3679  *      @skb: skb to get stamp from
3680  *      @stamp: pointer to struct __kernel_old_timeval to store stamp in
3681  *
3682  *      Timestamps are stored in the skb as offsets to a base timestamp.
3683  *      This function converts the offset back to a struct timeval and stores
3684  *      it in stamp.
3685  */
3686 static inline void skb_get_timestamp(const struct sk_buff *skb,
3687                                      struct __kernel_old_timeval *stamp)
3688 {
3689         *stamp = ns_to_kernel_old_timeval(skb->tstamp);
3690 }
3691 
3692 static inline void skb_get_new_timestamp(const struct sk_buff *skb,
3693                                          struct __kernel_sock_timeval *stamp)
3694 {
3695         struct timespec64 ts = ktime_to_timespec64(skb->tstamp);
3696 
3697         stamp->tv_sec = ts.tv_sec;
3698         stamp->tv_usec = ts.tv_nsec / 1000;
3699 }
3700 
3701 static inline void skb_get_timestampns(const struct sk_buff *skb,
3702                                        struct __kernel_old_timespec *stamp)
3703 {
3704         struct timespec64 ts = ktime_to_timespec64(skb->tstamp);
3705 
3706         stamp->tv_sec = ts.tv_sec;
3707         stamp->tv_nsec = ts.tv_nsec;
3708 }
3709 
3710 static inline void skb_get_new_timestampns(const struct sk_buff *skb,
3711                                            struct __kernel_timespec *stamp)
3712 {
3713         struct timespec64 ts = ktime_to_timespec64(skb->tstamp);
3714 
3715         stamp->tv_sec = ts.tv_sec;
3716         stamp->tv_nsec = ts.tv_nsec;
3717 }
3718 
3719 static inline void __net_timestamp(struct sk_buff *skb)
3720 {
3721         skb->tstamp = ktime_get_real();
3722 }
3723 
3724 static inline ktime_t net_timedelta(ktime_t t)
3725 {
3726         return ktime_sub(ktime_get_real(), t);
3727 }
3728 
3729 static inline ktime_t net_invalid_timestamp(void)
3730 {
3731         return 0;
3732 }
3733 
3734 static inline u8 skb_metadata_len(const struct sk_buff *skb)
3735 {
3736         return skb_shinfo(skb)->meta_len;
3737 }
3738 
3739 static inline void *skb_metadata_end(const struct sk_buff *skb)
3740 {
3741         return skb_mac_header(skb);
3742 }
3743 
3744 static inline bool __skb_metadata_differs(const struct sk_buff *skb_a,
3745                                           const struct sk_buff *skb_b,
3746                                           u8 meta_len)
3747 {
3748         const void *a = skb_metadata_end(skb_a);
3749         const void *b = skb_metadata_end(skb_b);
3750         /* Using more efficient varaiant than plain call to memcmp(). */
3751 #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64
3752         u64 diffs = 0;
3753 
3754         switch (meta_len) {
3755 #define __it(x, op) (x -= sizeof(u##op))
3756 #define __it_diff(a, b, op) (*(u##op *)__it(a, op)) ^ (*(u##op *)__it(b, op))
3757         case 32: diffs |= __it_diff(a, b, 64);
3758                 fallthrough;
3759         case 24: diffs |= __it_diff(a, b, 64);
3760                 fallthrough;
3761         case 16: diffs |= __it_diff(a, b, 64);
3762                 fallthrough;
3763         case  8: diffs |= __it_diff(a, b, 64);
3764                 break;
3765         case 28: diffs |= __it_diff(a, b, 64);
3766                 fallthrough;
3767         case 20: diffs |= __it_diff(a, b, 64);
3768                 fallthrough;
3769         case 12: diffs |= __it_diff(a, b, 64);
3770                 fallthrough;
3771         case  4: diffs |= __it_diff(a, b, 32);
3772                 break;
3773         }
3774         return diffs;
3775 #else
3776         return memcmp(a - meta_len, b - meta_len, meta_len);
3777 #endif
3778 }
3779 
3780 static inline bool skb_metadata_differs(const struct sk_buff *skb_a,
3781                                         const struct sk_buff *skb_b)
3782 {
3783         u8 len_a = skb_metadata_len(skb_a);
3784         u8 len_b = skb_metadata_len(skb_b);
3785 
3786         if (!(len_a | len_b))
3787                 return false;
3788 
3789         return len_a != len_b ?
3790                true : __skb_metadata_differs(skb_a, skb_b, len_a);
3791 }
3792 
3793 static inline void skb_metadata_set(struct sk_buff *skb, u8 meta_len)
3794 {
3795         skb_shinfo(skb)->meta_len = meta_len;
3796 }
3797 
3798 static inline void skb_metadata_clear(struct sk_buff *skb)
3799 {
3800         skb_metadata_set(skb, 0);
3801 }
3802 
3803 struct sk_buff *skb_clone_sk(struct sk_buff *skb);
3804 
3805 #ifdef CONFIG_NETWORK_PHY_TIMESTAMPING
3806 
3807 void skb_clone_tx_timestamp(struct sk_buff *skb);
3808 bool skb_defer_rx_timestamp(struct sk_buff *skb);
3809 
3810 #else /* CONFIG_NETWORK_PHY_TIMESTAMPING */
3811 
3812 static inline void skb_clone_tx_timestamp(struct sk_buff *skb)
3813 {
3814 }
3815 
3816 static inline bool skb_defer_rx_timestamp(struct sk_buff *skb)
3817 {
3818         return false;
3819 }
3820 
3821 #endif /* !CONFIG_NETWORK_PHY_TIMESTAMPING */
3822 
3823 /**
3824  * skb_complete_tx_timestamp() - deliver cloned skb with tx timestamps
3825  *
3826  * PHY drivers may accept clones of transmitted packets for
3827  * timestamping via their phy_driver.txtstamp method. These drivers
3828  * must call this function to return the skb back to the stack with a
3829  * timestamp.
3830  *
3831  * @skb: clone of the original outgoing packet
3832  * @hwtstamps: hardware time stamps
3833  *
3834  */
3835 void skb_complete_tx_timestamp(struct sk_buff *skb,
3836                                struct skb_shared_hwtstamps *hwtstamps);
3837 
3838 void __skb_tstamp_tx(struct sk_buff *orig_skb,
3839                      struct skb_shared_hwtstamps *hwtstamps,
3840                      struct sock *sk, int tstype);
3841 
3842 /**
3843  * skb_tstamp_tx - queue clone of skb with send time stamps
3844  * @orig_skb:   the original outgoing packet
3845  * @hwtstamps:  hardware time stamps, may be NULL if not available
3846  *
3847  * If the skb has a socket associated, then this function clones the
3848  * skb (thus sharing the actual data and optional structures), stores
3849  * the optional hardware time stamping information (if non NULL) or
3850  * generates a software time stamp (otherwise), then queues the clone
3851  * to the error queue of the socket.  Errors are silently ignored.
3852  */
3853 void skb_tstamp_tx(struct sk_buff *orig_skb,
3854                    struct skb_shared_hwtstamps *hwtstamps);
3855 
3856 /**
3857  * skb_tx_timestamp() - Driver hook for transmit timestamping
3858  *
3859  * Ethernet MAC Drivers should call this function in their hard_xmit()
3860  * function immediately before giving the sk_buff to the MAC hardware.
3861  *
3862  * Specifically, one should make absolutely sure that this function is
3863  * called before TX completion of this packet can trigger.  Otherwise
3864  * the packet could potentially already be freed.
3865  *
3866  * @skb: A socket buffer.
3867  */
3868 static inline void skb_tx_timestamp(struct sk_buff *skb)
3869 {
3870         skb_clone_tx_timestamp(skb);
3871         if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP)
3872                 skb_tstamp_tx(skb, NULL);
3873 }
3874 
3875 /**
3876  * skb_complete_wifi_ack - deliver skb with wifi status
3877  *
3878  * @skb: the original outgoing packet
3879  * @acked: ack status
3880  *
3881  */
3882 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked);
3883 
3884 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len);
3885 __sum16 __skb_checksum_complete(struct sk_buff *skb);
3886 
3887 static inline int skb_csum_unnecessary(const struct sk_buff *skb)
3888 {
3889         return ((skb->ip_summed == CHECKSUM_UNNECESSARY) ||
3890                 skb->csum_valid ||
3891                 (skb->ip_summed == CHECKSUM_PARTIAL &&
3892                  skb_checksum_start_offset(skb) >= 0));
3893 }
3894 
3895 /**
3896  *      skb_checksum_complete - Calculate checksum of an entire packet
3897  *      @skb: packet to process
3898  *
3899  *      This function calculates the checksum over the entire packet plus
3900  *      the value of skb->csum.  The latter can be used to supply the
3901  *      checksum of a pseudo header as used by TCP/UDP.  It returns the
3902  *      checksum.
3903  *
3904  *      For protocols that contain complete checksums such as ICMP/TCP/UDP,
3905  *      this function can be used to verify that checksum on received
3906  *      packets.  In that case the function should return zero if the
3907  *      checksum is correct.  In particular, this function will return zero
3908  *      if skb->ip_summed is CHECKSUM_UNNECESSARY which indicates that the
3909  *      hardware has already verified the correctness of the checksum.
3910  */
3911 static inline __sum16 skb_checksum_complete(struct sk_buff *skb)
3912 {
3913         return skb_csum_unnecessary(skb) ?
3914                0 : __skb_checksum_complete(skb);
3915 }
3916 
3917 static inline void __skb_decr_checksum_unnecessary(struct sk_buff *skb)
3918 {
3919         if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
3920                 if (skb->csum_level == 0)
3921                         skb->ip_summed = CHECKSUM_NONE;
3922                 else
3923                         skb->csum_level--;
3924         }
3925 }
3926 
3927 static inline void __skb_incr_checksum_unnecessary(struct sk_buff *skb)
3928 {
3929         if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
3930                 if (skb->csum_level < SKB_MAX_CSUM_LEVEL)
3931                         skb->csum_level++;
3932         } else if (skb->ip_summed == CHECKSUM_NONE) {
3933                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3934                 skb->csum_level = 0;
3935         }
3936 }
3937 
3938 static inline void __skb_reset_checksum_unnecessary(struct sk_buff *skb)
3939 {
3940         if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
3941                 skb->ip_summed = CHECKSUM_NONE;
3942                 skb->csum_level = 0;
3943         }
3944 }
3945 
3946 /* Check if we need to perform checksum complete validation.
3947  *
3948  * Returns true if checksum complete is needed, false otherwise
3949  * (either checksum is unnecessary or zero checksum is allowed).
3950  */
3951 static inline bool __skb_checksum_validate_needed(struct sk_buff *skb,
3952                                                   bool zero_okay,
3953                                                   __sum16 check)
3954 {
3955         if (skb_csum_unnecessary(skb) || (zero_okay && !check)) {
3956                 skb->csum_valid = 1;
3957                 __skb_decr_checksum_unnecessary(skb);
3958                 return false;
3959         }
3960 
3961         return true;
3962 }
3963 
3964 /* For small packets <= CHECKSUM_BREAK perform checksum complete directly
3965  * in checksum_init.
3966  */
3967 #define CHECKSUM_BREAK 76
3968 
3969 /* Unset checksum-complete
3970  *
3971  * Unset checksum complete can be done when packet is being modified
3972  * (uncompressed for instance) and checksum-complete value is
3973  * invalidated.
3974  */
3975 static inline void skb_checksum_complete_unset(struct sk_buff *skb)
3976 {
3977         if (skb->ip_summed == CHECKSUM_COMPLETE)
3978                 skb->ip_summed = CHECKSUM_NONE;
3979 }
3980 
3981 /* Validate (init) checksum based on checksum complete.
3982  *
3983  * Return values:
3984  *   0: checksum is validated or try to in skb_checksum_complete. In the latter
3985  *      case the ip_summed will not be CHECKSUM_UNNECESSARY and the pseudo
3986  *      checksum is stored in skb->csum for use in __skb_checksum_complete
3987  *   non-zero: value of invalid checksum
3988  *
3989  */
3990 static inline __sum16 __skb_checksum_validate_complete(struct sk_buff *skb,
3991                                                        bool complete,
3992                                                        __wsum psum)
3993 {
3994         if (skb->ip_summed == CHECKSUM_COMPLETE) {
3995                 if (!csum_fold(csum_add(psum, skb->csum))) {
3996                         skb->csum_valid = 1;
3997                         return 0;
3998                 }
3999         }
4000 
4001         skb->csum = psum;
4002 
4003         if (complete || skb->len <= CHECKSUM_BREAK) {
4004                 __sum16 csum;
4005 
4006                 csum = __skb_checksum_complete(skb);
4007                 skb->csum_valid = !csum;
4008                 return csum;
4009         }
4010 
4011         return 0;
4012 }
4013 
4014 static inline __wsum null_compute_pseudo(struct sk_buff *skb, int proto)
4015 {
4016         return 0;
4017 }
4018 
4019 /* Perform checksum validate (init). Note that this is a macro since we only
4020  * want to calculate the pseudo header which is an input function if necessary.
4021  * First we try to validate without any computation (checksum unnecessary) and
4022  * then calculate based on checksum complete calling the function to compute
4023  * pseudo header.
4024  *
4025  * Return values:
4026  *   0: checksum is validated or try to in skb_checksum_complete
4027  *   non-zero: value of invalid checksum
4028  */
4029 #define __skb_checksum_validate(skb, proto, complete,                   \
4030                                 zero_okay, check, compute_pseudo)       \
4031 ({                                                                      \
4032         __sum16 __ret = 0;                                              \
4033         skb->csum_valid = 0;                                            \
4034         if (__skb_checksum_validate_needed(skb, zero_okay, check))      \
4035                 __ret = __skb_checksum_validate_complete(skb,           \
4036                                 complete, compute_pseudo(skb, proto));  \
4037         __ret;                                                          \
4038 })
4039 
4040 #define skb_checksum_init(skb, proto, compute_pseudo)                   \
4041         __skb_checksum_validate(skb, proto, false, false, 0, compute_pseudo)
4042 
4043 #define skb_checksum_init_zero_check(skb, proto, check, compute_pseudo) \
4044         __skb_checksum_validate(skb, proto, false, true, check, compute_pseudo)
4045 
4046 #define skb_checksum_validate(skb, proto, compute_pseudo)               \
4047         __skb_checksum_validate(skb, proto, true, false, 0, compute_pseudo)
4048 
4049 #define skb_checksum_validate_zero_check(skb, proto, check,             \
4050                                          compute_pseudo)                \
4051         __skb_checksum_validate(skb, proto, true, true, check, compute_pseudo)
4052 
4053 #define skb_checksum_simple_validate(skb)                               \
4054         __skb_checksum_validate(skb, 0, true, false, 0, null_compute_pseudo)
4055 
4056 static inline bool __skb_checksum_convert_check(struct sk_buff *skb)
4057 {
4058         return (skb->ip_summed == CHECKSUM_NONE && skb->csum_valid);
4059 }
4060 
4061 static inline void __skb_checksum_convert(struct sk_buff *skb, __wsum pseudo)
4062 {
4063         skb->csum = ~pseudo;
4064         skb->ip_summed = CHECKSUM_COMPLETE;
4065 }
4066 
4067 #define skb_checksum_try_convert(skb, proto, compute_pseudo)    \
4068 do {                                                                    \
4069         if (__skb_checksum_convert_check(skb))                          \
4070                 __skb_checksum_convert(skb, compute_pseudo(skb, proto)); \
4071 } while (0)
4072 
4073 static inline void skb_remcsum_adjust_partial(struct sk_buff *skb, void *ptr,
4074                                               u16 start, u16 offset)
4075 {
4076         skb->ip_summed = CHECKSUM_PARTIAL;
4077         skb->csum_start = ((unsigned char *)ptr + start) - skb->head;
4078         skb->csum_offset = offset - start;
4079 }
4080 
4081 /* Update skbuf and packet to reflect the remote checksum offload operation.
4082  * When called, ptr indicates the starting point for skb->csum when
4083  * ip_summed is CHECKSUM_COMPLETE. If we need create checksum complete
4084  * here, skb_postpull_rcsum is done so skb->csum start is ptr.
4085  */
4086 static inline void skb_remcsum_process(struct sk_buff *skb, void *ptr,
4087                                        int start, int offset, bool nopartial)
4088 {
4089         __wsum delta;
4090 
4091         if (!nopartial) {
4092                 skb_remcsum_adjust_partial(skb, ptr, start, offset);
4093                 return;
4094         }
4095 
4096          if (unlikely(skb->ip_summed != CHECKSUM_COMPLETE)) {
4097                 __skb_checksum_complete(skb);
4098                 skb_postpull_rcsum(skb, skb->data, ptr - (void *)skb->data);
4099         }
4100 
4101         delta = remcsum_adjust(ptr, skb->csum, start, offset);
4102 
4103         /* Adjust skb->csum since we changed the packet */
4104         skb->csum = csum_add(skb->csum, delta);
4105 }
4106 
4107 static inline struct nf_conntrack *skb_nfct(const struct sk_buff *skb)
4108 {
4109 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
4110         return (void *)(skb->_nfct & NFCT_PTRMASK);
4111 #else
4112         return NULL;
4113 #endif
4114 }
4115 
4116 static inline unsigned long skb_get_nfct(const struct sk_buff *skb)
4117 {
4118 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
4119         return skb->_nfct;
4120 #else
4121         return 0UL;
4122 #endif
4123 }
4124 
4125 static inline void skb_set_nfct(struct sk_buff *skb, unsigned long nfct)
4126 {
4127 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
4128         skb->_nfct = nfct;
4129 #endif
4130 }
4131 
4132 #ifdef CONFIG_SKB_EXTENSIONS
4133 enum skb_ext_id {
4134 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
4135         SKB_EXT_BRIDGE_NF,
4136 #endif
4137 #ifdef CONFIG_XFRM
4138         SKB_EXT_SEC_PATH,
4139 #endif
4140 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
4141         TC_SKB_EXT,
4142 #endif
4143 #if IS_ENABLED(CONFIG_MPTCP)
4144         SKB_EXT_MPTCP,
4145 #endif
4146         SKB_EXT_NUM, /* must be last */
4147 };
4148 
4149 /**
4150  *      struct skb_ext - sk_buff extensions
4151  *      @refcnt: 1 on allocation, deallocated on 0
4152  *      @offset: offset to add to @data to obtain extension address
4153  *      @chunks: size currently allocated, stored in SKB_EXT_ALIGN_SHIFT units
4154  *      @data: start of extension data, variable sized
4155  *
4156  *      Note: offsets/lengths are stored in chunks of 8 bytes, this allows
4157  *      to use 'u8' types while allowing up to 2kb worth of extension data.
4158  */
4159 struct skb_ext {
4160         refcount_t refcnt;
4161         u8 offset[SKB_EXT_NUM]; /* in chunks of 8 bytes */
4162         u8 chunks;              /* same */
4163         char data[] __aligned(8);
4164 };
4165 
4166 struct skb_ext *__skb_ext_alloc(gfp_t flags);
4167 void *__skb_ext_set(struct sk_buff *skb, enum skb_ext_id id,
4168                     struct skb_ext *ext);
4169 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id);
4170 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id);
4171 void __skb_ext_put(struct skb_ext *ext);
4172 
4173 static inline void skb_ext_put(struct sk_buff *skb)
4174 {
4175         if (skb->active_extensions)
4176                 __skb_ext_put(skb->extensions);
4177 }
4178 
4179 static inline void __skb_ext_copy(struct sk_buff *dst,
4180                                   const struct sk_buff *src)
4181 {
4182         dst->active_extensions = src->active_extensions;
4183 
4184         if (src->active_extensions) {
4185                 struct skb_ext *ext = src->extensions;
4186 
4187                 refcount_inc(&ext->refcnt);
4188                 dst->extensions = ext;
4189         }
4190 }
4191 
4192 static inline void skb_ext_copy(struct sk_buff *dst, const struct sk_buff *src)
4193 {
4194         skb_ext_put(dst);
4195         __skb_ext_copy(dst, src);
4196 }
4197 
4198 static inline bool __skb_ext_exist(const struct skb_ext *ext, enum skb_ext_id i)
4199 {
4200         return !!ext->offset[i];
4201 }
4202 
4203 static inline bool skb_ext_exist(const struct sk_buff *skb, enum skb_ext_id id)
4204 {
4205         return skb->active_extensions & (1 << id);
4206 }
4207 
4208 static inline void skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
4209 {
4210         if (skb_ext_exist(skb, id))
4211                 __skb_ext_del(skb, id);
4212 }
4213 
4214 static inline void *skb_ext_find(const struct sk_buff *skb, enum skb_ext_id id)
4215 {
4216         if (skb_ext_exist(skb, id)) {
4217                 struct skb_ext *ext = skb->extensions;
4218 
4219                 return (void *)ext + (ext->offset[id] << 3);
4220         }
4221 
4222         return NULL;
4223 }
4224 
4225 static inline void skb_ext_reset(struct sk_buff *skb)
4226 {
4227         if (unlikely(skb->active_extensions)) {
4228                 __skb_ext_put(skb->extensions);
4229                 skb->active_extensions = 0;
4230         }
4231 }
4232 
4233 static inline bool skb_has_extensions(struct sk_buff *skb)
4234 {
4235         return unlikely(skb->active_extensions);
4236 }
4237 #else
4238 static inline void skb_ext_put(struct sk_buff *skb) {}
4239 static inline void skb_ext_reset(struct sk_buff *skb) {}
4240 static inline void skb_ext_del(struct sk_buff *skb, int unused) {}
4241 static inline void __skb_ext_copy(struct sk_buff *d, const struct sk_buff *s) {}
4242 static inline void skb_ext_copy(struct sk_buff *dst, const struct sk_buff *s) {}
4243 static inline bool skb_has_extensions(struct sk_buff *skb) { return false; }
4244 #endif /* CONFIG_SKB_EXTENSIONS */
4245 
4246 static inline void nf_reset_ct(struct sk_buff *skb)
4247 {
4248 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
4249         nf_conntrack_put(skb_nfct(skb));
4250         skb->_nfct = 0;
4251 #endif
4252 }
4253 
4254 static inline void nf_reset_trace(struct sk_buff *skb)
4255 {
4256 #if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || defined(CONFIG_NF_TABLES)
4257         skb->nf_trace = 0;
4258 #endif
4259 }
4260 
4261 static inline void ipvs_reset(struct sk_buff *skb)
4262 {
4263 #if IS_ENABLED(CONFIG_IP_VS)
4264         skb->ipvs_property = 0;
4265 #endif
4266 }
4267 
4268 /* Note: This doesn't put any conntrack info in dst. */
4269 static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src,
4270                              bool copy)
4271 {
4272 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
4273         dst->_nfct = src->_nfct;
4274         nf_conntrack_get(skb_nfct(src));
4275 #endif
4276 #if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || defined(CONFIG_NF_TABLES)
4277         if (copy)
4278                 dst->nf_trace = src->nf_trace;
4279 #endif
4280 }
4281 
4282 static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src)
4283 {
4284 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
4285         nf_conntrack_put(skb_nfct(dst));
4286 #endif
4287         __nf_copy(dst, src, true);
4288 }
4289 
4290 #ifdef CONFIG_NETWORK_SECMARK
4291 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
4292 {
4293         to->secmark = from->secmark;
4294 }
4295 
4296 static inline void skb_init_secmark(struct sk_buff *skb)
4297 {
4298         skb->secmark = 0;
4299 }
4300 #else
4301 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
4302 { }
4303 
4304 static inline void skb_init_secmark(struct sk_buff *skb)
4305 { }
4306 #endif
4307 
4308 static inline int secpath_exists(const struct sk_buff *skb)
4309 {
4310 #ifdef CONFIG_XFRM
4311         return skb_ext_exist(skb, SKB_EXT_SEC_PATH);
4312 #else
4313         return 0;
4314 #endif
4315 }
4316 
4317 static inline bool skb_irq_freeable(const struct sk_buff *skb)
4318 {
4319         return !skb->destructor &&
4320                 !secpath_exists(skb) &&
4321                 !skb_nfct(skb) &&
4322                 !skb->_skb_refdst &&
4323                 !skb_has_frag_list(skb);
4324 }
4325 
4326 static inline void skb_set_queue_mapping(struct sk_buff *skb, u16 queue_mapping)
4327 {
4328         skb->queue_mapping = queue_mapping;
4329 }
4330 
4331 static inline u16 skb_get_queue_mapping(const struct sk_buff *skb)
4332 {
4333         return skb->queue_mapping;
4334 }
4335 
4336 static inline void skb_copy_queue_mapping(struct sk_buff *to, const struct sk_buff *from)
4337 {
4338         to->queue_mapping = from->queue_mapping;
4339 }
4340 
4341 static inline void skb_record_rx_queue(struct sk_buff *skb, u16 rx_queue)
4342 {
4343         skb->queue_mapping = rx_queue + 1;
4344 }
4345 
4346 static inline u16 skb_get_rx_queue(const struct sk_buff *skb)
4347 {
4348         return skb->queue_mapping - 1;
4349 }
4350 
4351 static inline bool skb_rx_queue_recorded(const struct sk_buff *skb)
4352 {
4353         return skb->queue_mapping != 0;
4354 }
4355 
4356 static inline void skb_set_dst_pending_confirm(struct sk_buff *skb, u32 val)
4357 {
4358         skb->dst_pending_confirm = val;
4359 }
4360 
4361 static inline bool skb_get_dst_pending_confirm(const struct sk_buff *skb)
4362 {
4363         return skb->dst_pending_confirm != 0;
4364 }
4365 
4366 static inline struct sec_path *skb_sec_path(const struct sk_buff *skb)
4367 {
4368 #ifdef CONFIG_XFRM
4369         return skb_ext_find(skb, SKB_EXT_SEC_PATH);
4370 #else
4371         return NULL;
4372 #endif
4373 }
4374 
4375 /* Keeps track of mac header offset relative to skb->head.
4376  * It is useful for TSO of Tunneling protocol. e.g. GRE.
4377  * For non-tunnel skb it points to skb_mac_header() and for
4378  * tunnel skb it points to outer mac header.
4379  * Keeps track of level of encapsulation of network headers.
4380  */
4381 struct skb_gso_cb {
4382         union {
4383                 int     mac_offset;
4384                 int     data_offset;
4385         };
4386         int     encap_level;
4387         __wsum  csum;
4388         __u16   csum_start;
4389 };
4390 #define SKB_GSO_CB_OFFSET       32
4391 #define SKB_GSO_CB(skb) ((struct skb_gso_cb *)((skb)->cb + SKB_GSO_CB_OFFSET))
4392 
4393 static inline int skb_tnl_header_len(const struct sk_buff *inner_skb)
4394 {
4395         return (skb_mac_header(inner_skb) - inner_skb->head) -
4396                 SKB_GSO_CB(inner_skb)->mac_offset;
4397 }
4398 
4399 static inline int gso_pskb_expand_head(struct sk_buff *skb, int extra)
4400 {
4401         int new_headroom, headroom;
4402         int ret;
4403 
4404         headroom = skb_headroom(skb);
4405         ret = pskb_expand_head(skb, extra, 0, GFP_ATOMIC);
4406         if (ret)
4407                 return ret;
4408 
4409         new_headroom = skb_headroom(skb);
4410         SKB_GSO_CB(skb)->mac_offset += (new_headroom - headroom);
4411         return 0;
4412 }
4413 
4414 static inline void gso_reset_checksum(struct sk_buff *skb, __wsum res)
4415 {
4416         /* Do not update partial checksums if remote checksum is enabled. */
4417         if (skb->remcsum_offload)
4418                 return;
4419 
4420         SKB_GSO_CB(skb)->csum = res;
4421         SKB_GSO_CB(skb)->csum_start = skb_checksum_start(skb) - skb->head;
4422 }
4423 
4424 /* Compute the checksum for a gso segment. First compute the checksum value
4425  * from the start of transport header to SKB_GSO_CB(skb)->csum_start, and
4426  * then add in skb->csum (checksum from csum_start to end of packet).
4427  * skb->csum and csum_start are then updated to reflect the checksum of the
4428  * resultant packet starting from the transport header-- the resultant checksum
4429  * is in the res argument (i.e. normally zero or ~ of checksum of a pseudo
4430  * header.
4431  */
4432 static inline __sum16 gso_make_checksum(struct sk_buff *skb, __wsum res)
4433 {
4434         unsigned char *csum_start = skb_transport_header(skb);
4435         int plen = (skb->head + SKB_GSO_CB(skb)->csum_start) - csum_start;
4436         __wsum partial = SKB_GSO_CB(skb)->csum;
4437 
4438         SKB_GSO_CB(skb)->csum = res;
4439         SKB_GSO_CB(skb)->csum_start = csum_start - skb->head;
4440 
4441         return csum_fold(csum_partial(csum_start, plen, partial));
4442 }
4443 
4444 static inline bool skb_is_gso(const struct sk_buff *skb)
4445 {
4446         return skb_shinfo(skb)->gso_size;
4447 }
4448 
4449 /* Note: Should be called only if skb_is_gso(skb) is true */
4450 static inline bool skb_is_gso_v6(const struct sk_buff *skb)
4451 {
4452         return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6;
4453 }
4454 
4455 /* Note: Should be called only if skb_is_gso(skb) is true */
4456 static inline bool skb_is_gso_sctp(const struct sk_buff *skb)
4457 {
4458         return skb_shinfo(skb)->gso_type & SKB_GSO_SCTP;
4459 }
4460 
4461 /* Note: Should be called only if skb_is_gso(skb) is true */
4462 static inline bool skb_is_gso_tcp(const struct sk_buff *skb)
4463 {
4464         return skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6);
4465 }
4466 
4467 static inline void skb_gso_reset(struct sk_buff *skb)
4468 {
4469         skb_shinfo(skb)->gso_size = 0;
4470         skb_shinfo(skb)->gso_segs = 0;
4471         skb_shinfo(skb)->gso_type = 0;
4472 }
4473 
4474 static inline void skb_increase_gso_size(struct skb_shared_info *shinfo,
4475                                          u16 increment)
4476 {
4477         if (WARN_ON_ONCE(shinfo->gso_size == GSO_BY_FRAGS))
4478                 return;
4479         shinfo->gso_size += increment;
4480 }
4481 
4482 static inline void skb_decrease_gso_size(struct skb_shared_info *shinfo,
4483                                          u16 decrement)
4484 {
4485         if (WARN_ON_ONCE(shinfo->gso_size == GSO_BY_FRAGS))
4486                 return;
4487         shinfo->gso_size -= decrement;
4488 }
4489 
4490 void __skb_warn_lro_forwarding(const struct sk_buff *skb);
4491 
4492 static inline bool skb_warn_if_lro(const struct sk_buff *skb)
4493 {
4494         /* LRO sets gso_size but not gso_type, whereas if GSO is really
4495          * wanted then gso_type will be set. */
4496         const struct skb_shared_info *shinfo = skb_shinfo(skb);
4497 
4498         if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 &&
4499             unlikely(shinfo->gso_type == 0)) {
4500                 __skb_warn_lro_forwarding(skb);
4501                 return true;
4502         }
4503         return false;
4504 }
4505 
4506 static inline void skb_forward_csum(struct sk_buff *skb)
4507 {
4508         /* Unfortunately we don't support this one.  Any brave souls? */
4509         if (skb->ip_summed == CHECKSUM_COMPLETE)
4510                 skb->ip_summed = CHECKSUM_NONE;
4511 }
4512 
4513 /**
4514  * skb_checksum_none_assert - make sure skb ip_summed is CHECKSUM_NONE
4515  * @skb: skb to check
4516  *
4517  * fresh skbs have their ip_summed set to CHECKSUM_NONE.
4518  * Instead of forcing ip_summed to CHECKSUM_NONE, we can
4519  * use this helper, to document places where we make this assertion.
4520  */
4521 static inline void skb_checksum_none_assert(const struct sk_buff *skb)
4522 {
4523 #ifdef DEBUG
4524         BUG_ON(skb->ip_summed != CHECKSUM_NONE);
4525 #endif
4526 }
4527 
4528 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off);
4529 
4530 int skb_checksum_setup(struct sk_buff *skb, bool recalculate);
4531 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
4532                                      unsigned int transport_len,
4533                                      __sum16(*skb_chkf)(struct sk_buff *skb));
4534 
4535 /**
4536  * skb_head_is_locked - Determine if the skb->head is locked down
4537  * @skb: skb to check
4538  *
4539  * The head on skbs build around a head frag can be removed if they are
4540  * not cloned.  This function returns true if the skb head is locked down
4541  * due to either being allocated via kmalloc, or by being a clone with
4542  * multiple references to the head.
4543  */
4544 static inline bool skb_head_is_locked(const struct sk_buff *skb)
4545 {
4546         return !skb->head_frag || skb_cloned(skb);
4547 }
4548 
4549 /* Local Checksum Offload.
4550  * Compute outer checksum based on the assumption that the
4551  * inner checksum will be offloaded later.
4552  * See Documentation/networking/checksum-offloads.rst for
4553  * explanation of how this works.
4554  * Fill in outer checksum adjustment (e.g. with sum of outer
4555  * pseudo-header) before calling.
4556  * Also ensure that inner checksum is in linear data area.
4557  */
4558 static inline __wsum lco_csum(struct sk_buff *skb)
4559 {
4560         unsigned char *csum_start = skb_checksum_start(skb);
4561         unsigned char *l4_hdr = skb_transport_header(skb);
4562         __wsum partial;
4563 
4564         /* Start with complement of inner checksum adjustment */
4565         partial = ~csum_unfold(*(__force __sum16 *)(csum_start +
4566                                                     skb->csum_offset));
4567 
4568         /* Add in checksum of our headers (incl. outer checksum
4569          * adjustment filled in by caller) and return result.
4570          */
4571         return csum_partial(l4_hdr, csum_start - l4_hdr, partial);
4572 }
4573 
4574 static inline bool skb_is_redirected(const struct sk_buff *skb)
4575 {
4576 #ifdef CONFIG_NET_REDIRECT
4577         return skb->redirected;
4578 #else
4579         return false;
4580 #endif
4581 }
4582 
4583 static inline void skb_set_redirected(struct sk_buff *skb, bool from_ingress)
4584 {
4585 #ifdef CONFIG_NET_REDIRECT
4586         skb->redirected = 1;
4587         skb->from_ingress = from_ingress;
4588         if (skb->from_ingress)
4589                 skb->tstamp = 0;
4590 #endif
4591 }
4592 
4593 static inline void skb_reset_redirect(struct sk_buff *skb)
4594 {
4595 #ifdef CONFIG_NET_REDIRECT
4596         skb->redirected = 0;
4597 #endif
4598 }
4599 
4600 #endif  /* __KERNEL__ */
4601 #endif  /* _LINUX_SKBUFF_H */
4602 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp