~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/include/linux/skbuff.h

Version: ~ [ linux-6.1-rc7 ] ~ [ linux-6.0.10 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.80 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.156 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.225 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.267 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.300 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.334 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* SPDX-License-Identifier: GPL-2.0-or-later */
  2 /*
  3  *      Definitions for the 'struct sk_buff' memory handlers.
  4  *
  5  *      Authors:
  6  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  7  *              Florian La Roche, <rzsfl@rz.uni-sb.de>
  8  */
  9 
 10 #ifndef _LINUX_SKBUFF_H
 11 #define _LINUX_SKBUFF_H
 12 
 13 #include <linux/kernel.h>
 14 #include <linux/compiler.h>
 15 #include <linux/time.h>
 16 #include <linux/bug.h>
 17 #include <linux/cache.h>
 18 #include <linux/rbtree.h>
 19 #include <linux/socket.h>
 20 #include <linux/refcount.h>
 21 
 22 #include <linux/atomic.h>
 23 #include <asm/types.h>
 24 #include <linux/spinlock.h>
 25 #include <linux/net.h>
 26 #include <linux/textsearch.h>
 27 #include <net/checksum.h>
 28 #include <linux/rcupdate.h>
 29 #include <linux/hrtimer.h>
 30 #include <linux/dma-mapping.h>
 31 #include <linux/netdev_features.h>
 32 #include <linux/sched.h>
 33 #include <linux/sched/clock.h>
 34 #include <net/flow_dissector.h>
 35 #include <linux/splice.h>
 36 #include <linux/in6.h>
 37 #include <linux/if_packet.h>
 38 #include <net/flow.h>
 39 
 40 /* The interface for checksum offload between the stack and networking drivers
 41  * is as follows...
 42  *
 43  * A. IP checksum related features
 44  *
 45  * Drivers advertise checksum offload capabilities in the features of a device.
 46  * From the stack's point of view these are capabilities offered by the driver,
 47  * a driver typically only advertises features that it is capable of offloading
 48  * to its device.
 49  *
 50  * The checksum related features are:
 51  *
 52  *      NETIF_F_HW_CSUM - The driver (or its device) is able to compute one
 53  *                        IP (one's complement) checksum for any combination
 54  *                        of protocols or protocol layering. The checksum is
 55  *                        computed and set in a packet per the CHECKSUM_PARTIAL
 56  *                        interface (see below).
 57  *
 58  *      NETIF_F_IP_CSUM - Driver (device) is only able to checksum plain
 59  *                        TCP or UDP packets over IPv4. These are specifically
 60  *                        unencapsulated packets of the form IPv4|TCP or
 61  *                        IPv4|UDP where the Protocol field in the IPv4 header
 62  *                        is TCP or UDP. The IPv4 header may contain IP options
 63  *                        This feature cannot be set in features for a device
 64  *                        with NETIF_F_HW_CSUM also set. This feature is being
 65  *                        DEPRECATED (see below).
 66  *
 67  *      NETIF_F_IPV6_CSUM - Driver (device) is only able to checksum plain
 68  *                        TCP or UDP packets over IPv6. These are specifically
 69  *                        unencapsulated packets of the form IPv6|TCP or
 70  *                        IPv4|UDP where the Next Header field in the IPv6
 71  *                        header is either TCP or UDP. IPv6 extension headers
 72  *                        are not supported with this feature. This feature
 73  *                        cannot be set in features for a device with
 74  *                        NETIF_F_HW_CSUM also set. This feature is being
 75  *                        DEPRECATED (see below).
 76  *
 77  *      NETIF_F_RXCSUM - Driver (device) performs receive checksum offload.
 78  *                       This flag is used only used to disable the RX checksum
 79  *                       feature for a device. The stack will accept receive
 80  *                       checksum indication in packets received on a device
 81  *                       regardless of whether NETIF_F_RXCSUM is set.
 82  *
 83  * B. Checksumming of received packets by device. Indication of checksum
 84  *    verification is in set skb->ip_summed. Possible values are:
 85  *
 86  * CHECKSUM_NONE:
 87  *
 88  *   Device did not checksum this packet e.g. due to lack of capabilities.
 89  *   The packet contains full (though not verified) checksum in packet but
 90  *   not in skb->csum. Thus, skb->csum is undefined in this case.
 91  *
 92  * CHECKSUM_UNNECESSARY:
 93  *
 94  *   The hardware you're dealing with doesn't calculate the full checksum
 95  *   (as in CHECKSUM_COMPLETE), but it does parse headers and verify checksums
 96  *   for specific protocols. For such packets it will set CHECKSUM_UNNECESSARY
 97  *   if their checksums are okay. skb->csum is still undefined in this case
 98  *   though. A driver or device must never modify the checksum field in the
 99  *   packet even if checksum is verified.
100  *
101  *   CHECKSUM_UNNECESSARY is applicable to following protocols:
102  *     TCP: IPv6 and IPv4.
103  *     UDP: IPv4 and IPv6. A device may apply CHECKSUM_UNNECESSARY to a
104  *       zero UDP checksum for either IPv4 or IPv6, the networking stack
105  *       may perform further validation in this case.
106  *     GRE: only if the checksum is present in the header.
107  *     SCTP: indicates the CRC in SCTP header has been validated.
108  *     FCOE: indicates the CRC in FC frame has been validated.
109  *
110  *   skb->csum_level indicates the number of consecutive checksums found in
111  *   the packet minus one that have been verified as CHECKSUM_UNNECESSARY.
112  *   For instance if a device receives an IPv6->UDP->GRE->IPv4->TCP packet
113  *   and a device is able to verify the checksums for UDP (possibly zero),
114  *   GRE (checksum flag is set), and TCP-- skb->csum_level would be set to
115  *   two. If the device were only able to verify the UDP checksum and not
116  *   GRE, either because it doesn't support GRE checksum of because GRE
117  *   checksum is bad, skb->csum_level would be set to zero (TCP checksum is
118  *   not considered in this case).
119  *
120  * CHECKSUM_COMPLETE:
121  *
122  *   This is the most generic way. The device supplied checksum of the _whole_
123  *   packet as seen by netif_rx() and fills out in skb->csum. Meaning, the
124  *   hardware doesn't need to parse L3/L4 headers to implement this.
125  *
126  *   Notes:
127  *   - Even if device supports only some protocols, but is able to produce
128  *     skb->csum, it MUST use CHECKSUM_COMPLETE, not CHECKSUM_UNNECESSARY.
129  *   - CHECKSUM_COMPLETE is not applicable to SCTP and FCoE protocols.
130  *
131  * CHECKSUM_PARTIAL:
132  *
133  *   A checksum is set up to be offloaded to a device as described in the
134  *   output description for CHECKSUM_PARTIAL. This may occur on a packet
135  *   received directly from another Linux OS, e.g., a virtualized Linux kernel
136  *   on the same host, or it may be set in the input path in GRO or remote
137  *   checksum offload. For the purposes of checksum verification, the checksum
138  *   referred to by skb->csum_start + skb->csum_offset and any preceding
139  *   checksums in the packet are considered verified. Any checksums in the
140  *   packet that are after the checksum being offloaded are not considered to
141  *   be verified.
142  *
143  * C. Checksumming on transmit for non-GSO. The stack requests checksum offload
144  *    in the skb->ip_summed for a packet. Values are:
145  *
146  * CHECKSUM_PARTIAL:
147  *
148  *   The driver is required to checksum the packet as seen by hard_start_xmit()
149  *   from skb->csum_start up to the end, and to record/write the checksum at
150  *   offset skb->csum_start + skb->csum_offset. A driver may verify that the
151  *   csum_start and csum_offset values are valid values given the length and
152  *   offset of the packet, however they should not attempt to validate that the
153  *   checksum refers to a legitimate transport layer checksum-- it is the
154  *   purview of the stack to validate that csum_start and csum_offset are set
155  *   correctly.
156  *
157  *   When the stack requests checksum offload for a packet, the driver MUST
158  *   ensure that the checksum is set correctly. A driver can either offload the
159  *   checksum calculation to the device, or call skb_checksum_help (in the case
160  *   that the device does not support offload for a particular checksum).
161  *
162  *   NETIF_F_IP_CSUM and NETIF_F_IPV6_CSUM are being deprecated in favor of
163  *   NETIF_F_HW_CSUM. New devices should use NETIF_F_HW_CSUM to indicate
164  *   checksum offload capability.
165  *   skb_csum_hwoffload_help() can be called to resolve CHECKSUM_PARTIAL based
166  *   on network device checksumming capabilities: if a packet does not match
167  *   them, skb_checksum_help or skb_crc32c_help (depending on the value of
168  *   csum_not_inet, see item D.) is called to resolve the checksum.
169  *
170  * CHECKSUM_NONE:
171  *
172  *   The skb was already checksummed by the protocol, or a checksum is not
173  *   required.
174  *
175  * CHECKSUM_UNNECESSARY:
176  *
177  *   This has the same meaning on as CHECKSUM_NONE for checksum offload on
178  *   output.
179  *
180  * CHECKSUM_COMPLETE:
181  *   Not used in checksum output. If a driver observes a packet with this value
182  *   set in skbuff, if should treat as CHECKSUM_NONE being set.
183  *
184  * D. Non-IP checksum (CRC) offloads
185  *
186  *   NETIF_F_SCTP_CRC - This feature indicates that a device is capable of
187  *     offloading the SCTP CRC in a packet. To perform this offload the stack
188  *     will set set csum_start and csum_offset accordingly, set ip_summed to
189  *     CHECKSUM_PARTIAL and set csum_not_inet to 1, to provide an indication in
190  *     the skbuff that the CHECKSUM_PARTIAL refers to CRC32c.
191  *     A driver that supports both IP checksum offload and SCTP CRC32c offload
192  *     must verify which offload is configured for a packet by testing the
193  *     value of skb->csum_not_inet; skb_crc32c_csum_help is provided to resolve
194  *     CHECKSUM_PARTIAL on skbs where csum_not_inet is set to 1.
195  *
196  *   NETIF_F_FCOE_CRC - This feature indicates that a device is capable of
197  *     offloading the FCOE CRC in a packet. To perform this offload the stack
198  *     will set ip_summed to CHECKSUM_PARTIAL and set csum_start and csum_offset
199  *     accordingly. Note the there is no indication in the skbuff that the
200  *     CHECKSUM_PARTIAL refers to an FCOE checksum, a driver that supports
201  *     both IP checksum offload and FCOE CRC offload must verify which offload
202  *     is configured for a packet presumably by inspecting packet headers.
203  *
204  * E. Checksumming on output with GSO.
205  *
206  * In the case of a GSO packet (skb_is_gso(skb) is true), checksum offload
207  * is implied by the SKB_GSO_* flags in gso_type. Most obviously, if the
208  * gso_type is SKB_GSO_TCPV4 or SKB_GSO_TCPV6, TCP checksum offload as
209  * part of the GSO operation is implied. If a checksum is being offloaded
210  * with GSO then ip_summed is CHECKSUM_PARTIAL, csum_start and csum_offset
211  * are set to refer to the outermost checksum being offload (two offloaded
212  * checksums are possible with UDP encapsulation).
213  */
214 
215 /* Don't change this without changing skb_csum_unnecessary! */
216 #define CHECKSUM_NONE           0
217 #define CHECKSUM_UNNECESSARY    1
218 #define CHECKSUM_COMPLETE       2
219 #define CHECKSUM_PARTIAL        3
220 
221 /* Maximum value in skb->csum_level */
222 #define SKB_MAX_CSUM_LEVEL      3
223 
224 #define SKB_DATA_ALIGN(X)       ALIGN(X, SMP_CACHE_BYTES)
225 #define SKB_WITH_OVERHEAD(X)    \
226         ((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
227 #define SKB_MAX_ORDER(X, ORDER) \
228         SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X))
229 #define SKB_MAX_HEAD(X)         (SKB_MAX_ORDER((X), 0))
230 #define SKB_MAX_ALLOC           (SKB_MAX_ORDER(0, 2))
231 
232 /* return minimum truesize of one skb containing X bytes of data */
233 #define SKB_TRUESIZE(X) ((X) +                                          \
234                          SKB_DATA_ALIGN(sizeof(struct sk_buff)) +       \
235                          SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
236 
237 struct net_device;
238 struct scatterlist;
239 struct pipe_inode_info;
240 struct iov_iter;
241 struct napi_struct;
242 struct bpf_prog;
243 union bpf_attr;
244 struct skb_ext;
245 
246 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
247 struct nf_conntrack {
248         atomic_t use;
249 };
250 #endif
251 
252 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
253 struct nf_bridge_info {
254         enum {
255                 BRNF_PROTO_UNCHANGED,
256                 BRNF_PROTO_8021Q,
257                 BRNF_PROTO_PPPOE
258         } orig_proto:8;
259         u8                      pkt_otherhost:1;
260         u8                      in_prerouting:1;
261         u8                      bridged_dnat:1;
262         __u16                   frag_max_size;
263         struct net_device       *physindev;
264 
265         /* always valid & non-NULL from FORWARD on, for physdev match */
266         struct net_device       *physoutdev;
267         union {
268                 /* prerouting: detect dnat in orig/reply direction */
269                 __be32          ipv4_daddr;
270                 struct in6_addr ipv6_daddr;
271 
272                 /* after prerouting + nat detected: store original source
273                  * mac since neigh resolution overwrites it, only used while
274                  * skb is out in neigh layer.
275                  */
276                 char neigh_header[8];
277         };
278 };
279 #endif
280 
281 struct sk_buff_head {
282         /* These two members must be first. */
283         struct sk_buff  *next;
284         struct sk_buff  *prev;
285 
286         __u32           qlen;
287         spinlock_t      lock;
288 };
289 
290 struct sk_buff;
291 
292 /* To allow 64K frame to be packed as single skb without frag_list we
293  * require 64K/PAGE_SIZE pages plus 1 additional page to allow for
294  * buffers which do not start on a page boundary.
295  *
296  * Since GRO uses frags we allocate at least 16 regardless of page
297  * size.
298  */
299 #if (65536/PAGE_SIZE + 1) < 16
300 #define MAX_SKB_FRAGS 16UL
301 #else
302 #define MAX_SKB_FRAGS (65536/PAGE_SIZE + 1)
303 #endif
304 extern int sysctl_max_skb_frags;
305 
306 /* Set skb_shinfo(skb)->gso_size to this in case you want skb_segment to
307  * segment using its current segmentation instead.
308  */
309 #define GSO_BY_FRAGS    0xFFFF
310 
311 typedef struct skb_frag_struct skb_frag_t;
312 
313 struct skb_frag_struct {
314         struct {
315                 struct page *p;
316         } page;
317 #if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
318         __u32 page_offset;
319         __u32 size;
320 #else
321         __u16 page_offset;
322         __u16 size;
323 #endif
324 };
325 
326 /**
327  * skb_frag_size - Returns the size of a skb fragment
328  * @frag: skb fragment
329  */
330 static inline unsigned int skb_frag_size(const skb_frag_t *frag)
331 {
332         return frag->size;
333 }
334 
335 /**
336  * skb_frag_size_set - Sets the size of a skb fragment
337  * @frag: skb fragment
338  * @size: size of fragment
339  */
340 static inline void skb_frag_size_set(skb_frag_t *frag, unsigned int size)
341 {
342         frag->size = size;
343 }
344 
345 /**
346  * skb_frag_size_add - Incrementes the size of a skb fragment by %delta
347  * @frag: skb fragment
348  * @delta: value to add
349  */
350 static inline void skb_frag_size_add(skb_frag_t *frag, int delta)
351 {
352         frag->size += delta;
353 }
354 
355 /**
356  * skb_frag_size_sub - Decrements the size of a skb fragment by %delta
357  * @frag: skb fragment
358  * @delta: value to subtract
359  */
360 static inline void skb_frag_size_sub(skb_frag_t *frag, int delta)
361 {
362         frag->size -= delta;
363 }
364 
365 /**
366  * skb_frag_must_loop - Test if %p is a high memory page
367  * @p: fragment's page
368  */
369 static inline bool skb_frag_must_loop(struct page *p)
370 {
371 #if defined(CONFIG_HIGHMEM)
372         if (PageHighMem(p))
373                 return true;
374 #endif
375         return false;
376 }
377 
378 /**
379  *      skb_frag_foreach_page - loop over pages in a fragment
380  *
381  *      @f:             skb frag to operate on
382  *      @f_off:         offset from start of f->page.p
383  *      @f_len:         length from f_off to loop over
384  *      @p:             (temp var) current page
385  *      @p_off:         (temp var) offset from start of current page,
386  *                                 non-zero only on first page.
387  *      @p_len:         (temp var) length in current page,
388  *                                 < PAGE_SIZE only on first and last page.
389  *      @copied:        (temp var) length so far, excluding current p_len.
390  *
391  *      A fragment can hold a compound page, in which case per-page
392  *      operations, notably kmap_atomic, must be called for each
393  *      regular page.
394  */
395 #define skb_frag_foreach_page(f, f_off, f_len, p, p_off, p_len, copied) \
396         for (p = skb_frag_page(f) + ((f_off) >> PAGE_SHIFT),            \
397              p_off = (f_off) & (PAGE_SIZE - 1),                         \
398              p_len = skb_frag_must_loop(p) ?                            \
399              min_t(u32, f_len, PAGE_SIZE - p_off) : f_len,              \
400              copied = 0;                                                \
401              copied < f_len;                                            \
402              copied += p_len, p++, p_off = 0,                           \
403              p_len = min_t(u32, f_len - copied, PAGE_SIZE))             \
404 
405 #define HAVE_HW_TIME_STAMP
406 
407 /**
408  * struct skb_shared_hwtstamps - hardware time stamps
409  * @hwtstamp:   hardware time stamp transformed into duration
410  *              since arbitrary point in time
411  *
412  * Software time stamps generated by ktime_get_real() are stored in
413  * skb->tstamp.
414  *
415  * hwtstamps can only be compared against other hwtstamps from
416  * the same device.
417  *
418  * This structure is attached to packets as part of the
419  * &skb_shared_info. Use skb_hwtstamps() to get a pointer.
420  */
421 struct skb_shared_hwtstamps {
422         ktime_t hwtstamp;
423 };
424 
425 /* Definitions for tx_flags in struct skb_shared_info */
426 enum {
427         /* generate hardware time stamp */
428         SKBTX_HW_TSTAMP = 1 << 0,
429 
430         /* generate software time stamp when queueing packet to NIC */
431         SKBTX_SW_TSTAMP = 1 << 1,
432 
433         /* device driver is going to provide hardware time stamp */
434         SKBTX_IN_PROGRESS = 1 << 2,
435 
436         /* device driver supports TX zero-copy buffers */
437         SKBTX_DEV_ZEROCOPY = 1 << 3,
438 
439         /* generate wifi status information (where possible) */
440         SKBTX_WIFI_STATUS = 1 << 4,
441 
442         /* This indicates at least one fragment might be overwritten
443          * (as in vmsplice(), sendfile() ...)
444          * If we need to compute a TX checksum, we'll need to copy
445          * all frags to avoid possible bad checksum
446          */
447         SKBTX_SHARED_FRAG = 1 << 5,
448 
449         /* generate software time stamp when entering packet scheduling */
450         SKBTX_SCHED_TSTAMP = 1 << 6,
451 };
452 
453 #define SKBTX_ZEROCOPY_FRAG     (SKBTX_DEV_ZEROCOPY | SKBTX_SHARED_FRAG)
454 #define SKBTX_ANY_SW_TSTAMP     (SKBTX_SW_TSTAMP    | \
455                                  SKBTX_SCHED_TSTAMP)
456 #define SKBTX_ANY_TSTAMP        (SKBTX_HW_TSTAMP | SKBTX_ANY_SW_TSTAMP)
457 
458 /*
459  * The callback notifies userspace to release buffers when skb DMA is done in
460  * lower device, the skb last reference should be 0 when calling this.
461  * The zerocopy_success argument is true if zero copy transmit occurred,
462  * false on data copy or out of memory error caused by data copy attempt.
463  * The ctx field is used to track device context.
464  * The desc field is used to track userspace buffer index.
465  */
466 struct ubuf_info {
467         void (*callback)(struct ubuf_info *, bool zerocopy_success);
468         union {
469                 struct {
470                         unsigned long desc;
471                         void *ctx;
472                 };
473                 struct {
474                         u32 id;
475                         u16 len;
476                         u16 zerocopy:1;
477                         u32 bytelen;
478                 };
479         };
480         refcount_t refcnt;
481 
482         struct mmpin {
483                 struct user_struct *user;
484                 unsigned int num_pg;
485         } mmp;
486 };
487 
488 #define skb_uarg(SKB)   ((struct ubuf_info *)(skb_shinfo(SKB)->destructor_arg))
489 
490 int mm_account_pinned_pages(struct mmpin *mmp, size_t size);
491 void mm_unaccount_pinned_pages(struct mmpin *mmp);
492 
493 struct ubuf_info *sock_zerocopy_alloc(struct sock *sk, size_t size);
494 struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size,
495                                         struct ubuf_info *uarg);
496 
497 static inline void sock_zerocopy_get(struct ubuf_info *uarg)
498 {
499         refcount_inc(&uarg->refcnt);
500 }
501 
502 void sock_zerocopy_put(struct ubuf_info *uarg);
503 void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref);
504 
505 void sock_zerocopy_callback(struct ubuf_info *uarg, bool success);
506 
507 int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len);
508 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
509                              struct msghdr *msg, int len,
510                              struct ubuf_info *uarg);
511 
512 /* This data is invariant across clones and lives at
513  * the end of the header data, ie. at skb->end.
514  */
515 struct skb_shared_info {
516         __u8            __unused;
517         __u8            meta_len;
518         __u8            nr_frags;
519         __u8            tx_flags;
520         unsigned short  gso_size;
521         /* Warning: this field is not always filled in (UFO)! */
522         unsigned short  gso_segs;
523         struct sk_buff  *frag_list;
524         struct skb_shared_hwtstamps hwtstamps;
525         unsigned int    gso_type;
526         u32             tskey;
527 
528         /*
529          * Warning : all fields before dataref are cleared in __alloc_skb()
530          */
531         atomic_t        dataref;
532 
533         /* Intermediate layers must ensure that destructor_arg
534          * remains valid until skb destructor */
535         void *          destructor_arg;
536 
537         /* must be last field, see pskb_expand_head() */
538         skb_frag_t      frags[MAX_SKB_FRAGS];
539 };
540 
541 /* We divide dataref into two halves.  The higher 16 bits hold references
542  * to the payload part of skb->data.  The lower 16 bits hold references to
543  * the entire skb->data.  A clone of a headerless skb holds the length of
544  * the header in skb->hdr_len.
545  *
546  * All users must obey the rule that the skb->data reference count must be
547  * greater than or equal to the payload reference count.
548  *
549  * Holding a reference to the payload part means that the user does not
550  * care about modifications to the header part of skb->data.
551  */
552 #define SKB_DATAREF_SHIFT 16
553 #define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1)
554 
555 
556 enum {
557         SKB_FCLONE_UNAVAILABLE, /* skb has no fclone (from head_cache) */
558         SKB_FCLONE_ORIG,        /* orig skb (from fclone_cache) */
559         SKB_FCLONE_CLONE,       /* companion fclone skb (from fclone_cache) */
560 };
561 
562 enum {
563         SKB_GSO_TCPV4 = 1 << 0,
564 
565         /* This indicates the skb is from an untrusted source. */
566         SKB_GSO_DODGY = 1 << 1,
567 
568         /* This indicates the tcp segment has CWR set. */
569         SKB_GSO_TCP_ECN = 1 << 2,
570 
571         SKB_GSO_TCP_FIXEDID = 1 << 3,
572 
573         SKB_GSO_TCPV6 = 1 << 4,
574 
575         SKB_GSO_FCOE = 1 << 5,
576 
577         SKB_GSO_GRE = 1 << 6,
578 
579         SKB_GSO_GRE_CSUM = 1 << 7,
580 
581         SKB_GSO_IPXIP4 = 1 << 8,
582 
583         SKB_GSO_IPXIP6 = 1 << 9,
584 
585         SKB_GSO_UDP_TUNNEL = 1 << 10,
586 
587         SKB_GSO_UDP_TUNNEL_CSUM = 1 << 11,
588 
589         SKB_GSO_PARTIAL = 1 << 12,
590 
591         SKB_GSO_TUNNEL_REMCSUM = 1 << 13,
592 
593         SKB_GSO_SCTP = 1 << 14,
594 
595         SKB_GSO_ESP = 1 << 15,
596 
597         SKB_GSO_UDP = 1 << 16,
598 
599         SKB_GSO_UDP_L4 = 1 << 17,
600 };
601 
602 #if BITS_PER_LONG > 32
603 #define NET_SKBUFF_DATA_USES_OFFSET 1
604 #endif
605 
606 #ifdef NET_SKBUFF_DATA_USES_OFFSET
607 typedef unsigned int sk_buff_data_t;
608 #else
609 typedef unsigned char *sk_buff_data_t;
610 #endif
611 
612 /**
613  *      struct sk_buff - socket buffer
614  *      @next: Next buffer in list
615  *      @prev: Previous buffer in list
616  *      @tstamp: Time we arrived/left
617  *      @rbnode: RB tree node, alternative to next/prev for netem/tcp
618  *      @sk: Socket we are owned by
619  *      @dev: Device we arrived on/are leaving by
620  *      @cb: Control buffer. Free for use by every layer. Put private vars here
621  *      @_skb_refdst: destination entry (with norefcount bit)
622  *      @sp: the security path, used for xfrm
623  *      @len: Length of actual data
624  *      @data_len: Data length
625  *      @mac_len: Length of link layer header
626  *      @hdr_len: writable header length of cloned skb
627  *      @csum: Checksum (must include start/offset pair)
628  *      @csum_start: Offset from skb->head where checksumming should start
629  *      @csum_offset: Offset from csum_start where checksum should be stored
630  *      @priority: Packet queueing priority
631  *      @ignore_df: allow local fragmentation
632  *      @cloned: Head may be cloned (check refcnt to be sure)
633  *      @ip_summed: Driver fed us an IP checksum
634  *      @nohdr: Payload reference only, must not modify header
635  *      @pkt_type: Packet class
636  *      @fclone: skbuff clone status
637  *      @ipvs_property: skbuff is owned by ipvs
638  *      @offload_fwd_mark: Packet was L2-forwarded in hardware
639  *      @offload_l3_fwd_mark: Packet was L3-forwarded in hardware
640  *      @tc_skip_classify: do not classify packet. set by IFB device
641  *      @tc_at_ingress: used within tc_classify to distinguish in/egress
642  *      @tc_redirected: packet was redirected by a tc action
643  *      @tc_from_ingress: if tc_redirected, tc_at_ingress at time of redirect
644  *      @peeked: this packet has been seen already, so stats have been
645  *              done for it, don't do them again
646  *      @nf_trace: netfilter packet trace flag
647  *      @protocol: Packet protocol from driver
648  *      @destructor: Destruct function
649  *      @tcp_tsorted_anchor: list structure for TCP (tp->tsorted_sent_queue)
650  *      @_nfct: Associated connection, if any (with nfctinfo bits)
651  *      @nf_bridge: Saved data about a bridged frame - see br_netfilter.c
652  *      @skb_iif: ifindex of device we arrived on
653  *      @tc_index: Traffic control index
654  *      @hash: the packet hash
655  *      @queue_mapping: Queue mapping for multiqueue devices
656  *      @pfmemalloc: skbuff was allocated from PFMEMALLOC reserves
657  *      @active_extensions: active extensions (skb_ext_id types)
658  *      @ndisc_nodetype: router type (from link layer)
659  *      @ooo_okay: allow the mapping of a socket to a queue to be changed
660  *      @l4_hash: indicate hash is a canonical 4-tuple hash over transport
661  *              ports.
662  *      @sw_hash: indicates hash was computed in software stack
663  *      @wifi_acked_valid: wifi_acked was set
664  *      @wifi_acked: whether frame was acked on wifi or not
665  *      @no_fcs:  Request NIC to treat last 4 bytes as Ethernet FCS
666  *      @csum_not_inet: use CRC32c to resolve CHECKSUM_PARTIAL
667  *      @dst_pending_confirm: need to confirm neighbour
668  *      @decrypted: Decrypted SKB
669  *      @napi_id: id of the NAPI struct this skb came from
670  *      @secmark: security marking
671  *      @mark: Generic packet mark
672  *      @vlan_proto: vlan encapsulation protocol
673  *      @vlan_tci: vlan tag control information
674  *      @inner_protocol: Protocol (encapsulation)
675  *      @inner_transport_header: Inner transport layer header (encapsulation)
676  *      @inner_network_header: Network layer header (encapsulation)
677  *      @inner_mac_header: Link layer header (encapsulation)
678  *      @transport_header: Transport layer header
679  *      @network_header: Network layer header
680  *      @mac_header: Link layer header
681  *      @tail: Tail pointer
682  *      @end: End pointer
683  *      @head: Head of buffer
684  *      @data: Data head pointer
685  *      @truesize: Buffer size
686  *      @users: User count - see {datagram,tcp}.c
687  *      @extensions: allocated extensions, valid if active_extensions is nonzero
688  */
689 
690 struct sk_buff {
691         union {
692                 struct {
693                         /* These two members must be first. */
694                         struct sk_buff          *next;
695                         struct sk_buff          *prev;
696 
697                         union {
698                                 struct net_device       *dev;
699                                 /* Some protocols might use this space to store information,
700                                  * while device pointer would be NULL.
701                                  * UDP receive path is one user.
702                                  */
703                                 unsigned long           dev_scratch;
704                         };
705                 };
706                 struct rb_node          rbnode; /* used in netem, ip4 defrag, and tcp stack */
707                 struct list_head        list;
708         };
709 
710         union {
711                 struct sock             *sk;
712                 int                     ip_defrag_offset;
713         };
714 
715         union {
716                 ktime_t         tstamp;
717                 u64             skb_mstamp_ns; /* earliest departure time */
718         };
719         /*
720          * This is the control buffer. It is free to use for every
721          * layer. Please put your private variables there. If you
722          * want to keep them across layers you have to do a skb_clone()
723          * first. This is owned by whoever has the skb queued ATM.
724          */
725         char                    cb[48] __aligned(8);
726 
727         union {
728                 struct {
729                         unsigned long   _skb_refdst;
730                         void            (*destructor)(struct sk_buff *skb);
731                 };
732                 struct list_head        tcp_tsorted_anchor;
733         };
734 
735 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
736         unsigned long            _nfct;
737 #endif
738         unsigned int            len,
739                                 data_len;
740         __u16                   mac_len,
741                                 hdr_len;
742 
743         /* Following fields are _not_ copied in __copy_skb_header()
744          * Note that queue_mapping is here mostly to fill a hole.
745          */
746         __u16                   queue_mapping;
747 
748 /* if you move cloned around you also must adapt those constants */
749 #ifdef __BIG_ENDIAN_BITFIELD
750 #define CLONED_MASK     (1 << 7)
751 #else
752 #define CLONED_MASK     1
753 #endif
754 #define CLONED_OFFSET()         offsetof(struct sk_buff, __cloned_offset)
755 
756         __u8                    __cloned_offset[0];
757         __u8                    cloned:1,
758                                 nohdr:1,
759                                 fclone:2,
760                                 peeked:1,
761                                 head_frag:1,
762                                 pfmemalloc:1;
763 #ifdef CONFIG_SKB_EXTENSIONS
764         __u8                    active_extensions;
765 #endif
766         /* fields enclosed in headers_start/headers_end are copied
767          * using a single memcpy() in __copy_skb_header()
768          */
769         /* private: */
770         __u32                   headers_start[0];
771         /* public: */
772 
773 /* if you move pkt_type around you also must adapt those constants */
774 #ifdef __BIG_ENDIAN_BITFIELD
775 #define PKT_TYPE_MAX    (7 << 5)
776 #else
777 #define PKT_TYPE_MAX    7
778 #endif
779 #define PKT_TYPE_OFFSET()       offsetof(struct sk_buff, __pkt_type_offset)
780 
781         __u8                    __pkt_type_offset[0];
782         __u8                    pkt_type:3;
783         __u8                    ignore_df:1;
784         __u8                    nf_trace:1;
785         __u8                    ip_summed:2;
786         __u8                    ooo_okay:1;
787 
788         __u8                    l4_hash:1;
789         __u8                    sw_hash:1;
790         __u8                    wifi_acked_valid:1;
791         __u8                    wifi_acked:1;
792         __u8                    no_fcs:1;
793         /* Indicates the inner headers are valid in the skbuff. */
794         __u8                    encapsulation:1;
795         __u8                    encap_hdr_csum:1;
796         __u8                    csum_valid:1;
797 
798 #ifdef __BIG_ENDIAN_BITFIELD
799 #define PKT_VLAN_PRESENT_BIT    7
800 #else
801 #define PKT_VLAN_PRESENT_BIT    0
802 #endif
803 #define PKT_VLAN_PRESENT_OFFSET()       offsetof(struct sk_buff, __pkt_vlan_present_offset)
804         __u8                    __pkt_vlan_present_offset[0];
805         __u8                    vlan_present:1;
806         __u8                    csum_complete_sw:1;
807         __u8                    csum_level:2;
808         __u8                    csum_not_inet:1;
809         __u8                    dst_pending_confirm:1;
810 #ifdef CONFIG_IPV6_NDISC_NODETYPE
811         __u8                    ndisc_nodetype:2;
812 #endif
813 
814         __u8                    ipvs_property:1;
815         __u8                    inner_protocol_type:1;
816         __u8                    remcsum_offload:1;
817 #ifdef CONFIG_NET_SWITCHDEV
818         __u8                    offload_fwd_mark:1;
819         __u8                    offload_l3_fwd_mark:1;
820 #endif
821 #ifdef CONFIG_NET_CLS_ACT
822         __u8                    tc_skip_classify:1;
823         __u8                    tc_at_ingress:1;
824         __u8                    tc_redirected:1;
825         __u8                    tc_from_ingress:1;
826 #endif
827 #ifdef CONFIG_TLS_DEVICE
828         __u8                    decrypted:1;
829 #endif
830 
831 #ifdef CONFIG_NET_SCHED
832         __u16                   tc_index;       /* traffic control index */
833 #endif
834 
835         union {
836                 __wsum          csum;
837                 struct {
838                         __u16   csum_start;
839                         __u16   csum_offset;
840                 };
841         };
842         __u32                   priority;
843         int                     skb_iif;
844         __u32                   hash;
845         __be16                  vlan_proto;
846         __u16                   vlan_tci;
847 #if defined(CONFIG_NET_RX_BUSY_POLL) || defined(CONFIG_XPS)
848         union {
849                 unsigned int    napi_id;
850                 unsigned int    sender_cpu;
851         };
852 #endif
853 #ifdef CONFIG_NETWORK_SECMARK
854         __u32           secmark;
855 #endif
856 
857         union {
858                 __u32           mark;
859                 __u32           reserved_tailroom;
860         };
861 
862         union {
863                 __be16          inner_protocol;
864                 __u8            inner_ipproto;
865         };
866 
867         __u16                   inner_transport_header;
868         __u16                   inner_network_header;
869         __u16                   inner_mac_header;
870 
871         __be16                  protocol;
872         __u16                   transport_header;
873         __u16                   network_header;
874         __u16                   mac_header;
875 
876         /* private: */
877         __u32                   headers_end[0];
878         /* public: */
879 
880         /* These elements must be at the end, see alloc_skb() for details.  */
881         sk_buff_data_t          tail;
882         sk_buff_data_t          end;
883         unsigned char           *head,
884                                 *data;
885         unsigned int            truesize;
886         refcount_t              users;
887 
888 #ifdef CONFIG_SKB_EXTENSIONS
889         /* only useable after checking ->active_extensions != 0 */
890         struct skb_ext          *extensions;
891 #endif
892 };
893 
894 #ifdef __KERNEL__
895 /*
896  *      Handling routines are only of interest to the kernel
897  */
898 
899 #define SKB_ALLOC_FCLONE        0x01
900 #define SKB_ALLOC_RX            0x02
901 #define SKB_ALLOC_NAPI          0x04
902 
903 /**
904  * skb_pfmemalloc - Test if the skb was allocated from PFMEMALLOC reserves
905  * @skb: buffer
906  */
907 static inline bool skb_pfmemalloc(const struct sk_buff *skb)
908 {
909         return unlikely(skb->pfmemalloc);
910 }
911 
912 /*
913  * skb might have a dst pointer attached, refcounted or not.
914  * _skb_refdst low order bit is set if refcount was _not_ taken
915  */
916 #define SKB_DST_NOREF   1UL
917 #define SKB_DST_PTRMASK ~(SKB_DST_NOREF)
918 
919 #define SKB_NFCT_PTRMASK        ~(7UL)
920 /**
921  * skb_dst - returns skb dst_entry
922  * @skb: buffer
923  *
924  * Returns skb dst_entry, regardless of reference taken or not.
925  */
926 static inline struct dst_entry *skb_dst(const struct sk_buff *skb)
927 {
928         /* If refdst was not refcounted, check we still are in a
929          * rcu_read_lock section
930          */
931         WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) &&
932                 !rcu_read_lock_held() &&
933                 !rcu_read_lock_bh_held());
934         return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK);
935 }
936 
937 /**
938  * skb_dst_set - sets skb dst
939  * @skb: buffer
940  * @dst: dst entry
941  *
942  * Sets skb dst, assuming a reference was taken on dst and should
943  * be released by skb_dst_drop()
944  */
945 static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst)
946 {
947         skb->_skb_refdst = (unsigned long)dst;
948 }
949 
950 /**
951  * skb_dst_set_noref - sets skb dst, hopefully, without taking reference
952  * @skb: buffer
953  * @dst: dst entry
954  *
955  * Sets skb dst, assuming a reference was not taken on dst.
956  * If dst entry is cached, we do not take reference and dst_release
957  * will be avoided by refdst_drop. If dst entry is not cached, we take
958  * reference, so that last dst_release can destroy the dst immediately.
959  */
960 static inline void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst)
961 {
962         WARN_ON(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
963         skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF;
964 }
965 
966 /**
967  * skb_dst_is_noref - Test if skb dst isn't refcounted
968  * @skb: buffer
969  */
970 static inline bool skb_dst_is_noref(const struct sk_buff *skb)
971 {
972         return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb);
973 }
974 
975 /**
976  * skb_rtable - Returns the skb &rtable
977  * @skb: buffer
978  */
979 static inline struct rtable *skb_rtable(const struct sk_buff *skb)
980 {
981         return (struct rtable *)skb_dst(skb);
982 }
983 
984 /* For mangling skb->pkt_type from user space side from applications
985  * such as nft, tc, etc, we only allow a conservative subset of
986  * possible pkt_types to be set.
987 */
988 static inline bool skb_pkt_type_ok(u32 ptype)
989 {
990         return ptype <= PACKET_OTHERHOST;
991 }
992 
993 /**
994  * skb_napi_id - Returns the skb's NAPI id
995  * @skb: buffer
996  */
997 static inline unsigned int skb_napi_id(const struct sk_buff *skb)
998 {
999 #ifdef CONFIG_NET_RX_BUSY_POLL
1000         return skb->napi_id;
1001 #else
1002         return 0;
1003 #endif
1004 }
1005 
1006 /**
1007  * skb_unref - decrement the skb's reference count
1008  * @skb: buffer
1009  *
1010  * Returns true if we can free the skb.
1011  */
1012 static inline bool skb_unref(struct sk_buff *skb)
1013 {
1014         if (unlikely(!skb))
1015                 return false;
1016         if (likely(refcount_read(&skb->users) == 1))
1017                 smp_rmb();
1018         else if (likely(!refcount_dec_and_test(&skb->users)))
1019                 return false;
1020 
1021         return true;
1022 }
1023 
1024 void skb_release_head_state(struct sk_buff *skb);
1025 void kfree_skb(struct sk_buff *skb);
1026 void kfree_skb_list(struct sk_buff *segs);
1027 void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt);
1028 void skb_tx_error(struct sk_buff *skb);
1029 void consume_skb(struct sk_buff *skb);
1030 void __consume_stateless_skb(struct sk_buff *skb);
1031 void  __kfree_skb(struct sk_buff *skb);
1032 extern struct kmem_cache *skbuff_head_cache;
1033 
1034 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
1035 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
1036                       bool *fragstolen, int *delta_truesize);
1037 
1038 struct sk_buff *__alloc_skb(unsigned int size, gfp_t priority, int flags,
1039                             int node);
1040 struct sk_buff *__build_skb(void *data, unsigned int frag_size);
1041 struct sk_buff *build_skb(void *data, unsigned int frag_size);
1042 struct sk_buff *build_skb_around(struct sk_buff *skb,
1043                                  void *data, unsigned int frag_size);
1044 
1045 /**
1046  * alloc_skb - allocate a network buffer
1047  * @size: size to allocate
1048  * @priority: allocation mask
1049  *
1050  * This function is a convenient wrapper around __alloc_skb().
1051  */
1052 static inline struct sk_buff *alloc_skb(unsigned int size,
1053                                         gfp_t priority)
1054 {
1055         return __alloc_skb(size, priority, 0, NUMA_NO_NODE);
1056 }
1057 
1058 struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
1059                                      unsigned long data_len,
1060                                      int max_page_order,
1061                                      int *errcode,
1062                                      gfp_t gfp_mask);
1063 struct sk_buff *alloc_skb_for_msg(struct sk_buff *first);
1064 
1065 /* Layout of fast clones : [skb1][skb2][fclone_ref] */
1066 struct sk_buff_fclones {
1067         struct sk_buff  skb1;
1068 
1069         struct sk_buff  skb2;
1070 
1071         refcount_t      fclone_ref;
1072 };
1073 
1074 /**
1075  *      skb_fclone_busy - check if fclone is busy
1076  *      @sk: socket
1077  *      @skb: buffer
1078  *
1079  * Returns true if skb is a fast clone, and its clone is not freed.
1080  * Some drivers call skb_orphan() in their ndo_start_xmit(),
1081  * so we also check that this didnt happen.
1082  */
1083 static inline bool skb_fclone_busy(const struct sock *sk,
1084                                    const struct sk_buff *skb)
1085 {
1086         const struct sk_buff_fclones *fclones;
1087 
1088         fclones = container_of(skb, struct sk_buff_fclones, skb1);
1089 
1090         return skb->fclone == SKB_FCLONE_ORIG &&
1091                refcount_read(&fclones->fclone_ref) > 1 &&
1092                fclones->skb2.sk == sk;
1093 }
1094 
1095 /**
1096  * alloc_skb_fclone - allocate a network buffer from fclone cache
1097  * @size: size to allocate
1098  * @priority: allocation mask
1099  *
1100  * This function is a convenient wrapper around __alloc_skb().
1101  */
1102 static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
1103                                                gfp_t priority)
1104 {
1105         return __alloc_skb(size, priority, SKB_ALLOC_FCLONE, NUMA_NO_NODE);
1106 }
1107 
1108 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src);
1109 void skb_headers_offset_update(struct sk_buff *skb, int off);
1110 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask);
1111 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t priority);
1112 void skb_copy_header(struct sk_buff *new, const struct sk_buff *old);
1113 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t priority);
1114 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
1115                                    gfp_t gfp_mask, bool fclone);
1116 static inline struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom,
1117                                           gfp_t gfp_mask)
1118 {
1119         return __pskb_copy_fclone(skb, headroom, gfp_mask, false);
1120 }
1121 
1122 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, gfp_t gfp_mask);
1123 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb,
1124                                      unsigned int headroom);
1125 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, int newheadroom,
1126                                 int newtailroom, gfp_t priority);
1127 int __must_check skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
1128                                      int offset, int len);
1129 int __must_check skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg,
1130                               int offset, int len);
1131 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer);
1132 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error);
1133 
1134 /**
1135  *      skb_pad                 -       zero pad the tail of an skb
1136  *      @skb: buffer to pad
1137  *      @pad: space to pad
1138  *
1139  *      Ensure that a buffer is followed by a padding area that is zero
1140  *      filled. Used by network drivers which may DMA or transfer data
1141  *      beyond the buffer end onto the wire.
1142  *
1143  *      May return error in out of memory cases. The skb is freed on error.
1144  */
1145 static inline int skb_pad(struct sk_buff *skb, int pad)
1146 {
1147         return __skb_pad(skb, pad, true);
1148 }
1149 #define dev_kfree_skb(a)        consume_skb(a)
1150 
1151 int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
1152                          int offset, size_t size);
1153 
1154 struct skb_seq_state {
1155         __u32           lower_offset;
1156         __u32           upper_offset;
1157         __u32           frag_idx;
1158         __u32           stepped_offset;
1159         struct sk_buff  *root_skb;
1160         struct sk_buff  *cur_skb;
1161         __u8            *frag_data;
1162 };
1163 
1164 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
1165                           unsigned int to, struct skb_seq_state *st);
1166 unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
1167                           struct skb_seq_state *st);
1168 void skb_abort_seq_read(struct skb_seq_state *st);
1169 
1170 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
1171                            unsigned int to, struct ts_config *config);
1172 
1173 /*
1174  * Packet hash types specify the type of hash in skb_set_hash.
1175  *
1176  * Hash types refer to the protocol layer addresses which are used to
1177  * construct a packet's hash. The hashes are used to differentiate or identify
1178  * flows of the protocol layer for the hash type. Hash types are either
1179  * layer-2 (L2), layer-3 (L3), or layer-4 (L4).
1180  *
1181  * Properties of hashes:
1182  *
1183  * 1) Two packets in different flows have different hash values
1184  * 2) Two packets in the same flow should have the same hash value
1185  *
1186  * A hash at a higher layer is considered to be more specific. A driver should
1187  * set the most specific hash possible.
1188  *
1189  * A driver cannot indicate a more specific hash than the layer at which a hash
1190  * was computed. For instance an L3 hash cannot be set as an L4 hash.
1191  *
1192  * A driver may indicate a hash level which is less specific than the
1193  * actual layer the hash was computed on. For instance, a hash computed
1194  * at L4 may be considered an L3 hash. This should only be done if the
1195  * driver can't unambiguously determine that the HW computed the hash at
1196  * the higher layer. Note that the "should" in the second property above
1197  * permits this.
1198  */
1199 enum pkt_hash_types {
1200         PKT_HASH_TYPE_NONE,     /* Undefined type */
1201         PKT_HASH_TYPE_L2,       /* Input: src_MAC, dest_MAC */
1202         PKT_HASH_TYPE_L3,       /* Input: src_IP, dst_IP */
1203         PKT_HASH_TYPE_L4,       /* Input: src_IP, dst_IP, src_port, dst_port */
1204 };
1205 
1206 static inline void skb_clear_hash(struct sk_buff *skb)
1207 {
1208         skb->hash = 0;
1209         skb->sw_hash = 0;
1210         skb->l4_hash = 0;
1211 }
1212 
1213 static inline void skb_clear_hash_if_not_l4(struct sk_buff *skb)
1214 {
1215         if (!skb->l4_hash)
1216                 skb_clear_hash(skb);
1217 }
1218 
1219 static inline void
1220 __skb_set_hash(struct sk_buff *skb, __u32 hash, bool is_sw, bool is_l4)
1221 {
1222         skb->l4_hash = is_l4;
1223         skb->sw_hash = is_sw;
1224         skb->hash = hash;
1225 }
1226 
1227 static inline void
1228 skb_set_hash(struct sk_buff *skb, __u32 hash, enum pkt_hash_types type)
1229 {
1230         /* Used by drivers to set hash from HW */
1231         __skb_set_hash(skb, hash, false, type == PKT_HASH_TYPE_L4);
1232 }
1233 
1234 static inline void
1235 __skb_set_sw_hash(struct sk_buff *skb, __u32 hash, bool is_l4)
1236 {
1237         __skb_set_hash(skb, hash, true, is_l4);
1238 }
1239 
1240 void __skb_get_hash(struct sk_buff *skb);
1241 u32 __skb_get_hash_symmetric(const struct sk_buff *skb);
1242 u32 skb_get_poff(const struct sk_buff *skb);
1243 u32 __skb_get_poff(const struct sk_buff *skb, void *data,
1244                    const struct flow_keys_basic *keys, int hlen);
1245 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
1246                             void *data, int hlen_proto);
1247 
1248 static inline __be32 skb_flow_get_ports(const struct sk_buff *skb,
1249                                         int thoff, u8 ip_proto)
1250 {
1251         return __skb_flow_get_ports(skb, thoff, ip_proto, NULL, 0);
1252 }
1253 
1254 void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
1255                              const struct flow_dissector_key *key,
1256                              unsigned int key_count);
1257 
1258 #ifdef CONFIG_NET
1259 int skb_flow_dissector_prog_query(const union bpf_attr *attr,
1260                                   union bpf_attr __user *uattr);
1261 int skb_flow_dissector_bpf_prog_attach(const union bpf_attr *attr,
1262                                        struct bpf_prog *prog);
1263 
1264 int skb_flow_dissector_bpf_prog_detach(const union bpf_attr *attr);
1265 #else
1266 static inline int skb_flow_dissector_prog_query(const union bpf_attr *attr,
1267                                                 union bpf_attr __user *uattr)
1268 {
1269         return -EOPNOTSUPP;
1270 }
1271 
1272 static inline int skb_flow_dissector_bpf_prog_attach(const union bpf_attr *attr,
1273                                                      struct bpf_prog *prog)
1274 {
1275         return -EOPNOTSUPP;
1276 }
1277 
1278 static inline int skb_flow_dissector_bpf_prog_detach(const union bpf_attr *attr)
1279 {
1280         return -EOPNOTSUPP;
1281 }
1282 #endif
1283 
1284 struct bpf_flow_dissector;
1285 bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
1286                       __be16 proto, int nhoff, int hlen);
1287 
1288 bool __skb_flow_dissect(const struct net *net,
1289                         const struct sk_buff *skb,
1290                         struct flow_dissector *flow_dissector,
1291                         void *target_container,
1292                         void *data, __be16 proto, int nhoff, int hlen,
1293                         unsigned int flags);
1294 
1295 static inline bool skb_flow_dissect(const struct sk_buff *skb,
1296                                     struct flow_dissector *flow_dissector,
1297                                     void *target_container, unsigned int flags)
1298 {
1299         return __skb_flow_dissect(NULL, skb, flow_dissector,
1300                                   target_container, NULL, 0, 0, 0, flags);
1301 }
1302 
1303 static inline bool skb_flow_dissect_flow_keys(const struct sk_buff *skb,
1304                                               struct flow_keys *flow,
1305                                               unsigned int flags)
1306 {
1307         memset(flow, 0, sizeof(*flow));
1308         return __skb_flow_dissect(NULL, skb, &flow_keys_dissector,
1309                                   flow, NULL, 0, 0, 0, flags);
1310 }
1311 
1312 static inline bool
1313 skb_flow_dissect_flow_keys_basic(const struct net *net,
1314                                  const struct sk_buff *skb,
1315                                  struct flow_keys_basic *flow, void *data,
1316                                  __be16 proto, int nhoff, int hlen,
1317                                  unsigned int flags)
1318 {
1319         memset(flow, 0, sizeof(*flow));
1320         return __skb_flow_dissect(net, skb, &flow_keys_basic_dissector, flow,
1321                                   data, proto, nhoff, hlen, flags);
1322 }
1323 
1324 void skb_flow_dissect_meta(const struct sk_buff *skb,
1325                            struct flow_dissector *flow_dissector,
1326                            void *target_container);
1327 
1328 /* Gets a skb connection tracking info, ctinfo map should be a
1329  * a map of mapsize to translate enum ip_conntrack_info states
1330  * to user states.
1331  */
1332 void
1333 skb_flow_dissect_ct(const struct sk_buff *skb,
1334                     struct flow_dissector *flow_dissector,
1335                     void *target_container,
1336                     u16 *ctinfo_map,
1337                     size_t mapsize);
1338 void
1339 skb_flow_dissect_tunnel_info(const struct sk_buff *skb,
1340                              struct flow_dissector *flow_dissector,
1341                              void *target_container);
1342 
1343 static inline __u32 skb_get_hash(struct sk_buff *skb)
1344 {
1345         if (!skb->l4_hash && !skb->sw_hash)
1346                 __skb_get_hash(skb);
1347 
1348         return skb->hash;
1349 }
1350 
1351 static inline __u32 skb_get_hash_flowi6(struct sk_buff *skb, const struct flowi6 *fl6)
1352 {
1353         if (!skb->l4_hash && !skb->sw_hash) {
1354                 struct flow_keys keys;
1355                 __u32 hash = __get_hash_from_flowi6(fl6, &keys);
1356 
1357                 __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys));
1358         }
1359 
1360         return skb->hash;
1361 }
1362 
1363 __u32 skb_get_hash_perturb(const struct sk_buff *skb,
1364                            const siphash_key_t *perturb);
1365 
1366 static inline __u32 skb_get_hash_raw(const struct sk_buff *skb)
1367 {
1368         return skb->hash;
1369 }
1370 
1371 static inline void skb_copy_hash(struct sk_buff *to, const struct sk_buff *from)
1372 {
1373         to->hash = from->hash;
1374         to->sw_hash = from->sw_hash;
1375         to->l4_hash = from->l4_hash;
1376 };
1377 
1378 static inline void skb_copy_decrypted(struct sk_buff *to,
1379                                       const struct sk_buff *from)
1380 {
1381 #ifdef CONFIG_TLS_DEVICE
1382         to->decrypted = from->decrypted;
1383 #endif
1384 }
1385 
1386 #ifdef NET_SKBUFF_DATA_USES_OFFSET
1387 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
1388 {
1389         return skb->head + skb->end;
1390 }
1391 
1392 static inline unsigned int skb_end_offset(const struct sk_buff *skb)
1393 {
1394         return skb->end;
1395 }
1396 #else
1397 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
1398 {
1399         return skb->end;
1400 }
1401 
1402 static inline unsigned int skb_end_offset(const struct sk_buff *skb)
1403 {
1404         return skb->end - skb->head;
1405 }
1406 #endif
1407 
1408 /* Internal */
1409 #define skb_shinfo(SKB) ((struct skb_shared_info *)(skb_end_pointer(SKB)))
1410 
1411 static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb)
1412 {
1413         return &skb_shinfo(skb)->hwtstamps;
1414 }
1415 
1416 static inline struct ubuf_info *skb_zcopy(struct sk_buff *skb)
1417 {
1418         bool is_zcopy = skb && skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY;
1419 
1420         return is_zcopy ? skb_uarg(skb) : NULL;
1421 }
1422 
1423 static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg,
1424                                  bool *have_ref)
1425 {
1426         if (skb && uarg && !skb_zcopy(skb)) {
1427                 if (unlikely(have_ref && *have_ref))
1428                         *have_ref = false;
1429                 else
1430                         sock_zerocopy_get(uarg);
1431                 skb_shinfo(skb)->destructor_arg = uarg;
1432                 skb_shinfo(skb)->tx_flags |= SKBTX_ZEROCOPY_FRAG;
1433         }
1434 }
1435 
1436 static inline void skb_zcopy_set_nouarg(struct sk_buff *skb, void *val)
1437 {
1438         skb_shinfo(skb)->destructor_arg = (void *)((uintptr_t) val | 0x1UL);
1439         skb_shinfo(skb)->tx_flags |= SKBTX_ZEROCOPY_FRAG;
1440 }
1441 
1442 static inline bool skb_zcopy_is_nouarg(struct sk_buff *skb)
1443 {
1444         return (uintptr_t) skb_shinfo(skb)->destructor_arg & 0x1UL;
1445 }
1446 
1447 static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb)
1448 {
1449         return (void *)((uintptr_t) skb_shinfo(skb)->destructor_arg & ~0x1UL);
1450 }
1451 
1452 /* Release a reference on a zerocopy structure */
1453 static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy)
1454 {
1455         struct ubuf_info *uarg = skb_zcopy(skb);
1456 
1457         if (uarg) {
1458                 if (skb_zcopy_is_nouarg(skb)) {
1459                         /* no notification callback */
1460                 } else if (uarg->callback == sock_zerocopy_callback) {
1461                         uarg->zerocopy = uarg->zerocopy && zerocopy;
1462                         sock_zerocopy_put(uarg);
1463                 } else {
1464                         uarg->callback(uarg, zerocopy);
1465                 }
1466 
1467                 skb_shinfo(skb)->tx_flags &= ~SKBTX_ZEROCOPY_FRAG;
1468         }
1469 }
1470 
1471 /* Abort a zerocopy operation and revert zckey on error in send syscall */
1472 static inline void skb_zcopy_abort(struct sk_buff *skb)
1473 {
1474         struct ubuf_info *uarg = skb_zcopy(skb);
1475 
1476         if (uarg) {
1477                 sock_zerocopy_put_abort(uarg, false);
1478                 skb_shinfo(skb)->tx_flags &= ~SKBTX_ZEROCOPY_FRAG;
1479         }
1480 }
1481 
1482 static inline void skb_mark_not_on_list(struct sk_buff *skb)
1483 {
1484         skb->next = NULL;
1485 }
1486 
1487 static inline void skb_list_del_init(struct sk_buff *skb)
1488 {
1489         __list_del_entry(&skb->list);
1490         skb_mark_not_on_list(skb);
1491 }
1492 
1493 /**
1494  *      skb_queue_empty - check if a queue is empty
1495  *      @list: queue head
1496  *
1497  *      Returns true if the queue is empty, false otherwise.
1498  */
1499 static inline int skb_queue_empty(const struct sk_buff_head *list)
1500 {
1501         return list->next == (const struct sk_buff *) list;
1502 }
1503 
1504 /**
1505  *      skb_queue_empty_lockless - check if a queue is empty
1506  *      @list: queue head
1507  *
1508  *      Returns true if the queue is empty, false otherwise.
1509  *      This variant can be used in lockless contexts.
1510  */
1511 static inline bool skb_queue_empty_lockless(const struct sk_buff_head *list)
1512 {
1513         return READ_ONCE(list->next) == (const struct sk_buff *) list;
1514 }
1515 
1516 
1517 /**
1518  *      skb_queue_is_last - check if skb is the last entry in the queue
1519  *      @list: queue head
1520  *      @skb: buffer
1521  *
1522  *      Returns true if @skb is the last buffer on the list.
1523  */
1524 static inline bool skb_queue_is_last(const struct sk_buff_head *list,
1525                                      const struct sk_buff *skb)
1526 {
1527         return skb->next == (const struct sk_buff *) list;
1528 }
1529 
1530 /**
1531  *      skb_queue_is_first - check if skb is the first entry in the queue
1532  *      @list: queue head
1533  *      @skb: buffer
1534  *
1535  *      Returns true if @skb is the first buffer on the list.
1536  */
1537 static inline bool skb_queue_is_first(const struct sk_buff_head *list,
1538                                       const struct sk_buff *skb)
1539 {
1540         return skb->prev == (const struct sk_buff *) list;
1541 }
1542 
1543 /**
1544  *      skb_queue_next - return the next packet in the queue
1545  *      @list: queue head
1546  *      @skb: current buffer
1547  *
1548  *      Return the next packet in @list after @skb.  It is only valid to
1549  *      call this if skb_queue_is_last() evaluates to false.
1550  */
1551 static inline struct sk_buff *skb_queue_next(const struct sk_buff_head *list,
1552                                              const struct sk_buff *skb)
1553 {
1554         /* This BUG_ON may seem severe, but if we just return then we
1555          * are going to dereference garbage.
1556          */
1557         BUG_ON(skb_queue_is_last(list, skb));
1558         return skb->next;
1559 }
1560 
1561 /**
1562  *      skb_queue_prev - return the prev packet in the queue
1563  *      @list: queue head
1564  *      @skb: current buffer
1565  *
1566  *      Return the prev packet in @list before @skb.  It is only valid to
1567  *      call this if skb_queue_is_first() evaluates to false.
1568  */
1569 static inline struct sk_buff *skb_queue_prev(const struct sk_buff_head *list,
1570                                              const struct sk_buff *skb)
1571 {
1572         /* This BUG_ON may seem severe, but if we just return then we
1573          * are going to dereference garbage.
1574          */
1575         BUG_ON(skb_queue_is_first(list, skb));
1576         return skb->prev;
1577 }
1578 
1579 /**
1580  *      skb_get - reference buffer
1581  *      @skb: buffer to reference
1582  *
1583  *      Makes another reference to a socket buffer and returns a pointer
1584  *      to the buffer.
1585  */
1586 static inline struct sk_buff *skb_get(struct sk_buff *skb)
1587 {
1588         refcount_inc(&skb->users);
1589         return skb;
1590 }
1591 
1592 /*
1593  * If users == 1, we are the only owner and can avoid redundant atomic changes.
1594  */
1595 
1596 /**
1597  *      skb_cloned - is the buffer a clone
1598  *      @skb: buffer to check
1599  *
1600  *      Returns true if the buffer was generated with skb_clone() and is
1601  *      one of multiple shared copies of the buffer. Cloned buffers are
1602  *      shared data so must not be written to under normal circumstances.
1603  */
1604 static inline int skb_cloned(const struct sk_buff *skb)
1605 {
1606         return skb->cloned &&
1607                (atomic_read(&skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1;
1608 }
1609 
1610 static inline int skb_unclone(struct sk_buff *skb, gfp_t pri)
1611 {
1612         might_sleep_if(gfpflags_allow_blocking(pri));
1613 
1614         if (skb_cloned(skb))
1615                 return pskb_expand_head(skb, 0, 0, pri);
1616 
1617         return 0;
1618 }
1619 
1620 /**
1621  *      skb_header_cloned - is the header a clone
1622  *      @skb: buffer to check
1623  *
1624  *      Returns true if modifying the header part of the buffer requires
1625  *      the data to be copied.
1626  */
1627 static inline int skb_header_cloned(const struct sk_buff *skb)
1628 {
1629         int dataref;
1630 
1631         if (!skb->cloned)
1632                 return 0;
1633 
1634         dataref = atomic_read(&skb_shinfo(skb)->dataref);
1635         dataref = (dataref & SKB_DATAREF_MASK) - (dataref >> SKB_DATAREF_SHIFT);
1636         return dataref != 1;
1637 }
1638 
1639 static inline int skb_header_unclone(struct sk_buff *skb, gfp_t pri)
1640 {
1641         might_sleep_if(gfpflags_allow_blocking(pri));
1642 
1643         if (skb_header_cloned(skb))
1644                 return pskb_expand_head(skb, 0, 0, pri);
1645 
1646         return 0;
1647 }
1648 
1649 /**
1650  *      __skb_header_release - release reference to header
1651  *      @skb: buffer to operate on
1652  */
1653 static inline void __skb_header_release(struct sk_buff *skb)
1654 {
1655         skb->nohdr = 1;
1656         atomic_set(&skb_shinfo(skb)->dataref, 1 + (1 << SKB_DATAREF_SHIFT));
1657 }
1658 
1659 
1660 /**
1661  *      skb_shared - is the buffer shared
1662  *      @skb: buffer to check
1663  *
1664  *      Returns true if more than one person has a reference to this
1665  *      buffer.
1666  */
1667 static inline int skb_shared(const struct sk_buff *skb)
1668 {
1669         return refcount_read(&skb->users) != 1;
1670 }
1671 
1672 /**
1673  *      skb_share_check - check if buffer is shared and if so clone it
1674  *      @skb: buffer to check
1675  *      @pri: priority for memory allocation
1676  *
1677  *      If the buffer is shared the buffer is cloned and the old copy
1678  *      drops a reference. A new clone with a single reference is returned.
1679  *      If the buffer is not shared the original buffer is returned. When
1680  *      being called from interrupt status or with spinlocks held pri must
1681  *      be GFP_ATOMIC.
1682  *
1683  *      NULL is returned on a memory allocation failure.
1684  */
1685 static inline struct sk_buff *skb_share_check(struct sk_buff *skb, gfp_t pri)
1686 {
1687         might_sleep_if(gfpflags_allow_blocking(pri));
1688         if (skb_shared(skb)) {
1689                 struct sk_buff *nskb = skb_clone(skb, pri);
1690 
1691                 if (likely(nskb))
1692                         consume_skb(skb);
1693                 else
1694                         kfree_skb(skb);
1695                 skb = nskb;
1696         }
1697         return skb;
1698 }
1699 
1700 /*
1701  *      Copy shared buffers into a new sk_buff. We effectively do COW on
1702  *      packets to handle cases where we have a local reader and forward
1703  *      and a couple of other messy ones. The normal one is tcpdumping
1704  *      a packet thats being forwarded.
1705  */
1706 
1707 /**
1708  *      skb_unshare - make a copy of a shared buffer
1709  *      @skb: buffer to check
1710  *      @pri: priority for memory allocation
1711  *
1712  *      If the socket buffer is a clone then this function creates a new
1713  *      copy of the data, drops a reference count on the old copy and returns
1714  *      the new copy with the reference count at 1. If the buffer is not a clone
1715  *      the original buffer is returned. When called with a spinlock held or
1716  *      from interrupt state @pri must be %GFP_ATOMIC
1717  *
1718  *      %NULL is returned on a memory allocation failure.
1719  */
1720 static inline struct sk_buff *skb_unshare(struct sk_buff *skb,
1721                                           gfp_t pri)
1722 {
1723         might_sleep_if(gfpflags_allow_blocking(pri));
1724         if (skb_cloned(skb)) {
1725                 struct sk_buff *nskb = skb_copy(skb, pri);
1726 
1727                 /* Free our shared copy */
1728                 if (likely(nskb))
1729                         consume_skb(skb);
1730                 else
1731                         kfree_skb(skb);
1732                 skb = nskb;
1733         }
1734         return skb;
1735 }
1736 
1737 /**
1738  *      skb_peek - peek at the head of an &sk_buff_head
1739  *      @list_: list to peek at
1740  *
1741  *      Peek an &sk_buff. Unlike most other operations you _MUST_
1742  *      be careful with this one. A peek leaves the buffer on the
1743  *      list and someone else may run off with it. You must hold
1744  *      the appropriate locks or have a private queue to do this.
1745  *
1746  *      Returns %NULL for an empty list or a pointer to the head element.
1747  *      The reference count is not incremented and the reference is therefore
1748  *      volatile. Use with caution.
1749  */
1750 static inline struct sk_buff *skb_peek(const struct sk_buff_head *list_)
1751 {
1752         struct sk_buff *skb = list_->next;
1753 
1754         if (skb == (struct sk_buff *)list_)
1755                 skb = NULL;
1756         return skb;
1757 }
1758 
1759 /**
1760  *      __skb_peek - peek at the head of a non-empty &sk_buff_head
1761  *      @list_: list to peek at
1762  *
1763  *      Like skb_peek(), but the caller knows that the list is not empty.
1764  */
1765 static inline struct sk_buff *__skb_peek(const struct sk_buff_head *list_)
1766 {
1767         return list_->next;
1768 }
1769 
1770 /**
1771  *      skb_peek_next - peek skb following the given one from a queue
1772  *      @skb: skb to start from
1773  *      @list_: list to peek at
1774  *
1775  *      Returns %NULL when the end of the list is met or a pointer to the
1776  *      next element. The reference count is not incremented and the
1777  *      reference is therefore volatile. Use with caution.
1778  */
1779 static inline struct sk_buff *skb_peek_next(struct sk_buff *skb,
1780                 const struct sk_buff_head *list_)
1781 {
1782         struct sk_buff *next = skb->next;
1783 
1784         if (next == (struct sk_buff *)list_)
1785                 next = NULL;
1786         return next;
1787 }
1788 
1789 /**
1790  *      skb_peek_tail - peek at the tail of an &sk_buff_head
1791  *      @list_: list to peek at
1792  *
1793  *      Peek an &sk_buff. Unlike most other operations you _MUST_
1794  *      be careful with this one. A peek leaves the buffer on the
1795  *      list and someone else may run off with it. You must hold
1796  *      the appropriate locks or have a private queue to do this.
1797  *
1798  *      Returns %NULL for an empty list or a pointer to the tail element.
1799  *      The reference count is not incremented and the reference is therefore
1800  *      volatile. Use with caution.
1801  */
1802 static inline struct sk_buff *skb_peek_tail(const struct sk_buff_head *list_)
1803 {
1804         struct sk_buff *skb = list_->prev;
1805 
1806         if (skb == (struct sk_buff *)list_)
1807                 skb = NULL;
1808         return skb;
1809 
1810 }
1811 
1812 /**
1813  *      skb_queue_len   - get queue length
1814  *      @list_: list to measure
1815  *
1816  *      Return the length of an &sk_buff queue.
1817  */
1818 static inline __u32 skb_queue_len(const struct sk_buff_head *list_)
1819 {
1820         return list_->qlen;
1821 }
1822 
1823 /**
1824  *      __skb_queue_head_init - initialize non-spinlock portions of sk_buff_head
1825  *      @list: queue to initialize
1826  *
1827  *      This initializes only the list and queue length aspects of
1828  *      an sk_buff_head object.  This allows to initialize the list
1829  *      aspects of an sk_buff_head without reinitializing things like
1830  *      the spinlock.  It can also be used for on-stack sk_buff_head
1831  *      objects where the spinlock is known to not be used.
1832  */
1833 static inline void __skb_queue_head_init(struct sk_buff_head *list)
1834 {
1835         list->prev = list->next = (struct sk_buff *)list;
1836         list->qlen = 0;
1837 }
1838 
1839 /*
1840  * This function creates a split out lock class for each invocation;
1841  * this is needed for now since a whole lot of users of the skb-queue
1842  * infrastructure in drivers have different locking usage (in hardirq)
1843  * than the networking core (in softirq only). In the long run either the
1844  * network layer or drivers should need annotation to consolidate the
1845  * main types of usage into 3 classes.
1846  */
1847 static inline void skb_queue_head_init(struct sk_buff_head *list)
1848 {
1849         spin_lock_init(&list->lock);
1850         __skb_queue_head_init(list);
1851 }
1852 
1853 static inline void skb_queue_head_init_class(struct sk_buff_head *list,
1854                 struct lock_class_key *class)
1855 {
1856         skb_queue_head_init(list);
1857         lockdep_set_class(&list->lock, class);
1858 }
1859 
1860 /*
1861  *      Insert an sk_buff on a list.
1862  *
1863  *      The "__skb_xxxx()" functions are the non-atomic ones that
1864  *      can only be called with interrupts disabled.
1865  */
1866 static inline void __skb_insert(struct sk_buff *newsk,
1867                                 struct sk_buff *prev, struct sk_buff *next,
1868                                 struct sk_buff_head *list)
1869 {
1870         /* see skb_queue_empty_lockless() for the opposite READ_ONCE() */
1871         WRITE_ONCE(newsk->next, next);
1872         WRITE_ONCE(newsk->prev, prev);
1873         WRITE_ONCE(next->prev, newsk);
1874         WRITE_ONCE(prev->next, newsk);
1875         list->qlen++;
1876 }
1877 
1878 static inline void __skb_queue_splice(const struct sk_buff_head *list,
1879                                       struct sk_buff *prev,
1880                                       struct sk_buff *next)
1881 {
1882         struct sk_buff *first = list->next;
1883         struct sk_buff *last = list->prev;
1884 
1885         WRITE_ONCE(first->prev, prev);
1886         WRITE_ONCE(prev->next, first);
1887 
1888         WRITE_ONCE(last->next, next);
1889         WRITE_ONCE(next->prev, last);
1890 }
1891 
1892 /**
1893  *      skb_queue_splice - join two skb lists, this is designed for stacks
1894  *      @list: the new list to add
1895  *      @head: the place to add it in the first list
1896  */
1897 static inline void skb_queue_splice(const struct sk_buff_head *list,
1898                                     struct sk_buff_head *head)
1899 {
1900         if (!skb_queue_empty(list)) {
1901                 __skb_queue_splice(list, (struct sk_buff *) head, head->next);
1902                 head->qlen += list->qlen;
1903         }
1904 }
1905 
1906 /**
1907  *      skb_queue_splice_init - join two skb lists and reinitialise the emptied list
1908  *      @list: the new list to add
1909  *      @head: the place to add it in the first list
1910  *
1911  *      The list at @list is reinitialised
1912  */
1913 static inline void skb_queue_splice_init(struct sk_buff_head *list,
1914                                          struct sk_buff_head *head)
1915 {
1916         if (!skb_queue_empty(list)) {
1917                 __skb_queue_splice(list, (struct sk_buff *) head, head->next);
1918                 head->qlen += list->qlen;
1919                 __skb_queue_head_init(list);
1920         }
1921 }
1922 
1923 /**
1924  *      skb_queue_splice_tail - join two skb lists, each list being a queue
1925  *      @list: the new list to add
1926  *      @head: the place to add it in the first list
1927  */
1928 static inline void skb_queue_splice_tail(const struct sk_buff_head *list,
1929                                          struct sk_buff_head *head)
1930 {
1931         if (!skb_queue_empty(list)) {
1932                 __skb_queue_splice(list, head->prev, (struct sk_buff *) head);
1933                 head->qlen += list->qlen;
1934         }
1935 }
1936 
1937 /**
1938  *      skb_queue_splice_tail_init - join two skb lists and reinitialise the emptied list
1939  *      @list: the new list to add
1940  *      @head: the place to add it in the first list
1941  *
1942  *      Each of the lists is a queue.
1943  *      The list at @list is reinitialised
1944  */
1945 static inline void skb_queue_splice_tail_init(struct sk_buff_head *list,
1946                                               struct sk_buff_head *head)
1947 {
1948         if (!skb_queue_empty(list)) {
1949                 __skb_queue_splice(list, head->prev, (struct sk_buff *) head);
1950                 head->qlen += list->qlen;
1951                 __skb_queue_head_init(list);
1952         }
1953 }
1954 
1955 /**
1956  *      __skb_queue_after - queue a buffer at the list head
1957  *      @list: list to use
1958  *      @prev: place after this buffer
1959  *      @newsk: buffer to queue
1960  *
1961  *      Queue a buffer int the middle of a list. This function takes no locks
1962  *      and you must therefore hold required locks before calling it.
1963  *
1964  *      A buffer cannot be placed on two lists at the same time.
1965  */
1966 static inline void __skb_queue_after(struct sk_buff_head *list,
1967                                      struct sk_buff *prev,
1968                                      struct sk_buff *newsk)
1969 {
1970         __skb_insert(newsk, prev, prev->next, list);
1971 }
1972 
1973 void skb_append(struct sk_buff *old, struct sk_buff *newsk,
1974                 struct sk_buff_head *list);
1975 
1976 static inline void __skb_queue_before(struct sk_buff_head *list,
1977                                       struct sk_buff *next,
1978                                       struct sk_buff *newsk)
1979 {
1980         __skb_insert(newsk, next->prev, next, list);
1981 }
1982 
1983 /**
1984  *      __skb_queue_head - queue a buffer at the list head
1985  *      @list: list to use
1986  *      @newsk: buffer to queue
1987  *
1988  *      Queue a buffer at the start of a list. This function takes no locks
1989  *      and you must therefore hold required locks before calling it.
1990  *
1991  *      A buffer cannot be placed on two lists at the same time.
1992  */
1993 static inline void __skb_queue_head(struct sk_buff_head *list,
1994                                     struct sk_buff *newsk)
1995 {
1996         __skb_queue_after(list, (struct sk_buff *)list, newsk);
1997 }
1998 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk);
1999 
2000 /**
2001  *      __skb_queue_tail - queue a buffer at the list tail
2002  *      @list: list to use
2003  *      @newsk: buffer to queue
2004  *
2005  *      Queue a buffer at the end of a list. This function takes no locks
2006  *      and you must therefore hold required locks before calling it.
2007  *
2008  *      A buffer cannot be placed on two lists at the same time.
2009  */
2010 static inline void __skb_queue_tail(struct sk_buff_head *list,
2011                                    struct sk_buff *newsk)
2012 {
2013         __skb_queue_before(list, (struct sk_buff *)list, newsk);
2014 }
2015 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk);
2016 
2017 /*
2018  * remove sk_buff from list. _Must_ be called atomically, and with
2019  * the list known..
2020  */
2021 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list);
2022 static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
2023 {
2024         struct sk_buff *next, *prev;
2025 
2026         list->qlen--;
2027         next       = skb->next;
2028         prev       = skb->prev;
2029         skb->next  = skb->prev = NULL;
2030         WRITE_ONCE(next->prev, prev);
2031         WRITE_ONCE(prev->next, next);
2032 }
2033 
2034 /**
2035  *      __skb_dequeue - remove from the head of the queue
2036  *      @list: list to dequeue from
2037  *
2038  *      Remove the head of the list. This function does not take any locks
2039  *      so must be used with appropriate locks held only. The head item is
2040  *      returned or %NULL if the list is empty.
2041  */
2042 static inline struct sk_buff *__skb_dequeue(struct sk_buff_head *list)
2043 {
2044         struct sk_buff *skb = skb_peek(list);
2045         if (skb)
2046                 __skb_unlink(skb, list);
2047         return skb;
2048 }
2049 struct sk_buff *skb_dequeue(struct sk_buff_head *list);
2050 
2051 /**
2052  *      __skb_dequeue_tail - remove from the tail of the queue
2053  *      @list: list to dequeue from
2054  *
2055  *      Remove the tail of the list. This function does not take any locks
2056  *      so must be used with appropriate locks held only. The tail item is
2057  *      returned or %NULL if the list is empty.
2058  */
2059 static inline struct sk_buff *__skb_dequeue_tail(struct sk_buff_head *list)
2060 {
2061         struct sk_buff *skb = skb_peek_tail(list);
2062         if (skb)
2063                 __skb_unlink(skb, list);
2064         return skb;
2065 }
2066 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list);
2067 
2068 
2069 static inline bool skb_is_nonlinear(const struct sk_buff *skb)
2070 {
2071         return skb->data_len;
2072 }
2073 
2074 static inline unsigned int skb_headlen(const struct sk_buff *skb)
2075 {
2076         return skb->len - skb->data_len;
2077 }
2078 
2079 static inline unsigned int __skb_pagelen(const struct sk_buff *skb)
2080 {
2081         unsigned int i, len = 0;
2082 
2083         for (i = skb_shinfo(skb)->nr_frags - 1; (int)i >= 0; i--)
2084                 len += skb_frag_size(&skb_shinfo(skb)->frags[i]);
2085         return len;
2086 }
2087 
2088 static inline unsigned int skb_pagelen(const struct sk_buff *skb)
2089 {
2090         return skb_headlen(skb) + __skb_pagelen(skb);
2091 }
2092 
2093 /**
2094  * __skb_fill_page_desc - initialise a paged fragment in an skb
2095  * @skb: buffer containing fragment to be initialised
2096  * @i: paged fragment index to initialise
2097  * @page: the page to use for this fragment
2098  * @off: the offset to the data with @page
2099  * @size: the length of the data
2100  *
2101  * Initialises the @i'th fragment of @skb to point to &size bytes at
2102  * offset @off within @page.
2103  *
2104  * Does not take any additional reference on the fragment.
2105  */
2106 static inline void __skb_fill_page_desc(struct sk_buff *skb, int i,
2107                                         struct page *page, int off, int size)
2108 {
2109         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2110 
2111         /*
2112          * Propagate page pfmemalloc to the skb if we can. The problem is
2113          * that not all callers have unique ownership of the page but rely
2114          * on page_is_pfmemalloc doing the right thing(tm).
2115          */
2116         frag->page.p              = page;
2117         frag->page_offset         = off;
2118         skb_frag_size_set(frag, size);
2119 
2120         page = compound_head(page);
2121         if (page_is_pfmemalloc(page))
2122                 skb->pfmemalloc = true;
2123 }
2124 
2125 /**
2126  * skb_fill_page_desc - initialise a paged fragment in an skb
2127  * @skb: buffer containing fragment to be initialised
2128  * @i: paged fragment index to initialise
2129  * @page: the page to use for this fragment
2130  * @off: the offset to the data with @page
2131  * @size: the length of the data
2132  *
2133  * As per __skb_fill_page_desc() -- initialises the @i'th fragment of
2134  * @skb to point to @size bytes at offset @off within @page. In
2135  * addition updates @skb such that @i is the last fragment.
2136  *
2137  * Does not take any additional reference on the fragment.
2138  */
2139 static inline void skb_fill_page_desc(struct sk_buff *skb, int i,
2140                                       struct page *page, int off, int size)
2141 {
2142         __skb_fill_page_desc(skb, i, page, off, size);
2143         skb_shinfo(skb)->nr_frags = i + 1;
2144 }
2145 
2146 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
2147                      int size, unsigned int truesize);
2148 
2149 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
2150                           unsigned int truesize);
2151 
2152 #define SKB_LINEAR_ASSERT(skb)  BUG_ON(skb_is_nonlinear(skb))
2153 
2154 #ifdef NET_SKBUFF_DATA_USES_OFFSET
2155 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
2156 {
2157         return skb->head + skb->tail;
2158 }
2159 
2160 static inline void skb_reset_tail_pointer(struct sk_buff *skb)
2161 {
2162         skb->tail = skb->data - skb->head;
2163 }
2164 
2165 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
2166 {
2167         skb_reset_tail_pointer(skb);
2168         skb->tail += offset;
2169 }
2170 
2171 #else /* NET_SKBUFF_DATA_USES_OFFSET */
2172 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
2173 {
2174         return skb->tail;
2175 }
2176 
2177 static inline void skb_reset_tail_pointer(struct sk_buff *skb)
2178 {
2179         skb->tail = skb->data;
2180 }
2181 
2182 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
2183 {
2184         skb->tail = skb->data + offset;
2185 }
2186 
2187 #endif /* NET_SKBUFF_DATA_USES_OFFSET */
2188 
2189 /*
2190  *      Add data to an sk_buff
2191  */
2192 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len);
2193 void *skb_put(struct sk_buff *skb, unsigned int len);
2194 static inline void *__skb_put(struct sk_buff *skb, unsigned int len)
2195 {
2196         void *tmp = skb_tail_pointer(skb);
2197         SKB_LINEAR_ASSERT(skb);
2198         skb->tail += len;
2199         skb->len  += len;
2200         return tmp;
2201 }
2202 
2203 static inline void *__skb_put_zero(struct sk_buff *skb, unsigned int len)
2204 {
2205         void *tmp = __skb_put(skb, len);
2206 
2207         memset(tmp, 0, len);
2208         return tmp;
2209 }
2210 
2211 static inline void *__skb_put_data(struct sk_buff *skb, const void *data,
2212                                    unsigned int len)
2213 {
2214         void *tmp = __skb_put(skb, len);
2215 
2216         memcpy(tmp, data, len);
2217         return tmp;
2218 }
2219 
2220 static inline void __skb_put_u8(struct sk_buff *skb, u8 val)
2221 {
2222         *(u8 *)__skb_put(skb, 1) = val;
2223 }
2224 
2225 static inline void *skb_put_zero(struct sk_buff *skb, unsigned int len)
2226 {
2227         void *tmp = skb_put(skb, len);
2228 
2229         memset(tmp, 0, len);
2230 
2231         return tmp;
2232 }
2233 
2234 static inline void *skb_put_data(struct sk_buff *skb, const void *data,
2235                                  unsigned int len)
2236 {
2237         void *tmp = skb_put(skb, len);
2238 
2239         memcpy(tmp, data, len);
2240 
2241         return tmp;
2242 }
2243 
2244 static inline void skb_put_u8(struct sk_buff *skb, u8 val)
2245 {
2246         *(u8 *)skb_put(skb, 1) = val;
2247 }
2248 
2249 void *skb_push(struct sk_buff *skb, unsigned int len);
2250 static inline void *__skb_push(struct sk_buff *skb, unsigned int len)
2251 {
2252         skb->data -= len;
2253         skb->len  += len;
2254         return skb->data;
2255 }
2256 
2257 void *skb_pull(struct sk_buff *skb, unsigned int len);
2258 static inline void *__skb_pull(struct sk_buff *skb, unsigned int len)
2259 {
2260         skb->len -= len;
2261         BUG_ON(skb->len < skb->data_len);
2262         return skb->data += len;
2263 }
2264 
2265 static inline void *skb_pull_inline(struct sk_buff *skb, unsigned int len)
2266 {
2267         return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
2268 }
2269 
2270 void *__pskb_pull_tail(struct sk_buff *skb, int delta);
2271 
2272 static inline void *__pskb_pull(struct sk_buff *skb, unsigned int len)
2273 {
2274         if (len > skb_headlen(skb) &&
2275             !__pskb_pull_tail(skb, len - skb_headlen(skb)))
2276                 return NULL;
2277         skb->len -= len;
2278         return skb->data += len;
2279 }
2280 
2281 static inline void *pskb_pull(struct sk_buff *skb, unsigned int len)
2282 {
2283         return unlikely(len > skb->len) ? NULL : __pskb_pull(skb, len);
2284 }
2285 
2286 static inline int pskb_may_pull(struct sk_buff *skb, unsigned int len)
2287 {
2288         if (likely(len <= skb_headlen(skb)))
2289                 return 1;
2290         if (unlikely(len > skb->len))
2291                 return 0;
2292         return __pskb_pull_tail(skb, len - skb_headlen(skb)) != NULL;
2293 }
2294 
2295 void skb_condense(struct sk_buff *skb);
2296 
2297 /**
2298  *      skb_headroom - bytes at buffer head
2299  *      @skb: buffer to check
2300  *
2301  *      Return the number of bytes of free space at the head of an &sk_buff.
2302  */
2303 static inline unsigned int skb_headroom(const struct sk_buff *skb)
2304 {
2305         return skb->data - skb->head;
2306 }
2307 
2308 /**
2309  *      skb_tailroom - bytes at buffer end
2310  *      @skb: buffer to check
2311  *
2312  *      Return the number of bytes of free space at the tail of an sk_buff
2313  */
2314 static inline int skb_tailroom(const struct sk_buff *skb)
2315 {
2316         return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail;
2317 }
2318 
2319 /**
2320  *      skb_availroom - bytes at buffer end
2321  *      @skb: buffer to check
2322  *
2323  *      Return the number of bytes of free space at the tail of an sk_buff
2324  *      allocated by sk_stream_alloc()
2325  */
2326 static inline int skb_availroom(const struct sk_buff *skb)
2327 {
2328         if (skb_is_nonlinear(skb))
2329                 return 0;
2330 
2331         return skb->end - skb->tail - skb->reserved_tailroom;
2332 }
2333 
2334 /**
2335  *      skb_reserve - adjust headroom
2336  *      @skb: buffer to alter
2337  *      @len: bytes to move
2338  *
2339  *      Increase the headroom of an empty &sk_buff by reducing the tail
2340  *      room. This is only allowed for an empty buffer.
2341  */
2342 static inline void skb_reserve(struct sk_buff *skb, int len)
2343 {
2344         skb->data += len;
2345         skb->tail += len;
2346 }
2347 
2348 /**
2349  *      skb_tailroom_reserve - adjust reserved_tailroom
2350  *      @skb: buffer to alter
2351  *      @mtu: maximum amount of headlen permitted
2352  *      @needed_tailroom: minimum amount of reserved_tailroom
2353  *
2354  *      Set reserved_tailroom so that headlen can be as large as possible but
2355  *      not larger than mtu and tailroom cannot be smaller than
2356  *      needed_tailroom.
2357  *      The required headroom should already have been reserved before using
2358  *      this function.
2359  */
2360 static inline void skb_tailroom_reserve(struct sk_buff *skb, unsigned int mtu,
2361                                         unsigned int needed_tailroom)
2362 {
2363         SKB_LINEAR_ASSERT(skb);
2364         if (mtu < skb_tailroom(skb) - needed_tailroom)
2365                 /* use at most mtu */
2366                 skb->reserved_tailroom = skb_tailroom(skb) - mtu;
2367         else
2368                 /* use up to all available space */
2369                 skb->reserved_tailroom = needed_tailroom;
2370 }
2371 
2372 #define ENCAP_TYPE_ETHER        0
2373 #define ENCAP_TYPE_IPPROTO      1
2374 
2375 static inline void skb_set_inner_protocol(struct sk_buff *skb,
2376                                           __be16 protocol)
2377 {
2378         skb->inner_protocol = protocol;
2379         skb->inner_protocol_type = ENCAP_TYPE_ETHER;
2380 }
2381 
2382 static inline void skb_set_inner_ipproto(struct sk_buff *skb,
2383                                          __u8 ipproto)
2384 {
2385         skb->inner_ipproto = ipproto;
2386         skb->inner_protocol_type = ENCAP_TYPE_IPPROTO;
2387 }
2388 
2389 static inline void skb_reset_inner_headers(struct sk_buff *skb)
2390 {
2391         skb->inner_mac_header = skb->mac_header;
2392         skb->inner_network_header = skb->network_header;
2393         skb->inner_transport_header = skb->transport_header;
2394 }
2395 
2396 static inline void skb_reset_mac_len(struct sk_buff *skb)
2397 {
2398         skb->mac_len = skb->network_header - skb->mac_header;
2399 }
2400 
2401 static inline unsigned char *skb_inner_transport_header(const struct sk_buff
2402                                                         *skb)
2403 {
2404         return skb->head + skb->inner_transport_header;
2405 }
2406 
2407 static inline int skb_inner_transport_offset(const struct sk_buff *skb)
2408 {
2409         return skb_inner_transport_header(skb) - skb->data;
2410 }
2411 
2412 static inline void skb_reset_inner_transport_header(struct sk_buff *skb)
2413 {
2414         skb->inner_transport_header = skb->data - skb->head;
2415 }
2416 
2417 static inline void skb_set_inner_transport_header(struct sk_buff *skb,
2418                                                    const int offset)
2419 {
2420         skb_reset_inner_transport_header(skb);
2421         skb->inner_transport_header += offset;
2422 }
2423 
2424 static inline unsigned char *skb_inner_network_header(const struct sk_buff *skb)
2425 {
2426         return skb->head + skb->inner_network_header;
2427 }
2428 
2429 static inline void skb_reset_inner_network_header(struct sk_buff *skb)
2430 {
2431         skb->inner_network_header = skb->data - skb->head;
2432 }
2433 
2434 static inline void skb_set_inner_network_header(struct sk_buff *skb,
2435                                                 const int offset)
2436 {
2437         skb_reset_inner_network_header(skb);
2438         skb->inner_network_header += offset;
2439 }
2440 
2441 static inline unsigned char *skb_inner_mac_header(const struct sk_buff *skb)
2442 {
2443         return skb->head + skb->inner_mac_header;
2444 }
2445 
2446 static inline void skb_reset_inner_mac_header(struct sk_buff *skb)
2447 {
2448         skb->inner_mac_header = skb->data - skb->head;
2449 }
2450 
2451 static inline void skb_set_inner_mac_header(struct sk_buff *skb,
2452                                             const int offset)
2453 {
2454         skb_reset_inner_mac_header(skb);
2455         skb->inner_mac_header += offset;
2456 }
2457 static inline bool skb_transport_header_was_set(const struct sk_buff *skb)
2458 {
2459         return skb->transport_header != (typeof(skb->transport_header))~0U;
2460 }
2461 
2462 static inline unsigned char *skb_transport_header(const struct sk_buff *skb)
2463 {
2464         return skb->head + skb->transport_header;
2465 }
2466 
2467 static inline void skb_reset_transport_header(struct sk_buff *skb)
2468 {
2469         skb->transport_header = skb->data - skb->head;
2470 }
2471 
2472 static inline void skb_set_transport_header(struct sk_buff *skb,
2473                                             const int offset)
2474 {
2475         skb_reset_transport_header(skb);
2476         skb->transport_header += offset;
2477 }
2478 
2479 static inline unsigned char *skb_network_header(const struct sk_buff *skb)
2480 {
2481         return skb->head + skb->network_header;
2482 }
2483 
2484 static inline void skb_reset_network_header(struct sk_buff *skb)
2485 {
2486         skb->network_header = skb->data - skb->head;
2487 }
2488 
2489 static inline void skb_set_network_header(struct sk_buff *skb, const int offset)
2490 {
2491         skb_reset_network_header(skb);
2492         skb->network_header += offset;
2493 }
2494 
2495 static inline unsigned char *skb_mac_header(const struct sk_buff *skb)
2496 {
2497         return skb->head + skb->mac_header;
2498 }
2499 
2500 static inline int skb_mac_offset(const struct sk_buff *skb)
2501 {
2502         return skb_mac_header(skb) - skb->data;
2503 }
2504 
2505 static inline u32 skb_mac_header_len(const struct sk_buff *skb)
2506 {
2507         return skb->network_header - skb->mac_header;
2508 }
2509 
2510 static inline int skb_mac_header_was_set(const struct sk_buff *skb)
2511 {
2512         return skb->mac_header != (typeof(skb->mac_header))~0U;
2513 }
2514 
2515 static inline void skb_reset_mac_header(struct sk_buff *skb)
2516 {
2517         skb->mac_header = skb->data - skb->head;
2518 }
2519 
2520 static inline void skb_set_mac_header(struct sk_buff *skb, const int offset)
2521 {
2522         skb_reset_mac_header(skb);
2523         skb->mac_header += offset;
2524 }
2525 
2526 static inline void skb_pop_mac_header(struct sk_buff *skb)
2527 {
2528         skb->mac_header = skb->network_header;
2529 }
2530 
2531 static inline void skb_probe_transport_header(struct sk_buff *skb)
2532 {
2533         struct flow_keys_basic keys;
2534 
2535         if (skb_transport_header_was_set(skb))
2536                 return;
2537 
2538         if (skb_flow_dissect_flow_keys_basic(NULL, skb, &keys,
2539                                              NULL, 0, 0, 0, 0))
2540                 skb_set_transport_header(skb, keys.control.thoff);
2541 }
2542 
2543 static inline void skb_mac_header_rebuild(struct sk_buff *skb)
2544 {
2545         if (skb_mac_header_was_set(skb)) {
2546                 const unsigned char *old_mac = skb_mac_header(skb);
2547 
2548                 skb_set_mac_header(skb, -skb->mac_len);
2549                 memmove(skb_mac_header(skb), old_mac, skb->mac_len);
2550         }
2551 }
2552 
2553 static inline int skb_checksum_start_offset(const struct sk_buff *skb)
2554 {
2555         return skb->csum_start - skb_headroom(skb);
2556 }
2557 
2558 static inline unsigned char *skb_checksum_start(const struct sk_buff *skb)
2559 {
2560         return skb->head + skb->csum_start;
2561 }
2562 
2563 static inline int skb_transport_offset(const struct sk_buff *skb)
2564 {
2565         return skb_transport_header(skb) - skb->data;
2566 }
2567 
2568 static inline u32 skb_network_header_len(const struct sk_buff *skb)
2569 {
2570         return skb->transport_header - skb->network_header;
2571 }
2572 
2573 static inline u32 skb_inner_network_header_len(const struct sk_buff *skb)
2574 {
2575         return skb->inner_transport_header - skb->inner_network_header;
2576 }
2577 
2578 static inline int skb_network_offset(const struct sk_buff *skb)
2579 {
2580         return skb_network_header(skb) - skb->data;
2581 }
2582 
2583 static inline int skb_inner_network_offset(const struct sk_buff *skb)
2584 {
2585         return skb_inner_network_header(skb) - skb->data;
2586 }
2587 
2588 static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len)
2589 {
2590         return pskb_may_pull(skb, skb_network_offset(skb) + len);
2591 }
2592 
2593 /*
2594  * CPUs often take a performance hit when accessing unaligned memory
2595  * locations. The actual performance hit varies, it can be small if the
2596  * hardware handles it or large if we have to take an exception and fix it
2597  * in software.
2598  *
2599  * Since an ethernet header is 14 bytes network drivers often end up with
2600  * the IP header at an unaligned offset. The IP header can be aligned by
2601  * shifting the start of the packet by 2 bytes. Drivers should do this
2602  * with:
2603  *
2604  * skb_reserve(skb, NET_IP_ALIGN);
2605  *
2606  * The downside to this alignment of the IP header is that the DMA is now
2607  * unaligned. On some architectures the cost of an unaligned DMA is high
2608  * and this cost outweighs the gains made by aligning the IP header.
2609  *
2610  * Since this trade off varies between architectures, we allow NET_IP_ALIGN
2611  * to be overridden.
2612  */
2613 #ifndef NET_IP_ALIGN
2614 #define NET_IP_ALIGN    2
2615 #endif
2616 
2617 /*
2618  * The networking layer reserves some headroom in skb data (via
2619  * dev_alloc_skb). This is used to avoid having to reallocate skb data when
2620  * the header has to grow. In the default case, if the header has to grow
2621  * 32 bytes or less we avoid the reallocation.
2622  *
2623  * Unfortunately this headroom changes the DMA alignment of the resulting
2624  * network packet. As for NET_IP_ALIGN, this unaligned DMA is expensive
2625  * on some architectures. An architecture can override this value,
2626  * perhaps setting it to a cacheline in size (since that will maintain
2627  * cacheline alignment of the DMA). It must be a power of 2.
2628  *
2629  * Various parts of the networking layer expect at least 32 bytes of
2630  * headroom, you should not reduce this.
2631  *
2632  * Using max(32, L1_CACHE_BYTES) makes sense (especially with RPS)
2633  * to reduce average number of cache lines per packet.
2634  * get_rps_cpus() for example only access one 64 bytes aligned block :
2635  * NET_IP_ALIGN(2) + ethernet_header(14) + IP_header(20/40) + ports(8)
2636  */
2637 #ifndef NET_SKB_PAD
2638 #define NET_SKB_PAD     max(32, L1_CACHE_BYTES)
2639 #endif
2640 
2641 int ___pskb_trim(struct sk_buff *skb, unsigned int len);
2642 
2643 static inline void __skb_set_length(struct sk_buff *skb, unsigned int len)
2644 {
2645         if (WARN_ON(skb_is_nonlinear(skb)))
2646                 return;
2647         skb->len = len;
2648         skb_set_tail_pointer(skb, len);
2649 }
2650 
2651 static inline void __skb_trim(struct sk_buff *skb, unsigned int len)
2652 {
2653         __skb_set_length(skb, len);
2654 }
2655 
2656 void skb_trim(struct sk_buff *skb, unsigned int len);
2657 
2658 static inline int __pskb_trim(struct sk_buff *skb, unsigned int len)
2659 {
2660         if (skb->data_len)
2661                 return ___pskb_trim(skb, len);
2662         __skb_trim(skb, len);
2663         return 0;
2664 }
2665 
2666 static inline int pskb_trim(struct sk_buff *skb, unsigned int len)
2667 {
2668         return (len < skb->len) ? __pskb_trim(skb, len) : 0;
2669 }
2670 
2671 /**
2672  *      pskb_trim_unique - remove end from a paged unique (not cloned) buffer
2673  *      @skb: buffer to alter
2674  *      @len: new length
2675  *
2676  *      This is identical to pskb_trim except that the caller knows that
2677  *      the skb is not cloned so we should never get an error due to out-
2678  *      of-memory.
2679  */
2680 static inline void pskb_trim_unique(struct sk_buff *skb, unsigned int len)
2681 {
2682         int err = pskb_trim(skb, len);
2683         BUG_ON(err);
2684 }
2685 
2686 static inline int __skb_grow(struct sk_buff *skb, unsigned int len)
2687 {
2688         unsigned int diff = len - skb->len;
2689 
2690         if (skb_tailroom(skb) < diff) {
2691                 int ret = pskb_expand_head(skb, 0, diff - skb_tailroom(skb),
2692                                            GFP_ATOMIC);
2693                 if (ret)
2694                         return ret;
2695         }
2696         __skb_set_length(skb, len);
2697         return 0;
2698 }
2699 
2700 /**
2701  *      skb_orphan - orphan a buffer
2702  *      @skb: buffer to orphan
2703  *
2704  *      If a buffer currently has an owner then we call the owner's
2705  *      destructor function and make the @skb unowned. The buffer continues
2706  *      to exist but is no longer charged to its former owner.
2707  */
2708 static inline void skb_orphan(struct sk_buff *skb)
2709 {
2710         if (skb->destructor) {
2711                 skb->destructor(skb);
2712                 skb->destructor = NULL;
2713                 skb->sk         = NULL;
2714         } else {
2715                 BUG_ON(skb->sk);
2716         }
2717 }
2718 
2719 /**
2720  *      skb_orphan_frags - orphan the frags contained in a buffer
2721  *      @skb: buffer to orphan frags from
2722  *      @gfp_mask: allocation mask for replacement pages
2723  *
2724  *      For each frag in the SKB which needs a destructor (i.e. has an
2725  *      owner) create a copy of that frag and release the original
2726  *      page by calling the destructor.
2727  */
2728 static inline int skb_orphan_frags(struct sk_buff *skb, gfp_t gfp_mask)
2729 {
2730         if (likely(!skb_zcopy(skb)))
2731                 return 0;
2732         if (!skb_zcopy_is_nouarg(skb) &&
2733             skb_uarg(skb)->callback == sock_zerocopy_callback)
2734                 return 0;
2735         return skb_copy_ubufs(skb, gfp_mask);
2736 }
2737 
2738 /* Frags must be orphaned, even if refcounted, if skb might loop to rx path */
2739 static inline int skb_orphan_frags_rx(struct sk_buff *skb, gfp_t gfp_mask)
2740 {
2741         if (likely(!skb_zcopy(skb)))
2742                 return 0;
2743         return skb_copy_ubufs(skb, gfp_mask);
2744 }
2745 
2746 /**
2747  *      __skb_queue_purge - empty a list
2748  *      @list: list to empty
2749  *
2750  *      Delete all buffers on an &sk_buff list. Each buffer is removed from
2751  *      the list and one reference dropped. This function does not take the
2752  *      list lock and the caller must hold the relevant locks to use it.
2753  */
2754 static inline void __skb_queue_purge(struct sk_buff_head *list)
2755 {
2756         struct sk_buff *skb;
2757         while ((skb = __skb_dequeue(list)) != NULL)
2758                 kfree_skb(skb);
2759 }
2760 void skb_queue_purge(struct sk_buff_head *list);
2761 
2762 unsigned int skb_rbtree_purge(struct rb_root *root);
2763 
2764 void *netdev_alloc_frag(unsigned int fragsz);
2765 
2766 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int length,
2767                                    gfp_t gfp_mask);
2768 
2769 /**
2770  *      netdev_alloc_skb - allocate an skbuff for rx on a specific device
2771  *      @dev: network device to receive on
2772  *      @length: length to allocate
2773  *
2774  *      Allocate a new &sk_buff and assign it a usage count of one. The
2775  *      buffer has unspecified headroom built in. Users should allocate
2776  *      the headroom they think they need without accounting for the
2777  *      built in space. The built in space is used for optimisations.
2778  *
2779  *      %NULL is returned if there is no free memory. Although this function
2780  *      allocates memory it can be called from an interrupt.
2781  */
2782 static inline struct sk_buff *netdev_alloc_skb(struct net_device *dev,
2783                                                unsigned int length)
2784 {
2785         return __netdev_alloc_skb(dev, length, GFP_ATOMIC);
2786 }
2787 
2788 /* legacy helper around __netdev_alloc_skb() */
2789 static inline struct sk_buff *__dev_alloc_skb(unsigned int length,
2790                                               gfp_t gfp_mask)
2791 {
2792         return __netdev_alloc_skb(NULL, length, gfp_mask);
2793 }
2794 
2795 /* legacy helper around netdev_alloc_skb() */
2796 static inline struct sk_buff *dev_alloc_skb(unsigned int length)
2797 {
2798         return netdev_alloc_skb(NULL, length);
2799 }
2800 
2801 
2802 static inline struct sk_buff *__netdev_alloc_skb_ip_align(struct net_device *dev,
2803                 unsigned int length, gfp_t gfp)
2804 {
2805         struct sk_buff *skb = __netdev_alloc_skb(dev, length + NET_IP_ALIGN, gfp);
2806 
2807         if (NET_IP_ALIGN && skb)
2808                 skb_reserve(skb, NET_IP_ALIGN);
2809         return skb;
2810 }
2811 
2812 static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev,
2813                 unsigned int length)
2814 {
2815         return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC);
2816 }
2817 
2818 static inline void skb_free_frag(void *addr)
2819 {
2820         page_frag_free(addr);
2821 }
2822 
2823 void *napi_alloc_frag(unsigned int fragsz);
2824 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi,
2825                                  unsigned int length, gfp_t gfp_mask);
2826 static inline struct sk_buff *napi_alloc_skb(struct napi_struct *napi,
2827                                              unsigned int length)
2828 {
2829         return __napi_alloc_skb(napi, length, GFP_ATOMIC);
2830 }
2831 void napi_consume_skb(struct sk_buff *skb, int budget);
2832 
2833 void __kfree_skb_flush(void);
2834 void __kfree_skb_defer(struct sk_buff *skb);
2835 
2836 /**
2837  * __dev_alloc_pages - allocate page for network Rx
2838  * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx
2839  * @order: size of the allocation
2840  *
2841  * Allocate a new page.
2842  *
2843  * %NULL is returned if there is no free memory.
2844 */
2845 static inline struct page *__dev_alloc_pages(gfp_t gfp_mask,
2846                                              unsigned int order)
2847 {
2848         /* This piece of code contains several assumptions.
2849          * 1.  This is for device Rx, therefor a cold page is preferred.
2850          * 2.  The expectation is the user wants a compound page.
2851          * 3.  If requesting a order 0 page it will not be compound
2852          *     due to the check to see if order has a value in prep_new_page
2853          * 4.  __GFP_MEMALLOC is ignored if __GFP_NOMEMALLOC is set due to
2854          *     code in gfp_to_alloc_flags that should be enforcing this.
2855          */
2856         gfp_mask |= __GFP_COMP | __GFP_MEMALLOC;
2857 
2858         return alloc_pages_node(NUMA_NO_NODE, gfp_mask, order);
2859 }
2860 
2861 static inline struct page *dev_alloc_pages(unsigned int order)
2862 {
2863         return __dev_alloc_pages(GFP_ATOMIC | __GFP_NOWARN, order);
2864 }
2865 
2866 /**
2867  * __dev_alloc_page - allocate a page for network Rx
2868  * @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx
2869  *
2870  * Allocate a new page.
2871  *
2872  * %NULL is returned if there is no free memory.
2873  */
2874 static inline struct page *__dev_alloc_page(gfp_t gfp_mask)
2875 {
2876         return __dev_alloc_pages(gfp_mask, 0);
2877 }
2878 
2879 static inline struct page *dev_alloc_page(void)
2880 {
2881         return dev_alloc_pages(0);
2882 }
2883 
2884 /**
2885  *      skb_propagate_pfmemalloc - Propagate pfmemalloc if skb is allocated after RX page
2886  *      @page: The page that was allocated from skb_alloc_page
2887  *      @skb: The skb that may need pfmemalloc set
2888  */
2889 static inline void skb_propagate_pfmemalloc(struct page *page,
2890                                              struct sk_buff *skb)
2891 {
2892         if (page_is_pfmemalloc(page))
2893                 skb->pfmemalloc = true;
2894 }
2895 
2896 /**
2897  * skb_frag_page - retrieve the page referred to by a paged fragment
2898  * @frag: the paged fragment
2899  *
2900  * Returns the &struct page associated with @frag.
2901  */
2902 static inline struct page *skb_frag_page(const skb_frag_t *frag)
2903 {
2904         return frag->page.p;
2905 }
2906 
2907 /**
2908  * __skb_frag_ref - take an addition reference on a paged fragment.
2909  * @frag: the paged fragment
2910  *
2911  * Takes an additional reference on the paged fragment @frag.
2912  */
2913 static inline void __skb_frag_ref(skb_frag_t *frag)
2914 {
2915         get_page(skb_frag_page(frag));
2916 }
2917 
2918 /**
2919  * skb_frag_ref - take an addition reference on a paged fragment of an skb.
2920  * @skb: the buffer
2921  * @f: the fragment offset.
2922  *
2923  * Takes an additional reference on the @f'th paged fragment of @skb.
2924  */
2925 static inline void skb_frag_ref(struct sk_buff *skb, int f)
2926 {
2927         __skb_frag_ref(&skb_shinfo(skb)->frags[f]);
2928 }
2929 
2930 /**
2931  * __skb_frag_unref - release a reference on a paged fragment.
2932  * @frag: the paged fragment
2933  *
2934  * Releases a reference on the paged fragment @frag.
2935  */
2936 static inline void __skb_frag_unref(skb_frag_t *frag)
2937 {
2938         put_page(skb_frag_page(frag));
2939 }
2940 
2941 /**
2942  * skb_frag_unref - release a reference on a paged fragment of an skb.
2943  * @skb: the buffer
2944  * @f: the fragment offset
2945  *
2946  * Releases a reference on the @f'th paged fragment of @skb.
2947  */
2948 static inline void skb_frag_unref(struct sk_buff *skb, int f)
2949 {
2950         __skb_frag_unref(&skb_shinfo(skb)->frags[f]);
2951 }
2952 
2953 /**
2954  * skb_frag_address - gets the address of the data contained in a paged fragment
2955  * @frag: the paged fragment buffer
2956  *
2957  * Returns the address of the data within @frag. The page must already
2958  * be mapped.
2959  */
2960 static inline void *skb_frag_address(const skb_frag_t *frag)
2961 {
2962         return page_address(skb_frag_page(frag)) + frag->page_offset;
2963 }
2964 
2965 /**
2966  * skb_frag_address_safe - gets the address of the data contained in a paged fragment
2967  * @frag: the paged fragment buffer
2968  *
2969  * Returns the address of the data within @frag. Checks that the page
2970  * is mapped and returns %NULL otherwise.
2971  */
2972 static inline void *skb_frag_address_safe(const skb_frag_t *frag)
2973 {
2974         void *ptr = page_address(skb_frag_page(frag));
2975         if (unlikely(!ptr))
2976                 return NULL;
2977 
2978         return ptr + frag->page_offset;
2979 }
2980 
2981 /**
2982  * __skb_frag_set_page - sets the page contained in a paged fragment
2983  * @frag: the paged fragment
2984  * @page: the page to set
2985  *
2986  * Sets the fragment @frag to contain @page.
2987  */
2988 static inline void __skb_frag_set_page(skb_frag_t *frag, struct page *page)
2989 {
2990         frag->page.p = page;
2991 }
2992 
2993 /**
2994  * skb_frag_set_page - sets the page contained in a paged fragment of an skb
2995  * @skb: the buffer
2996  * @f: the fragment offset
2997  * @page: the page to set
2998  *
2999  * Sets the @f'th fragment of @skb to contain @page.
3000  */
3001 static inline void skb_frag_set_page(struct sk_buff *skb, int f,
3002                                      struct page *page)
3003 {
3004         __skb_frag_set_page(&skb_shinfo(skb)->frags[f], page);
3005 }
3006 
3007 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t prio);
3008 
3009 /**
3010  * skb_frag_dma_map - maps a paged fragment via the DMA API
3011  * @dev: the device to map the fragment to
3012  * @frag: the paged fragment to map
3013  * @offset: the offset within the fragment (starting at the
3014  *          fragment's own offset)
3015  * @size: the number of bytes to map
3016  * @dir: the direction of the mapping (``PCI_DMA_*``)
3017  *
3018  * Maps the page associated with @frag to @device.
3019  */
3020 static inline dma_addr_t skb_frag_dma_map(struct device *dev,
3021                                           const skb_frag_t *frag,
3022                                           size_t offset, size_t size,
3023                                           enum dma_data_direction dir)
3024 {
3025         return dma_map_page(dev, skb_frag_page(frag),
3026                             frag->page_offset + offset, size, dir);
3027 }
3028 
3029 static inline struct sk_buff *pskb_copy(struct sk_buff *skb,
3030                                         gfp_t gfp_mask)
3031 {
3032         return __pskb_copy(skb, skb_headroom(skb), gfp_mask);
3033 }
3034 
3035 
3036 static inline struct sk_buff *pskb_copy_for_clone(struct sk_buff *skb,
3037                                                   gfp_t gfp_mask)
3038 {
3039         return __pskb_copy_fclone(skb, skb_headroom(skb), gfp_mask, true);
3040 }
3041 
3042 
3043 /**
3044  *      skb_clone_writable - is the header of a clone writable
3045  *      @skb: buffer to check
3046  *      @len: length up to which to write
3047  *
3048  *      Returns true if modifying the header part of the cloned buffer
3049  *      does not requires the data to be copied.
3050  */
3051 static inline int skb_clone_writable(const struct sk_buff *skb, unsigned int len)
3052 {
3053         return !skb_header_cloned(skb) &&
3054                skb_headroom(skb) + len <= skb->hdr_len;
3055 }
3056 
3057 static inline int skb_try_make_writable(struct sk_buff *skb,
3058                                         unsigned int write_len)
3059 {
3060         return skb_cloned(skb) && !skb_clone_writable(skb, write_len) &&
3061                pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3062 }
3063 
3064 static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom,
3065                             int cloned)
3066 {
3067         int delta = 0;
3068 
3069         if (headroom > skb_headroom(skb))
3070                 delta = headroom - skb_headroom(skb);
3071 
3072         if (delta || cloned)
3073                 return pskb_expand_head(skb, ALIGN(delta, NET_SKB_PAD), 0,
3074                                         GFP_ATOMIC);
3075         return 0;
3076 }
3077 
3078 /**
3079  *      skb_cow - copy header of skb when it is required
3080  *      @skb: buffer to cow
3081  *      @headroom: needed headroom
3082  *
3083  *      If the skb passed lacks sufficient headroom or its data part
3084  *      is shared, data is reallocated. If reallocation fails, an error
3085  *      is returned and original skb is not changed.
3086  *
3087  *      The result is skb with writable area skb->head...skb->tail
3088  *      and at least @headroom of space at head.
3089  */
3090 static inline int skb_cow(struct sk_buff *skb, unsigned int headroom)
3091 {
3092         return __skb_cow(skb, headroom, skb_cloned(skb));
3093 }
3094 
3095 /**
3096  *      skb_cow_head - skb_cow but only making the head writable
3097  *      @skb: buffer to cow
3098  *      @headroom: needed headroom
3099  *
3100  *      This function is identical to skb_cow except that we replace the
3101  *      skb_cloned check by skb_header_cloned.  It should be used when
3102  *      you only need to push on some header and do not need to modify
3103  *      the data.
3104  */
3105 static inline int skb_cow_head(struct sk_buff *skb, unsigned int headroom)
3106 {
3107         return __skb_cow(skb, headroom, skb_header_cloned(skb));
3108 }
3109 
3110 /**
3111  *      skb_padto       - pad an skbuff up to a minimal size
3112  *      @skb: buffer to pad
3113  *      @len: minimal length
3114  *
3115  *      Pads up a buffer to ensure the trailing bytes exist and are
3116  *      blanked. If the buffer already contains sufficient data it
3117  *      is untouched. Otherwise it is extended. Returns zero on
3118  *      success. The skb is freed on error.
3119  */
3120 static inline int skb_padto(struct sk_buff *skb, unsigned int len)
3121 {
3122         unsigned int size = skb->len;
3123         if (likely(size >= len))
3124                 return 0;
3125         return skb_pad(skb, len - size);
3126 }
3127 
3128 /**
3129  *      __skb_put_padto - increase size and pad an skbuff up to a minimal size
3130  *      @skb: buffer to pad
3131  *      @len: minimal length
3132  *      @free_on_error: free buffer on error
3133  *
3134  *      Pads up a buffer to ensure the trailing bytes exist and are
3135  *      blanked. If the buffer already contains sufficient data it
3136  *      is untouched. Otherwise it is extended. Returns zero on
3137  *      success. The skb is freed on error if @free_on_error is true.
3138  */
3139 static inline int __skb_put_padto(struct sk_buff *skb, unsigned int len,
3140                                   bool free_on_error)
3141 {
3142         unsigned int size = skb->len;
3143 
3144         if (unlikely(size < len)) {
3145                 len -= size;
3146                 if (__skb_pad(skb, len, free_on_error))
3147                         return -ENOMEM;
3148                 __skb_put(skb, len);
3149         }
3150         return 0;
3151 }
3152 
3153 /**
3154  *      skb_put_padto - increase size and pad an skbuff up to a minimal size
3155  *      @skb: buffer to pad
3156  *      @len: minimal length
3157  *
3158  *      Pads up a buffer to ensure the trailing bytes exist and are
3159  *      blanked. If the buffer already contains sufficient data it
3160  *      is untouched. Otherwise it is extended. Returns zero on
3161  *      success. The skb is freed on error.
3162  */
3163 static inline int skb_put_padto(struct sk_buff *skb, unsigned int len)
3164 {
3165         return __skb_put_padto(skb, len, true);
3166 }
3167 
3168 static inline int skb_add_data(struct sk_buff *skb,
3169                                struct iov_iter *from, int copy)
3170 {
3171         const int off = skb->len;
3172 
3173         if (skb->ip_summed == CHECKSUM_NONE) {
3174                 __wsum csum = 0;
3175                 if (csum_and_copy_from_iter_full(skb_put(skb, copy), copy,
3176                                                  &csum, from)) {
3177                         skb->csum = csum_block_add(skb->csum, csum, off);
3178                         return 0;
3179                 }
3180         } else if (copy_from_iter_full(skb_put(skb, copy), copy, from))
3181                 return 0;
3182 
3183         __skb_trim(skb, off);
3184         return -EFAULT;
3185 }
3186 
3187 static inline bool skb_can_coalesce(struct sk_buff *skb, int i,
3188                                     const struct page *page, int off)
3189 {
3190         if (skb_zcopy(skb))
3191                 return false;
3192         if (i) {
3193                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
3194 
3195                 return page == skb_frag_page(frag) &&
3196                        off == frag->page_offset + skb_frag_size(frag);
3197         }
3198         return false;
3199 }
3200 
3201 static inline int __skb_linearize(struct sk_buff *skb)
3202 {
3203         return __pskb_pull_tail(skb, skb->data_len) ? 0 : -ENOMEM;
3204 }
3205 
3206 /**
3207  *      skb_linearize - convert paged skb to linear one
3208  *      @skb: buffer to linarize
3209  *
3210  *      If there is no free memory -ENOMEM is returned, otherwise zero
3211  *      is returned and the old skb data released.
3212  */
3213 static inline int skb_linearize(struct sk_buff *skb)
3214 {
3215         return skb_is_nonlinear(skb) ? __skb_linearize(skb) : 0;
3216 }
3217 
3218 /**
3219  * skb_has_shared_frag - can any frag be overwritten
3220  * @skb: buffer to test
3221  *
3222  * Return true if the skb has at least one frag that might be modified
3223  * by an external entity (as in vmsplice()/sendfile())
3224  */
3225 static inline bool skb_has_shared_frag(const struct sk_buff *skb)
3226 {
3227         return skb_is_nonlinear(skb) &&
3228                skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG;
3229 }
3230 
3231 /**
3232  *      skb_linearize_cow - make sure skb is linear and writable
3233  *      @skb: buffer to process
3234  *
3235  *      If there is no free memory -ENOMEM is returned, otherwise zero
3236  *      is returned and the old skb data released.
3237  */
3238 static inline int skb_linearize_cow(struct sk_buff *skb)
3239 {
3240         return skb_is_nonlinear(skb) || skb_cloned(skb) ?
3241                __skb_linearize(skb) : 0;
3242 }
3243 
3244 static __always_inline void
3245 __skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len,
3246                      unsigned int off)
3247 {
3248         if (skb->ip_summed == CHECKSUM_COMPLETE)
3249                 skb->csum = csum_block_sub(skb->csum,
3250                                            csum_partial(start, len, 0), off);
3251         else if (skb->ip_summed == CHECKSUM_PARTIAL &&
3252                  skb_checksum_start_offset(skb) < 0)
3253                 skb->ip_summed = CHECKSUM_NONE;
3254 }
3255 
3256 /**
3257  *      skb_postpull_rcsum - update checksum for received skb after pull
3258  *      @skb: buffer to update
3259  *      @start: start of data before pull
3260  *      @len: length of data pulled
3261  *
3262  *      After doing a pull on a received packet, you need to call this to
3263  *      update the CHECKSUM_COMPLETE checksum, or set ip_summed to
3264  *      CHECKSUM_NONE so that it can be recomputed from scratch.
3265  */
3266 static inline void skb_postpull_rcsum(struct sk_buff *skb,
3267                                       const void *start, unsigned int len)
3268 {
3269         __skb_postpull_rcsum(skb, start, len, 0);
3270 }
3271 
3272 static __always_inline void
3273 __skb_postpush_rcsum(struct sk_buff *skb, const void *start, unsigned int len,
3274                      unsigned int off)
3275 {
3276         if (skb->ip_summed == CHECKSUM_COMPLETE)
3277                 skb->csum = csum_block_add(skb->csum,
3278                                            csum_partial(start, len, 0), off);
3279 }
3280 
3281 /**
3282  *      skb_postpush_rcsum - update checksum for received skb after push
3283  *      @skb: buffer to update
3284  *      @start: start of data after push
3285  *      @len: length of data pushed
3286  *
3287  *      After doing a push on a received packet, you need to call this to
3288  *      update the CHECKSUM_COMPLETE checksum.
3289  */
3290 static inline void skb_postpush_rcsum(struct sk_buff *skb,
3291                                       const void *start, unsigned int len)
3292 {
3293         __skb_postpush_rcsum(skb, start, len, 0);
3294 }
3295 
3296 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len);
3297 
3298 /**
3299  *      skb_push_rcsum - push skb and update receive checksum
3300  *      @skb: buffer to update
3301  *      @len: length of data pulled
3302  *
3303  *      This function performs an skb_push on the packet and updates
3304  *      the CHECKSUM_COMPLETE checksum.  It should be used on
3305  *      receive path processing instead of skb_push unless you know
3306  *      that the checksum difference is zero (e.g., a valid IP header)
3307  *      or you are setting ip_summed to CHECKSUM_NONE.
3308  */
3309 static inline void *skb_push_rcsum(struct sk_buff *skb, unsigned int len)
3310 {
3311         skb_push(skb, len);
3312         skb_postpush_rcsum(skb, skb->data, len);
3313         return skb->data;
3314 }
3315 
3316 int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len);
3317 /**
3318  *      pskb_trim_rcsum - trim received skb and update checksum
3319  *      @skb: buffer to trim
3320  *      @len: new length
3321  *
3322  *      This is exactly the same as pskb_trim except that it ensures the
3323  *      checksum of received packets are still valid after the operation.
3324  *      It can change skb pointers.
3325  */
3326 
3327 static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len)
3328 {
3329         if (likely(len >= skb->len))
3330                 return 0;
3331         return pskb_trim_rcsum_slow(skb, len);
3332 }
3333 
3334 static inline int __skb_trim_rcsum(struct sk_buff *skb, unsigned int len)
3335 {
3336         if (skb->ip_summed == CHECKSUM_COMPLETE)
3337                 skb->ip_summed = CHECKSUM_NONE;
3338         __skb_trim(skb, len);
3339         return 0;
3340 }
3341 
3342 static inline int __skb_grow_rcsum(struct sk_buff *skb, unsigned int len)
3343 {
3344         if (skb->ip_summed == CHECKSUM_COMPLETE)
3345                 skb->ip_summed = CHECKSUM_NONE;
3346         return __skb_grow(skb, len);
3347 }
3348 
3349 #define rb_to_skb(rb) rb_entry_safe(rb, struct sk_buff, rbnode)
3350 #define skb_rb_first(root) rb_to_skb(rb_first(root))
3351 #define skb_rb_last(root)  rb_to_skb(rb_last(root))
3352 #define skb_rb_next(skb)   rb_to_skb(rb_next(&(skb)->rbnode))
3353 #define skb_rb_prev(skb)   rb_to_skb(rb_prev(&(skb)->rbnode))
3354 
3355 #define skb_queue_walk(queue, skb) \
3356                 for (skb = (queue)->next;                                       \
3357                      skb != (struct sk_buff *)(queue);                          \
3358                      skb = skb->next)
3359 
3360 #define skb_queue_walk_safe(queue, skb, tmp)                                    \
3361                 for (skb = (queue)->next, tmp = skb->next;                      \
3362                      skb != (struct sk_buff *)(queue);                          \
3363                      skb = tmp, tmp = skb->next)
3364 
3365 #define skb_queue_walk_from(queue, skb)                                         \
3366                 for (; skb != (struct sk_buff *)(queue);                        \
3367                      skb = skb->next)
3368 
3369 #define skb_rbtree_walk(skb, root)                                              \
3370                 for (skb = skb_rb_first(root); skb != NULL;                     \
3371                      skb = skb_rb_next(skb))
3372 
3373 #define skb_rbtree_walk_from(skb)                                               \
3374                 for (; skb != NULL;                                             \
3375                      skb = skb_rb_next(skb))
3376 
3377 #define skb_rbtree_walk_from_safe(skb, tmp)                                     \
3378                 for (; tmp = skb ? skb_rb_next(skb) : NULL, (skb != NULL);      \
3379                      skb = tmp)
3380 
3381 #define skb_queue_walk_from_safe(queue, skb, tmp)                               \
3382                 for (tmp = skb->next;                                           \
3383                      skb != (struct sk_buff *)(queue);                          \
3384                      skb = tmp, tmp = skb->next)
3385 
3386 #define skb_queue_reverse_walk(queue, skb) \
3387                 for (skb = (queue)->prev;                                       \
3388                      skb != (struct sk_buff *)(queue);                          \
3389                      skb = skb->prev)
3390 
3391 #define skb_queue_reverse_walk_safe(queue, skb, tmp)                            \
3392                 for (skb = (queue)->prev, tmp = skb->prev;                      \
3393                      skb != (struct sk_buff *)(queue);                          \
3394                      skb = tmp, tmp = skb->prev)
3395 
3396 #define skb_queue_reverse_walk_from_safe(queue, skb, tmp)                       \
3397                 for (tmp = skb->prev;                                           \
3398                      skb != (struct sk_buff *)(queue);                          \
3399                      skb = tmp, tmp = skb->prev)
3400 
3401 static inline bool skb_has_frag_list(const struct sk_buff *skb)
3402 {
3403         return skb_shinfo(skb)->frag_list != NULL;
3404 }
3405 
3406 static inline void skb_frag_list_init(struct sk_buff *skb)
3407 {
3408         skb_shinfo(skb)->frag_list = NULL;
3409 }
3410 
3411 #define skb_walk_frags(skb, iter)       \
3412         for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next)
3413 
3414 
3415 int __skb_wait_for_more_packets(struct sock *sk, int *err, long *timeo_p,
3416                                 const struct sk_buff *skb);
3417 struct sk_buff *__skb_try_recv_from_queue(struct sock *sk,
3418                                           struct sk_buff_head *queue,
3419                                           unsigned int flags,
3420                                           void (*destructor)(struct sock *sk,
3421                                                            struct sk_buff *skb),
3422                                           int *off, int *err,
3423                                           struct sk_buff **last);
3424 struct sk_buff *__skb_try_recv_datagram(struct sock *sk, unsigned flags,
3425                                         void (*destructor)(struct sock *sk,
3426                                                            struct sk_buff *skb),
3427                                         int *off, int *err,
3428                                         struct sk_buff **last);
3429 struct sk_buff *__skb_recv_datagram(struct sock *sk, unsigned flags,
3430                                     void (*destructor)(struct sock *sk,
3431                                                        struct sk_buff *skb),
3432                                     int *off, int *err);
3433 struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned flags, int noblock,
3434                                   int *err);
3435 __poll_t datagram_poll(struct file *file, struct socket *sock,
3436                            struct poll_table_struct *wait);
3437 int skb_copy_datagram_iter(const struct sk_buff *from, int offset,
3438                            struct iov_iter *to, int size);
3439 static inline int skb_copy_datagram_msg(const struct sk_buff *from, int offset,
3440                                         struct msghdr *msg, int size)
3441 {
3442         return skb_copy_datagram_iter(from, offset, &msg->msg_iter, size);
3443 }
3444 int skb_copy_and_csum_datagram_msg(struct sk_buff *skb, int hlen,
3445                                    struct msghdr *msg);
3446 int skb_copy_and_hash_datagram_iter(const struct sk_buff *skb, int offset,
3447                            struct iov_iter *to, int len,
3448                            struct ahash_request *hash);
3449 int skb_copy_datagram_from_iter(struct sk_buff *skb, int offset,
3450                                  struct iov_iter *from, int len);
3451 int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *frm);
3452 void skb_free_datagram(struct sock *sk, struct sk_buff *skb);
3453 void __skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb, int len);
3454 static inline void skb_free_datagram_locked(struct sock *sk,
3455                                             struct sk_buff *skb)
3456 {
3457         __skb_free_datagram_locked(sk, skb, 0);
3458 }
3459 int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags);
3460 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len);
3461 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len);
3462 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, u8 *to,
3463                               int len, __wsum csum);
3464 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
3465                     struct pipe_inode_info *pipe, unsigned int len,
3466                     unsigned int flags);
3467 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
3468                          int len);
3469 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
3470 unsigned int skb_zerocopy_headlen(const struct sk_buff *from);
3471 int skb_zerocopy(struct sk_buff *to, struct sk_buff *from,
3472                  int len, int hlen);
3473 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len);
3474 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen);
3475 void skb_scrub_packet(struct sk_buff *skb, bool xnet);
3476 bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu);
3477 bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len);
3478 struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features);
3479 struct sk_buff *skb_vlan_untag(struct sk_buff *skb);
3480 int skb_ensure_writable(struct sk_buff *skb, int write_len);
3481 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci);
3482 int skb_vlan_pop(struct sk_buff *skb);
3483 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci);
3484 int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
3485                   int mac_len, bool ethernet);
3486 int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
3487                  bool ethernet);
3488 int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse);
3489 int skb_mpls_dec_ttl(struct sk_buff *skb);
3490 struct sk_buff *pskb_extract(struct sk_buff *skb, int off, int to_copy,
3491                              gfp_t gfp);
3492 
3493 static inline int memcpy_from_msg(void *data, struct msghdr *msg, int len)
3494 {
3495         return copy_from_iter_full(data, len, &msg->msg_iter) ? 0 : -EFAULT;
3496 }
3497 
3498 static inline int memcpy_to_msg(struct msghdr *msg, void *data, int len)
3499 {
3500         return copy_to_iter(data, len, &msg->msg_iter) == len ? 0 : -EFAULT;
3501 }
3502 
3503 struct skb_checksum_ops {
3504         __wsum (*update)(const void *mem, int len, __wsum wsum);
3505         __wsum (*combine)(__wsum csum, __wsum csum2, int offset, int len);
3506 };
3507 
3508 extern const struct skb_checksum_ops *crc32c_csum_stub __read_mostly;
3509 
3510 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
3511                       __wsum csum, const struct skb_checksum_ops *ops);
3512 __wsum skb_checksum(const struct sk_buff *skb, int offset, int len,
3513                     __wsum csum);
3514 
3515 static inline void * __must_check
3516 __skb_header_pointer(const struct sk_buff *skb, int offset,
3517                      int len, void *data, int hlen, void *buffer)
3518 {
3519         if (hlen - offset >= len)
3520                 return data + offset;
3521 
3522         if (!skb ||
3523             skb_copy_bits(skb, offset, buffer, len) < 0)
3524                 return NULL;
3525 
3526         return buffer;
3527 }
3528 
3529 static inline void * __must_check
3530 skb_header_pointer(const struct sk_buff *skb, int offset, int len, void *buffer)
3531 {
3532         return __skb_header_pointer(skb, offset, len, skb->data,
3533                                     skb_headlen(skb), buffer);
3534 }
3535 
3536 /**
3537  *      skb_needs_linearize - check if we need to linearize a given skb
3538  *                            depending on the given device features.
3539  *      @skb: socket buffer to check
3540  *      @features: net device features
3541  *
3542  *      Returns true if either:
3543  *      1. skb has frag_list and the device doesn't support FRAGLIST, or
3544  *      2. skb is fragmented and the device does not support SG.
3545  */
3546 static inline bool skb_needs_linearize(struct sk_buff *skb,
3547                                        netdev_features_t features)
3548 {
3549         return skb_is_nonlinear(skb) &&
3550                ((skb_has_frag_list(skb) && !(features & NETIF_F_FRAGLIST)) ||
3551                 (skb_shinfo(skb)->nr_frags && !(features & NETIF_F_SG)));
3552 }
3553 
3554 static inline void skb_copy_from_linear_data(const struct sk_buff *skb,
3555                                              void *to,
3556                                              const unsigned int len)
3557 {
3558         memcpy(to, skb->data, len);
3559 }
3560 
3561 static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb,
3562                                                     const int offset, void *to,
3563                                                     const unsigned int len)
3564 {
3565         memcpy(to, skb->data + offset, len);
3566 }
3567 
3568 static inline void skb_copy_to_linear_data(struct sk_buff *skb,
3569                                            const void *from,
3570                                            const unsigned int len)
3571 {
3572         memcpy(skb->data, from, len);
3573 }
3574 
3575 static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb,
3576                                                   const int offset,
3577                                                   const void *from,
3578                                                   const unsigned int len)
3579 {
3580         memcpy(skb->data + offset, from, len);
3581 }
3582 
3583 void skb_init(void);
3584 
3585 static inline ktime_t skb_get_ktime(const struct sk_buff *skb)
3586 {
3587         return skb->tstamp;
3588 }
3589 
3590 /**
3591  *      skb_get_timestamp - get timestamp from a skb
3592  *      @skb: skb to get stamp from
3593  *      @stamp: pointer to struct __kernel_old_timeval to store stamp in
3594  *
3595  *      Timestamps are stored in the skb as offsets to a base timestamp.
3596  *      This function converts the offset back to a struct timeval and stores
3597  *      it in stamp.
3598  */
3599 static inline void skb_get_timestamp(const struct sk_buff *skb,
3600                                      struct __kernel_old_timeval *stamp)
3601 {
3602         *stamp = ns_to_kernel_old_timeval(skb->tstamp);
3603 }
3604 
3605 static inline void skb_get_new_timestamp(const struct sk_buff *skb,
3606                                          struct __kernel_sock_timeval *stamp)
3607 {
3608         struct timespec64 ts = ktime_to_timespec64(skb->tstamp);
3609 
3610         stamp->tv_sec = ts.tv_sec;
3611         stamp->tv_usec = ts.tv_nsec / 1000;
3612 }
3613 
3614 static inline void skb_get_timestampns(const struct sk_buff *skb,
3615                                        struct timespec *stamp)
3616 {
3617         *stamp = ktime_to_timespec(skb->tstamp);
3618 }
3619 
3620 static inline void skb_get_new_timestampns(const struct sk_buff *skb,
3621                                            struct __kernel_timespec *stamp)
3622 {
3623         struct timespec64 ts = ktime_to_timespec64(skb->tstamp);
3624 
3625         stamp->tv_sec = ts.tv_sec;
3626         stamp->tv_nsec = ts.tv_nsec;
3627 }
3628 
3629 static inline void __net_timestamp(struct sk_buff *skb)
3630 {
3631         skb->tstamp = ktime_get_real();
3632 }
3633 
3634 static inline ktime_t net_timedelta(ktime_t t)
3635 {
3636         return ktime_sub(ktime_get_real(), t);
3637 }
3638 
3639 static inline ktime_t net_invalid_timestamp(void)
3640 {
3641         return 0;
3642 }
3643 
3644 static inline u8 skb_metadata_len(const struct sk_buff *skb)
3645 {
3646         return skb_shinfo(skb)->meta_len;
3647 }
3648 
3649 static inline void *skb_metadata_end(const struct sk_buff *skb)
3650 {
3651         return skb_mac_header(skb);
3652 }
3653 
3654 static inline bool __skb_metadata_differs(const struct sk_buff *skb_a,
3655                                           const struct sk_buff *skb_b,
3656                                           u8 meta_len)
3657 {
3658         const void *a = skb_metadata_end(skb_a);
3659         const void *b = skb_metadata_end(skb_b);
3660         /* Using more efficient varaiant than plain call to memcmp(). */
3661 #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64
3662         u64 diffs = 0;
3663 
3664         switch (meta_len) {
3665 #define __it(x, op) (x -= sizeof(u##op))
3666 #define __it_diff(a, b, op) (*(u##op *)__it(a, op)) ^ (*(u##op *)__it(b, op))
3667         case 32: diffs |= __it_diff(a, b, 64);
3668                  /* fall through */
3669         case 24: diffs |= __it_diff(a, b, 64);
3670                  /* fall through */
3671         case 16: diffs |= __it_diff(a, b, 64);
3672                  /* fall through */
3673         case  8: diffs |= __it_diff(a, b, 64);
3674                 break;
3675         case 28: diffs |= __it_diff(a, b, 64);
3676                  /* fall through */
3677         case 20: diffs |= __it_diff(a, b, 64);
3678                  /* fall through */
3679         case 12: diffs |= __it_diff(a, b, 64);
3680                  /* fall through */
3681         case  4: diffs |= __it_diff(a, b, 32);
3682                 break;
3683         }
3684         return diffs;
3685 #else
3686         return memcmp(a - meta_len, b - meta_len, meta_len);
3687 #endif
3688 }
3689 
3690 static inline bool skb_metadata_differs(const struct sk_buff *skb_a,
3691                                         const struct sk_buff *skb_b)
3692 {
3693         u8 len_a = skb_metadata_len(skb_a);
3694         u8 len_b = skb_metadata_len(skb_b);
3695 
3696         if (!(len_a | len_b))
3697                 return false;
3698 
3699         return len_a != len_b ?
3700                true : __skb_metadata_differs(skb_a, skb_b, len_a);
3701 }
3702 
3703 static inline void skb_metadata_set(struct sk_buff *skb, u8 meta_len)
3704 {
3705         skb_shinfo(skb)->meta_len = meta_len;
3706 }
3707 
3708 static inline void skb_metadata_clear(struct sk_buff *skb)
3709 {
3710         skb_metadata_set(skb, 0);
3711 }
3712 
3713 struct sk_buff *skb_clone_sk(struct sk_buff *skb);
3714 
3715 #ifdef CONFIG_NETWORK_PHY_TIMESTAMPING
3716 
3717 void skb_clone_tx_timestamp(struct sk_buff *skb);
3718 bool skb_defer_rx_timestamp(struct sk_buff *skb);
3719 
3720 #else /* CONFIG_NETWORK_PHY_TIMESTAMPING */
3721 
3722 static inline void skb_clone_tx_timestamp(struct sk_buff *skb)
3723 {
3724 }
3725 
3726 static inline bool skb_defer_rx_timestamp(struct sk_buff *skb)
3727 {
3728         return false;
3729 }
3730 
3731 #endif /* !CONFIG_NETWORK_PHY_TIMESTAMPING */
3732 
3733 /**
3734  * skb_complete_tx_timestamp() - deliver cloned skb with tx timestamps
3735  *
3736  * PHY drivers may accept clones of transmitted packets for
3737  * timestamping via their phy_driver.txtstamp method. These drivers
3738  * must call this function to return the skb back to the stack with a
3739  * timestamp.
3740  *
3741  * @skb: clone of the the original outgoing packet
3742  * @hwtstamps: hardware time stamps
3743  *
3744  */
3745 void skb_complete_tx_timestamp(struct sk_buff *skb,
3746                                struct skb_shared_hwtstamps *hwtstamps);
3747 
3748 void __skb_tstamp_tx(struct sk_buff *orig_skb,
3749                      struct skb_shared_hwtstamps *hwtstamps,
3750                      struct sock *sk, int tstype);
3751 
3752 /**
3753  * skb_tstamp_tx - queue clone of skb with send time stamps
3754  * @orig_skb:   the original outgoing packet
3755  * @hwtstamps:  hardware time stamps, may be NULL if not available
3756  *
3757  * If the skb has a socket associated, then this function clones the
3758  * skb (thus sharing the actual data and optional structures), stores
3759  * the optional hardware time stamping information (if non NULL) or
3760  * generates a software time stamp (otherwise), then queues the clone
3761  * to the error queue of the socket.  Errors are silently ignored.
3762  */
3763 void skb_tstamp_tx(struct sk_buff *orig_skb,
3764                    struct skb_shared_hwtstamps *hwtstamps);
3765 
3766 /**
3767  * skb_tx_timestamp() - Driver hook for transmit timestamping
3768  *
3769  * Ethernet MAC Drivers should call this function in their hard_xmit()
3770  * function immediately before giving the sk_buff to the MAC hardware.
3771  *
3772  * Specifically, one should make absolutely sure that this function is
3773  * called before TX completion of this packet can trigger.  Otherwise
3774  * the packet could potentially already be freed.
3775  *
3776  * @skb: A socket buffer.
3777  */
3778 static inline void skb_tx_timestamp(struct sk_buff *skb)
3779 {
3780         skb_clone_tx_timestamp(skb);
3781         if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP)
3782                 skb_tstamp_tx(skb, NULL);
3783 }
3784 
3785 /**
3786  * skb_complete_wifi_ack - deliver skb with wifi status
3787  *
3788  * @skb: the original outgoing packet
3789  * @acked: ack status
3790  *
3791  */
3792 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked);
3793 
3794 __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len);
3795 __sum16 __skb_checksum_complete(struct sk_buff *skb);
3796 
3797 static inline int skb_csum_unnecessary(const struct sk_buff *skb)
3798 {
3799         return ((skb->ip_summed == CHECKSUM_UNNECESSARY) ||
3800                 skb->csum_valid ||
3801                 (skb->ip_summed == CHECKSUM_PARTIAL &&
3802                  skb_checksum_start_offset(skb) >= 0));
3803 }
3804 
3805 /**
3806  *      skb_checksum_complete - Calculate checksum of an entire packet
3807  *      @skb: packet to process
3808  *
3809  *      This function calculates the checksum over the entire packet plus
3810  *      the value of skb->csum.  The latter can be used to supply the
3811  *      checksum of a pseudo header as used by TCP/UDP.  It returns the
3812  *      checksum.
3813  *
3814  *      For protocols that contain complete checksums such as ICMP/TCP/UDP,
3815  *      this function can be used to verify that checksum on received
3816  *      packets.  In that case the function should return zero if the
3817  *      checksum is correct.  In particular, this function will return zero
3818  *      if skb->ip_summed is CHECKSUM_UNNECESSARY which indicates that the
3819  *      hardware has already verified the correctness of the checksum.
3820  */
3821 static inline __sum16 skb_checksum_complete(struct sk_buff *skb)
3822 {
3823         return skb_csum_unnecessary(skb) ?
3824                0 : __skb_checksum_complete(skb);
3825 }
3826 
3827 static inline void __skb_decr_checksum_unnecessary(struct sk_buff *skb)
3828 {
3829         if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
3830                 if (skb->csum_level == 0)
3831                         skb->ip_summed = CHECKSUM_NONE;
3832                 else
3833                         skb->csum_level--;
3834         }
3835 }
3836 
3837 static inline void __skb_incr_checksum_unnecessary(struct sk_buff *skb)
3838 {
3839         if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
3840                 if (skb->csum_level < SKB_MAX_CSUM_LEVEL)
3841                         skb->csum_level++;
3842         } else if (skb->ip_summed == CHECKSUM_NONE) {
3843                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3844                 skb->csum_level = 0;
3845         }
3846 }
3847 
3848 /* Check if we need to perform checksum complete validation.
3849  *
3850  * Returns true if checksum complete is needed, false otherwise
3851  * (either checksum is unnecessary or zero checksum is allowed).
3852  */
3853 static inline bool __skb_checksum_validate_needed(struct sk_buff *skb,
3854                                                   bool zero_okay,
3855                                                   __sum16 check)
3856 {
3857         if (skb_csum_unnecessary(skb) || (zero_okay && !check)) {
3858                 skb->csum_valid = 1;
3859                 __skb_decr_checksum_unnecessary(skb);
3860                 return false;
3861         }
3862 
3863         return true;
3864 }
3865 
3866 /* For small packets <= CHECKSUM_BREAK perform checksum complete directly
3867  * in checksum_init.
3868  */
3869 #define CHECKSUM_BREAK 76
3870 
3871 /* Unset checksum-complete
3872  *
3873  * Unset checksum complete can be done when packet is being modified
3874  * (uncompressed for instance) and checksum-complete value is
3875  * invalidated.
3876  */
3877 static inline void skb_checksum_complete_unset(struct sk_buff *skb)
3878 {
3879         if (skb->ip_summed == CHECKSUM_COMPLETE)
3880                 skb->ip_summed = CHECKSUM_NONE;
3881 }
3882 
3883 /* Validate (init) checksum based on checksum complete.
3884  *
3885  * Return values:
3886  *   0: checksum is validated or try to in skb_checksum_complete. In the latter
3887  *      case the ip_summed will not be CHECKSUM_UNNECESSARY and the pseudo
3888  *      checksum is stored in skb->csum for use in __skb_checksum_complete
3889  *   non-zero: value of invalid checksum
3890  *
3891  */
3892 static inline __sum16 __skb_checksum_validate_complete(struct sk_buff *skb,
3893                                                        bool complete,
3894                                                        __wsum psum)
3895 {
3896         if (skb->ip_summed == CHECKSUM_COMPLETE) {
3897                 if (!csum_fold(csum_add(psum, skb->csum))) {
3898                         skb->csum_valid = 1;
3899                         return 0;
3900                 }
3901         }
3902 
3903         skb->csum = psum;
3904 
3905         if (complete || skb->len <= CHECKSUM_BREAK) {
3906                 __sum16 csum;
3907 
3908                 csum = __skb_checksum_complete(skb);
3909                 skb->csum_valid = !csum;
3910                 return csum;
3911         }
3912 
3913         return 0;
3914 }
3915 
3916 static inline __wsum null_compute_pseudo(struct sk_buff *skb, int proto)
3917 {
3918         return 0;
3919 }
3920 
3921 /* Perform checksum validate (init). Note that this is a macro since we only
3922  * want to calculate the pseudo header which is an input function if necessary.
3923  * First we try to validate without any computation (checksum unnecessary) and
3924  * then calculate based on checksum complete calling the function to compute
3925  * pseudo header.
3926  *
3927  * Return values:
3928  *   0: checksum is validated or try to in skb_checksum_complete
3929  *   non-zero: value of invalid checksum
3930  */
3931 #define __skb_checksum_validate(skb, proto, complete,                   \
3932                                 zero_okay, check, compute_pseudo)       \
3933 ({                                                                      \
3934         __sum16 __ret = 0;                                              \
3935         skb->csum_valid = 0;                                            \
3936         if (__skb_checksum_validate_needed(skb, zero_okay, check))      \
3937                 __ret = __skb_checksum_validate_complete(skb,           \
3938                                 complete, compute_pseudo(skb, proto));  \
3939         __ret;                                                          \
3940 })
3941 
3942 #define skb_checksum_init(skb, proto, compute_pseudo)                   \
3943         __skb_checksum_validate(skb, proto, false, false, 0, compute_pseudo)
3944 
3945 #define skb_checksum_init_zero_check(skb, proto, check, compute_pseudo) \
3946         __skb_checksum_validate(skb, proto, false, true, check, compute_pseudo)
3947 
3948 #define skb_checksum_validate(skb, proto, compute_pseudo)               \
3949         __skb_checksum_validate(skb, proto, true, false, 0, compute_pseudo)
3950 
3951 #define skb_checksum_validate_zero_check(skb, proto, check,             \
3952                                          compute_pseudo)                \
3953         __skb_checksum_validate(skb, proto, true, true, check, compute_pseudo)
3954 
3955 #define skb_checksum_simple_validate(skb)                               \
3956         __skb_checksum_validate(skb, 0, true, false, 0, null_compute_pseudo)
3957 
3958 static inline bool __skb_checksum_convert_check(struct sk_buff *skb)
3959 {
3960         return (skb->ip_summed == CHECKSUM_NONE && skb->csum_valid);
3961 }
3962 
3963 static inline void __skb_checksum_convert(struct sk_buff *skb, __wsum pseudo)
3964 {
3965         skb->csum = ~pseudo;
3966         skb->ip_summed = CHECKSUM_COMPLETE;
3967 }
3968 
3969 #define skb_checksum_try_convert(skb, proto, compute_pseudo)    \
3970 do {                                                                    \
3971         if (__skb_checksum_convert_check(skb))                          \
3972                 __skb_checksum_convert(skb, compute_pseudo(skb, proto)); \
3973 } while (0)
3974 
3975 static inline void skb_remcsum_adjust_partial(struct sk_buff *skb, void *ptr,
3976                                               u16 start, u16 offset)
3977 {
3978         skb->ip_summed = CHECKSUM_PARTIAL;
3979         skb->csum_start = ((unsigned char *)ptr + start) - skb->head;
3980         skb->csum_offset = offset - start;
3981 }
3982 
3983 /* Update skbuf and packet to reflect the remote checksum offload operation.
3984  * When called, ptr indicates the starting point for skb->csum when
3985  * ip_summed is CHECKSUM_COMPLETE. If we need create checksum complete
3986  * here, skb_postpull_rcsum is done so skb->csum start is ptr.
3987  */
3988 static inline void skb_remcsum_process(struct sk_buff *skb, void *ptr,
3989                                        int start, int offset, bool nopartial)
3990 {
3991         __wsum delta;
3992 
3993         if (!nopartial) {
3994                 skb_remcsum_adjust_partial(skb, ptr, start, offset);
3995                 return;
3996         }
3997 
3998          if (unlikely(skb->ip_summed != CHECKSUM_COMPLETE)) {
3999                 __skb_checksum_complete(skb);
4000                 skb_postpull_rcsum(skb, skb->data, ptr - (void *)skb->data);
4001         }
4002 
4003         delta = remcsum_adjust(ptr, skb->csum, start, offset);
4004 
4005         /* Adjust skb->csum since we changed the packet */
4006         skb->csum = csum_add(skb->csum, delta);
4007 }
4008 
4009 static inline struct nf_conntrack *skb_nfct(const struct sk_buff *skb)
4010 {
4011 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
4012         return (void *)(skb->_nfct & SKB_NFCT_PTRMASK);
4013 #else
4014         return NULL;
4015 #endif
4016 }
4017 
4018 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
4019 void nf_conntrack_destroy(struct nf_conntrack *nfct);
4020 static inline void nf_conntrack_put(struct nf_conntrack *nfct)
4021 {
4022         if (nfct && atomic_dec_and_test(&nfct->use))
4023                 nf_conntrack_destroy(nfct);
4024 }
4025 static inline void nf_conntrack_get(struct nf_conntrack *nfct)
4026 {
4027         if (nfct)
4028                 atomic_inc(&nfct->use);
4029 }
4030 #endif
4031 
4032 #ifdef CONFIG_SKB_EXTENSIONS
4033 enum skb_ext_id {
4034 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
4035         SKB_EXT_BRIDGE_NF,
4036 #endif
4037 #ifdef CONFIG_XFRM
4038         SKB_EXT_SEC_PATH,
4039 #endif
4040         SKB_EXT_NUM, /* must be last */
4041 };
4042 
4043 /**
4044  *      struct skb_ext - sk_buff extensions
4045  *      @refcnt: 1 on allocation, deallocated on 0
4046  *      @offset: offset to add to @data to obtain extension address
4047  *      @chunks: size currently allocated, stored in SKB_EXT_ALIGN_SHIFT units
4048  *      @data: start of extension data, variable sized
4049  *
4050  *      Note: offsets/lengths are stored in chunks of 8 bytes, this allows
4051  *      to use 'u8' types while allowing up to 2kb worth of extension data.
4052  */
4053 struct skb_ext {
4054         refcount_t refcnt;
4055         u8 offset[SKB_EXT_NUM]; /* in chunks of 8 bytes */
4056         u8 chunks;              /* same */
4057         char data[0] __aligned(8);
4058 };
4059 
4060 void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id);
4061 void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id);
4062 void __skb_ext_put(struct skb_ext *ext);
4063 
4064 static inline void skb_ext_put(struct sk_buff *skb)
4065 {
4066         if (skb->active_extensions)
4067                 __skb_ext_put(skb->extensions);
4068 }
4069 
4070 static inline void __skb_ext_copy(struct sk_buff *dst,
4071                                   const struct sk_buff *src)
4072 {
4073         dst->active_extensions = src->active_extensions;
4074 
4075         if (src->active_extensions) {
4076                 struct skb_ext *ext = src->extensions;
4077 
4078                 refcount_inc(&ext->refcnt);
4079                 dst->extensions = ext;
4080         }
4081 }
4082 
4083 static inline void skb_ext_copy(struct sk_buff *dst, const struct sk_buff *src)
4084 {
4085         skb_ext_put(dst);
4086         __skb_ext_copy(dst, src);
4087 }
4088 
4089 static inline bool __skb_ext_exist(const struct skb_ext *ext, enum skb_ext_id i)
4090 {
4091         return !!ext->offset[i];
4092 }
4093 
4094 static inline bool skb_ext_exist(const struct sk_buff *skb, enum skb_ext_id id)
4095 {
4096         return skb->active_extensions & (1 << id);
4097 }
4098 
4099 static inline void skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
4100 {
4101         if (skb_ext_exist(skb, id))
4102                 __skb_ext_del(skb, id);
4103 }
4104 
4105 static inline void *skb_ext_find(const struct sk_buff *skb, enum skb_ext_id id)
4106 {
4107         if (skb_ext_exist(skb, id)) {
4108                 struct skb_ext *ext = skb->extensions;
4109 
4110                 return (void *)ext + (ext->offset[id] << 3);
4111         }
4112 
4113         return NULL;
4114 }
4115 #else
4116 static inline void skb_ext_put(struct sk_buff *skb) {}
4117 static inline void skb_ext_del(struct sk_buff *skb, int unused) {}
4118 static inline void __skb_ext_copy(struct sk_buff *d, const struct sk_buff *s) {}
4119 static inline void skb_ext_copy(struct sk_buff *dst, const struct sk_buff *s) {}
4120 #endif /* CONFIG_SKB_EXTENSIONS */
4121 
4122 static inline void nf_reset(struct sk_buff *skb)
4123 {
4124 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
4125         nf_conntrack_put(skb_nfct(skb));
4126         skb->_nfct = 0;
4127 #endif
4128 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
4129         skb_ext_del(skb, SKB_EXT_BRIDGE_NF);
4130 #endif
4131 }
4132 
4133 static inline void nf_reset_trace(struct sk_buff *skb)
4134 {
4135 #if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || defined(CONFIG_NF_TABLES)
4136         skb->nf_trace = 0;
4137 #endif
4138 }
4139 
4140 static inline void ipvs_reset(struct sk_buff *skb)
4141 {
4142 #if IS_ENABLED(CONFIG_IP_VS)
4143         skb->ipvs_property = 0;
4144 #endif
4145 }
4146 
4147 /* Note: This doesn't put any conntrack info in dst. */
4148 static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src,
4149                              bool copy)
4150 {
4151 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
4152         dst->_nfct = src->_nfct;
4153         nf_conntrack_get(skb_nfct(src));
4154 #endif
4155 #if IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TRACE) || defined(CONFIG_NF_TABLES)
4156         if (copy)
4157                 dst->nf_trace = src->nf_trace;
4158 #endif
4159 }
4160 
4161 static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src)
4162 {
4163 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
4164         nf_conntrack_put(skb_nfct(dst));
4165 #endif
4166         __nf_copy(dst, src, true);
4167 }
4168 
4169 #ifdef CONFIG_NETWORK_SECMARK
4170 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
4171 {
4172         to->secmark = from->secmark;
4173 }
4174 
4175 static inline void skb_init_secmark(struct sk_buff *skb)
4176 {
4177         skb->secmark = 0;
4178 }
4179 #else
4180 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
4181 { }
4182 
4183 static inline void skb_init_secmark(struct sk_buff *skb)
4184 { }
4185 #endif
4186 
4187 static inline int secpath_exists(const struct sk_buff *skb)
4188 {
4189 #ifdef CONFIG_XFRM
4190         return skb_ext_exist(skb, SKB_EXT_SEC_PATH);
4191 #else
4192         return 0;
4193 #endif
4194 }
4195 
4196 static inline bool skb_irq_freeable(const struct sk_buff *skb)
4197 {
4198         return !skb->destructor &&
4199                 !secpath_exists(skb) &&
4200                 !skb_nfct(skb) &&
4201                 !skb->_skb_refdst &&
4202                 !skb_has_frag_list(skb);
4203 }
4204 
4205 static inline void skb_set_queue_mapping(struct sk_buff *skb, u16 queue_mapping)
4206 {
4207         skb->queue_mapping = queue_mapping;
4208 }
4209 
4210 static inline u16 skb_get_queue_mapping(const struct sk_buff *skb)
4211 {
4212         return skb->queue_mapping;
4213 }
4214 
4215 static inline void skb_copy_queue_mapping(struct sk_buff *to, const struct sk_buff *from)
4216 {
4217         to->queue_mapping = from->queue_mapping;
4218 }
4219 
4220 static inline void skb_record_rx_queue(struct sk_buff *skb, u16 rx_queue)
4221 {
4222         skb->queue_mapping = rx_queue + 1;
4223 }
4224 
4225 static inline u16 skb_get_rx_queue(const struct sk_buff *skb)
4226 {
4227         return skb->queue_mapping - 1;
4228 }
4229 
4230 static inline bool skb_rx_queue_recorded(const struct sk_buff *skb)
4231 {
4232         return skb->queue_mapping != 0;
4233 }
4234 
4235 static inline void skb_set_dst_pending_confirm(struct sk_buff *skb, u32 val)
4236 {
4237         skb->dst_pending_confirm = val;
4238 }
4239 
4240 static inline bool skb_get_dst_pending_confirm(const struct sk_buff *skb)
4241 {
4242         return skb->dst_pending_confirm != 0;
4243 }
4244 
4245 static inline struct sec_path *skb_sec_path(const struct sk_buff *skb)
4246 {
4247 #ifdef CONFIG_XFRM
4248         return skb_ext_find(skb, SKB_EXT_SEC_PATH);
4249 #else
4250         return NULL;
4251 #endif
4252 }
4253 
4254 /* Keeps track of mac header offset relative to skb->head.
4255  * It is useful for TSO of Tunneling protocol. e.g. GRE.
4256  * For non-tunnel skb it points to skb_mac_header() and for
4257  * tunnel skb it points to outer mac header.
4258  * Keeps track of level of encapsulation of network headers.
4259  */
4260 struct skb_gso_cb {
4261         union {
4262                 int     mac_offset;
4263                 int     data_offset;
4264         };
4265         int     encap_level;
4266         __wsum  csum;
4267         __u16   csum_start;
4268 };
4269 #define SKB_SGO_CB_OFFSET       32
4270 #define SKB_GSO_CB(skb) ((struct skb_gso_cb *)((skb)->cb + SKB_SGO_CB_OFFSET))
4271 
4272 static inline int skb_tnl_header_len(const struct sk_buff *inner_skb)
4273 {
4274         return (skb_mac_header(inner_skb) - inner_skb->head) -
4275                 SKB_GSO_CB(inner_skb)->mac_offset;
4276 }
4277 
4278 static inline int gso_pskb_expand_head(struct sk_buff *skb, int extra)
4279 {
4280         int new_headroom, headroom;
4281         int ret;
4282 
4283         headroom = skb_headroom(skb);
4284         ret = pskb_expand_head(skb, extra, 0, GFP_ATOMIC);
4285         if (ret)
4286                 return ret;
4287 
4288         new_headroom = skb_headroom(skb);
4289         SKB_GSO_CB(skb)->mac_offset += (new_headroom - headroom);
4290         return 0;
4291 }
4292 
4293 static inline void gso_reset_checksum(struct sk_buff *skb, __wsum res)
4294 {
4295         /* Do not update partial checksums if remote checksum is enabled. */
4296         if (skb->remcsum_offload)
4297                 return;
4298 
4299         SKB_GSO_CB(skb)->csum = res;
4300         SKB_GSO_CB(skb)->csum_start = skb_checksum_start(skb) - skb->head;
4301 }
4302 
4303 /* Compute the checksum for a gso segment. First compute the checksum value
4304  * from the start of transport header to SKB_GSO_CB(skb)->csum_start, and
4305  * then add in skb->csum (checksum from csum_start to end of packet).
4306  * skb->csum and csum_start are then updated to reflect the checksum of the
4307  * resultant packet starting from the transport header-- the resultant checksum
4308  * is in the res argument (i.e. normally zero or ~ of checksum of a pseudo
4309  * header.
4310  */
4311 static inline __sum16 gso_make_checksum(struct sk_buff *skb, __wsum res)
4312 {
4313         unsigned char *csum_start = skb_transport_header(skb);
4314         int plen = (skb->head + SKB_GSO_CB(skb)->csum_start) - csum_start;
4315         __wsum partial = SKB_GSO_CB(skb)->csum;
4316 
4317         SKB_GSO_CB(skb)->csum = res;
4318         SKB_GSO_CB(skb)->csum_start = csum_start - skb->head;
4319 
4320         return csum_fold(csum_partial(csum_start, plen, partial));
4321 }
4322 
4323 static inline bool skb_is_gso(const struct sk_buff *skb)
4324 {
4325         return skb_shinfo(skb)->gso_size;
4326 }
4327 
4328 /* Note: Should be called only if skb_is_gso(skb) is true */
4329 static inline bool skb_is_gso_v6(const struct sk_buff *skb)
4330 {
4331         return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6;
4332 }
4333 
4334 /* Note: Should be called only if skb_is_gso(skb) is true */
4335 static inline bool skb_is_gso_sctp(const struct sk_buff *skb)
4336 {
4337         return skb_shinfo(skb)->gso_type & SKB_GSO_SCTP;
4338 }
4339 
4340 /* Note: Should be called only if skb_is_gso(skb) is true */
4341 static inline bool skb_is_gso_tcp(const struct sk_buff *skb)
4342 {
4343         return skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6);
4344 }
4345 
4346 static inline void skb_gso_reset(struct sk_buff *skb)
4347 {
4348         skb_shinfo(skb)->gso_size = 0;
4349         skb_shinfo(skb)->gso_segs = 0;
4350         skb_shinfo(skb)->gso_type = 0;
4351 }
4352 
4353 static inline void skb_increase_gso_size(struct skb_shared_info *shinfo,
4354                                          u16 increment)
4355 {
4356         if (WARN_ON_ONCE(shinfo->gso_size == GSO_BY_FRAGS))
4357                 return;
4358         shinfo->gso_size += increment;
4359 }
4360 
4361 static inline void skb_decrease_gso_size(struct skb_shared_info *shinfo,
4362                                          u16 decrement)
4363 {
4364         if (WARN_ON_ONCE(shinfo->gso_size == GSO_BY_FRAGS))
4365                 return;
4366         shinfo->gso_size -= decrement;
4367 }
4368 
4369 void __skb_warn_lro_forwarding(const struct sk_buff *skb);
4370 
4371 static inline bool skb_warn_if_lro(const struct sk_buff *skb)
4372 {
4373         /* LRO sets gso_size but not gso_type, whereas if GSO is really
4374          * wanted then gso_type will be set. */
4375         const struct skb_shared_info *shinfo = skb_shinfo(skb);
4376 
4377         if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 &&
4378             unlikely(shinfo->gso_type == 0)) {
4379                 __skb_warn_lro_forwarding(skb);
4380                 return true;
4381         }
4382         return false;
4383 }
4384 
4385 static inline void skb_forward_csum(struct sk_buff *skb)
4386 {
4387         /* Unfortunately we don't support this one.  Any brave souls? */
4388         if (skb->ip_summed == CHECKSUM_COMPLETE)
4389                 skb->ip_summed = CHECKSUM_NONE;
4390 }
4391 
4392 /**
4393  * skb_checksum_none_assert - make sure skb ip_summed is CHECKSUM_NONE
4394  * @skb: skb to check
4395  *
4396  * fresh skbs have their ip_summed set to CHECKSUM_NONE.
4397  * Instead of forcing ip_summed to CHECKSUM_NONE, we can
4398  * use this helper, to document places where we make this assertion.
4399  */
4400 static inline void skb_checksum_none_assert(const struct sk_buff *skb)
4401 {
4402 #ifdef DEBUG
4403         BUG_ON(skb->ip_summed != CHECKSUM_NONE);
4404 #endif
4405 }
4406 
4407 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off);
4408 
4409 int skb_checksum_setup(struct sk_buff *skb, bool recalculate);
4410 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
4411                                      unsigned int transport_len,
4412                                      __sum16(*skb_chkf)(struct sk_buff *skb));
4413 
4414 /**
4415  * skb_head_is_locked - Determine if the skb->head is locked down
4416  * @skb: skb to check
4417  *
4418  * The head on skbs build around a head frag can be removed if they are
4419  * not cloned.  This function returns true if the skb head is locked down
4420  * due to either being allocated via kmalloc, or by being a clone with
4421  * multiple references to the head.
4422  */
4423 static inline bool skb_head_is_locked(const struct sk_buff *skb)
4424 {
4425         return !skb->head_frag || skb_cloned(skb);
4426 }
4427 
4428 /* Local Checksum Offload.
4429  * Compute outer checksum based on the assumption that the
4430  * inner checksum will be offloaded later.
4431  * See Documentation/networking/checksum-offloads.rst for
4432  * explanation of how this works.
4433  * Fill in outer checksum adjustment (e.g. with sum of outer
4434  * pseudo-header) before calling.
4435  * Also ensure that inner checksum is in linear data area.
4436  */
4437 static inline __wsum lco_csum(struct sk_buff *skb)
4438 {
4439         unsigned char *csum_start = skb_checksum_start(skb);
4440         unsigned char *l4_hdr = skb_transport_header(skb);
4441         __wsum partial;
4442 
4443         /* Start with complement of inner checksum adjustment */
4444         partial = ~csum_unfold(*(__force __sum16 *)(csum_start +
4445                                                     skb->csum_offset));
4446 
4447         /* Add in checksum of our headers (incl. outer checksum
4448          * adjustment filled in by caller) and return result.
4449          */
4450         return csum_partial(l4_hdr, csum_start - l4_hdr, partial);
4451 }
4452 
4453 #endif  /* __KERNEL__ */
4454 #endif  /* _LINUX_SKBUFF_H */
4455 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp