~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/core/skbuff.c

Version: ~ [ linux-5.10-rc5 ] ~ [ linux-5.9.10 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.79 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.159 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.208 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.245 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.245 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *      Routines having to do with the 'struct sk_buff' memory handlers.
  3  *
  4  *      Authors:        Alan Cox <alan@lxorguk.ukuu.org.uk>
  5  *                      Florian La Roche <rzsfl@rz.uni-sb.de>
  6  *
  7  *      Fixes:
  8  *              Alan Cox        :       Fixed the worst of the load
  9  *                                      balancer bugs.
 10  *              Dave Platt      :       Interrupt stacking fix.
 11  *      Richard Kooijman        :       Timestamp fixes.
 12  *              Alan Cox        :       Changed buffer format.
 13  *              Alan Cox        :       destructor hook for AF_UNIX etc.
 14  *              Linus Torvalds  :       Better skb_clone.
 15  *              Alan Cox        :       Added skb_copy.
 16  *              Alan Cox        :       Added all the changed routines Linus
 17  *                                      only put in the headers
 18  *              Ray VanTassle   :       Fixed --skb->lock in free
 19  *              Alan Cox        :       skb_copy copy arp field
 20  *              Andi Kleen      :       slabified it.
 21  *              Robert Olsson   :       Removed skb_head_pool
 22  *
 23  *      NOTE:
 24  *              The __skb_ routines should be called with interrupts
 25  *      disabled, or you better be *real* sure that the operation is atomic
 26  *      with respect to whatever list is being frobbed (e.g. via lock_sock()
 27  *      or via disabling bottom half handlers, etc).
 28  *
 29  *      This program is free software; you can redistribute it and/or
 30  *      modify it under the terms of the GNU General Public License
 31  *      as published by the Free Software Foundation; either version
 32  *      2 of the License, or (at your option) any later version.
 33  */
 34 
 35 /*
 36  *      The functions in this file will not compile correctly with gcc 2.4.x
 37  */
 38 
 39 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 40 
 41 #include <linux/module.h>
 42 #include <linux/types.h>
 43 #include <linux/kernel.h>
 44 #include <linux/mm.h>
 45 #include <linux/interrupt.h>
 46 #include <linux/in.h>
 47 #include <linux/inet.h>
 48 #include <linux/slab.h>
 49 #include <linux/tcp.h>
 50 #include <linux/udp.h>
 51 #include <linux/sctp.h>
 52 #include <linux/netdevice.h>
 53 #ifdef CONFIG_NET_CLS_ACT
 54 #include <net/pkt_sched.h>
 55 #endif
 56 #include <linux/string.h>
 57 #include <linux/skbuff.h>
 58 #include <linux/splice.h>
 59 #include <linux/cache.h>
 60 #include <linux/rtnetlink.h>
 61 #include <linux/init.h>
 62 #include <linux/scatterlist.h>
 63 #include <linux/errqueue.h>
 64 #include <linux/prefetch.h>
 65 #include <linux/if_vlan.h>
 66 
 67 #include <net/protocol.h>
 68 #include <net/dst.h>
 69 #include <net/sock.h>
 70 #include <net/checksum.h>
 71 #include <net/ip6_checksum.h>
 72 #include <net/xfrm.h>
 73 
 74 #include <linux/uaccess.h>
 75 #include <trace/events/skb.h>
 76 #include <linux/highmem.h>
 77 #include <linux/capability.h>
 78 #include <linux/user_namespace.h>
 79 
 80 struct kmem_cache *skbuff_head_cache __ro_after_init;
 81 static struct kmem_cache *skbuff_fclone_cache __ro_after_init;
 82 int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
 83 EXPORT_SYMBOL(sysctl_max_skb_frags);
 84 
 85 /**
 86  *      skb_panic - private function for out-of-line support
 87  *      @skb:   buffer
 88  *      @sz:    size
 89  *      @addr:  address
 90  *      @msg:   skb_over_panic or skb_under_panic
 91  *
 92  *      Out-of-line support for skb_put() and skb_push().
 93  *      Called via the wrapper skb_over_panic() or skb_under_panic().
 94  *      Keep out of line to prevent kernel bloat.
 95  *      __builtin_return_address is not used because it is not always reliable.
 96  */
 97 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
 98                       const char msg[])
 99 {
100         pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n",
101                  msg, addr, skb->len, sz, skb->head, skb->data,
102                  (unsigned long)skb->tail, (unsigned long)skb->end,
103                  skb->dev ? skb->dev->name : "<NULL>");
104         BUG();
105 }
106 
107 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
108 {
109         skb_panic(skb, sz, addr, __func__);
110 }
111 
112 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
113 {
114         skb_panic(skb, sz, addr, __func__);
115 }
116 
117 /*
118  * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
119  * the caller if emergency pfmemalloc reserves are being used. If it is and
120  * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
121  * may be used. Otherwise, the packet data may be discarded until enough
122  * memory is free
123  */
124 #define kmalloc_reserve(size, gfp, node, pfmemalloc) \
125          __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc)
126 
127 static void *__kmalloc_reserve(size_t size, gfp_t flags, int node,
128                                unsigned long ip, bool *pfmemalloc)
129 {
130         void *obj;
131         bool ret_pfmemalloc = false;
132 
133         /*
134          * Try a regular allocation, when that fails and we're not entitled
135          * to the reserves, fail.
136          */
137         obj = kmalloc_node_track_caller(size,
138                                         flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
139                                         node);
140         if (obj || !(gfp_pfmemalloc_allowed(flags)))
141                 goto out;
142 
143         /* Try again but now we are using pfmemalloc reserves */
144         ret_pfmemalloc = true;
145         obj = kmalloc_node_track_caller(size, flags, node);
146 
147 out:
148         if (pfmemalloc)
149                 *pfmemalloc = ret_pfmemalloc;
150 
151         return obj;
152 }
153 
154 /*      Allocate a new skbuff. We do this ourselves so we can fill in a few
155  *      'private' fields and also do memory statistics to find all the
156  *      [BEEP] leaks.
157  *
158  */
159 
160 /**
161  *      __alloc_skb     -       allocate a network buffer
162  *      @size: size to allocate
163  *      @gfp_mask: allocation mask
164  *      @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
165  *              instead of head cache and allocate a cloned (child) skb.
166  *              If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
167  *              allocations in case the data is required for writeback
168  *      @node: numa node to allocate memory on
169  *
170  *      Allocate a new &sk_buff. The returned buffer has no headroom and a
171  *      tail room of at least size bytes. The object has a reference count
172  *      of one. The return is the buffer. On a failure the return is %NULL.
173  *
174  *      Buffers may only be allocated from interrupts using a @gfp_mask of
175  *      %GFP_ATOMIC.
176  */
177 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
178                             int flags, int node)
179 {
180         struct kmem_cache *cache;
181         struct skb_shared_info *shinfo;
182         struct sk_buff *skb;
183         u8 *data;
184         bool pfmemalloc;
185 
186         cache = (flags & SKB_ALLOC_FCLONE)
187                 ? skbuff_fclone_cache : skbuff_head_cache;
188 
189         if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
190                 gfp_mask |= __GFP_MEMALLOC;
191 
192         /* Get the HEAD */
193         skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
194         if (!skb)
195                 goto out;
196         prefetchw(skb);
197 
198         /* We do our best to align skb_shared_info on a separate cache
199          * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
200          * aligned memory blocks, unless SLUB/SLAB debug is enabled.
201          * Both skb->head and skb_shared_info are cache line aligned.
202          */
203         size = SKB_DATA_ALIGN(size);
204         size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
205         data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
206         if (!data)
207                 goto nodata;
208         /* kmalloc(size) might give us more room than requested.
209          * Put skb_shared_info exactly at the end of allocated zone,
210          * to allow max possible filling before reallocation.
211          */
212         size = SKB_WITH_OVERHEAD(ksize(data));
213         prefetchw(data + size);
214 
215         /*
216          * Only clear those fields we need to clear, not those that we will
217          * actually initialise below. Hence, don't put any more fields after
218          * the tail pointer in struct sk_buff!
219          */
220         memset(skb, 0, offsetof(struct sk_buff, tail));
221         /* Account for allocated memory : skb + skb->head */
222         skb->truesize = SKB_TRUESIZE(size);
223         skb->pfmemalloc = pfmemalloc;
224         refcount_set(&skb->users, 1);
225         skb->head = data;
226         skb->data = data;
227         skb_reset_tail_pointer(skb);
228         skb->end = skb->tail + size;
229         skb->mac_header = (typeof(skb->mac_header))~0U;
230         skb->transport_header = (typeof(skb->transport_header))~0U;
231 
232         /* make sure we initialize shinfo sequentially */
233         shinfo = skb_shinfo(skb);
234         memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
235         atomic_set(&shinfo->dataref, 1);
236 
237         if (flags & SKB_ALLOC_FCLONE) {
238                 struct sk_buff_fclones *fclones;
239 
240                 fclones = container_of(skb, struct sk_buff_fclones, skb1);
241 
242                 skb->fclone = SKB_FCLONE_ORIG;
243                 refcount_set(&fclones->fclone_ref, 1);
244 
245                 fclones->skb2.fclone = SKB_FCLONE_CLONE;
246         }
247 out:
248         return skb;
249 nodata:
250         kmem_cache_free(cache, skb);
251         skb = NULL;
252         goto out;
253 }
254 EXPORT_SYMBOL(__alloc_skb);
255 
256 /**
257  * __build_skb - build a network buffer
258  * @data: data buffer provided by caller
259  * @frag_size: size of data, or 0 if head was kmalloced
260  *
261  * Allocate a new &sk_buff. Caller provides space holding head and
262  * skb_shared_info. @data must have been allocated by kmalloc() only if
263  * @frag_size is 0, otherwise data should come from the page allocator
264  *  or vmalloc()
265  * The return is the new skb buffer.
266  * On a failure the return is %NULL, and @data is not freed.
267  * Notes :
268  *  Before IO, driver allocates only data buffer where NIC put incoming frame
269  *  Driver should add room at head (NET_SKB_PAD) and
270  *  MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
271  *  After IO, driver calls build_skb(), to allocate sk_buff and populate it
272  *  before giving packet to stack.
273  *  RX rings only contains data buffers, not full skbs.
274  */
275 struct sk_buff *__build_skb(void *data, unsigned int frag_size)
276 {
277         struct skb_shared_info *shinfo;
278         struct sk_buff *skb;
279         unsigned int size = frag_size ? : ksize(data);
280 
281         skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
282         if (!skb)
283                 return NULL;
284 
285         size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
286 
287         memset(skb, 0, offsetof(struct sk_buff, tail));
288         skb->truesize = SKB_TRUESIZE(size);
289         refcount_set(&skb->users, 1);
290         skb->head = data;
291         skb->data = data;
292         skb_reset_tail_pointer(skb);
293         skb->end = skb->tail + size;
294         skb->mac_header = (typeof(skb->mac_header))~0U;
295         skb->transport_header = (typeof(skb->transport_header))~0U;
296 
297         /* make sure we initialize shinfo sequentially */
298         shinfo = skb_shinfo(skb);
299         memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
300         atomic_set(&shinfo->dataref, 1);
301 
302         return skb;
303 }
304 
305 /* build_skb() is wrapper over __build_skb(), that specifically
306  * takes care of skb->head and skb->pfmemalloc
307  * This means that if @frag_size is not zero, then @data must be backed
308  * by a page fragment, not kmalloc() or vmalloc()
309  */
310 struct sk_buff *build_skb(void *data, unsigned int frag_size)
311 {
312         struct sk_buff *skb = __build_skb(data, frag_size);
313 
314         if (skb && frag_size) {
315                 skb->head_frag = 1;
316                 if (page_is_pfmemalloc(virt_to_head_page(data)))
317                         skb->pfmemalloc = 1;
318         }
319         return skb;
320 }
321 EXPORT_SYMBOL(build_skb);
322 
323 #define NAPI_SKB_CACHE_SIZE     64
324 
325 struct napi_alloc_cache {
326         struct page_frag_cache page;
327         unsigned int skb_count;
328         void *skb_cache[NAPI_SKB_CACHE_SIZE];
329 };
330 
331 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
332 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
333 
334 static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
335 {
336         struct page_frag_cache *nc;
337         unsigned long flags;
338         void *data;
339 
340         local_irq_save(flags);
341         nc = this_cpu_ptr(&netdev_alloc_cache);
342         data = page_frag_alloc(nc, fragsz, gfp_mask);
343         local_irq_restore(flags);
344         return data;
345 }
346 
347 /**
348  * netdev_alloc_frag - allocate a page fragment
349  * @fragsz: fragment size
350  *
351  * Allocates a frag from a page for receive buffer.
352  * Uses GFP_ATOMIC allocations.
353  */
354 void *netdev_alloc_frag(unsigned int fragsz)
355 {
356         return __netdev_alloc_frag(fragsz, GFP_ATOMIC);
357 }
358 EXPORT_SYMBOL(netdev_alloc_frag);
359 
360 static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
361 {
362         struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
363 
364         return page_frag_alloc(&nc->page, fragsz, gfp_mask);
365 }
366 
367 void *napi_alloc_frag(unsigned int fragsz)
368 {
369         return __napi_alloc_frag(fragsz, GFP_ATOMIC);
370 }
371 EXPORT_SYMBOL(napi_alloc_frag);
372 
373 /**
374  *      __netdev_alloc_skb - allocate an skbuff for rx on a specific device
375  *      @dev: network device to receive on
376  *      @len: length to allocate
377  *      @gfp_mask: get_free_pages mask, passed to alloc_skb
378  *
379  *      Allocate a new &sk_buff and assign it a usage count of one. The
380  *      buffer has NET_SKB_PAD headroom built in. Users should allocate
381  *      the headroom they think they need without accounting for the
382  *      built in space. The built in space is used for optimisations.
383  *
384  *      %NULL is returned if there is no free memory.
385  */
386 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len,
387                                    gfp_t gfp_mask)
388 {
389         struct page_frag_cache *nc;
390         unsigned long flags;
391         struct sk_buff *skb;
392         bool pfmemalloc;
393         void *data;
394 
395         len += NET_SKB_PAD;
396 
397         if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
398             (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
399                 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
400                 if (!skb)
401                         goto skb_fail;
402                 goto skb_success;
403         }
404 
405         len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
406         len = SKB_DATA_ALIGN(len);
407 
408         if (sk_memalloc_socks())
409                 gfp_mask |= __GFP_MEMALLOC;
410 
411         local_irq_save(flags);
412 
413         nc = this_cpu_ptr(&netdev_alloc_cache);
414         data = page_frag_alloc(nc, len, gfp_mask);
415         pfmemalloc = nc->pfmemalloc;
416 
417         local_irq_restore(flags);
418 
419         if (unlikely(!data))
420                 return NULL;
421 
422         skb = __build_skb(data, len);
423         if (unlikely(!skb)) {
424                 skb_free_frag(data);
425                 return NULL;
426         }
427 
428         /* use OR instead of assignment to avoid clearing of bits in mask */
429         if (pfmemalloc)
430                 skb->pfmemalloc = 1;
431         skb->head_frag = 1;
432 
433 skb_success:
434         skb_reserve(skb, NET_SKB_PAD);
435         skb->dev = dev;
436 
437 skb_fail:
438         return skb;
439 }
440 EXPORT_SYMBOL(__netdev_alloc_skb);
441 
442 /**
443  *      __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance
444  *      @napi: napi instance this buffer was allocated for
445  *      @len: length to allocate
446  *      @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages
447  *
448  *      Allocate a new sk_buff for use in NAPI receive.  This buffer will
449  *      attempt to allocate the head from a special reserved region used
450  *      only for NAPI Rx allocation.  By doing this we can save several
451  *      CPU cycles by avoiding having to disable and re-enable IRQs.
452  *
453  *      %NULL is returned if there is no free memory.
454  */
455 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
456                                  gfp_t gfp_mask)
457 {
458         struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
459         struct sk_buff *skb;
460         void *data;
461 
462         len += NET_SKB_PAD + NET_IP_ALIGN;
463 
464         if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
465             (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
466                 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
467                 if (!skb)
468                         goto skb_fail;
469                 goto skb_success;
470         }
471 
472         len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
473         len = SKB_DATA_ALIGN(len);
474 
475         if (sk_memalloc_socks())
476                 gfp_mask |= __GFP_MEMALLOC;
477 
478         data = page_frag_alloc(&nc->page, len, gfp_mask);
479         if (unlikely(!data))
480                 return NULL;
481 
482         skb = __build_skb(data, len);
483         if (unlikely(!skb)) {
484                 skb_free_frag(data);
485                 return NULL;
486         }
487 
488         /* use OR instead of assignment to avoid clearing of bits in mask */
489         if (nc->page.pfmemalloc)
490                 skb->pfmemalloc = 1;
491         skb->head_frag = 1;
492 
493 skb_success:
494         skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
495         skb->dev = napi->dev;
496 
497 skb_fail:
498         return skb;
499 }
500 EXPORT_SYMBOL(__napi_alloc_skb);
501 
502 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
503                      int size, unsigned int truesize)
504 {
505         skb_fill_page_desc(skb, i, page, off, size);
506         skb->len += size;
507         skb->data_len += size;
508         skb->truesize += truesize;
509 }
510 EXPORT_SYMBOL(skb_add_rx_frag);
511 
512 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
513                           unsigned int truesize)
514 {
515         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
516 
517         skb_frag_size_add(frag, size);
518         skb->len += size;
519         skb->data_len += size;
520         skb->truesize += truesize;
521 }
522 EXPORT_SYMBOL(skb_coalesce_rx_frag);
523 
524 static void skb_drop_list(struct sk_buff **listp)
525 {
526         kfree_skb_list(*listp);
527         *listp = NULL;
528 }
529 
530 static inline void skb_drop_fraglist(struct sk_buff *skb)
531 {
532         skb_drop_list(&skb_shinfo(skb)->frag_list);
533 }
534 
535 static void skb_clone_fraglist(struct sk_buff *skb)
536 {
537         struct sk_buff *list;
538 
539         skb_walk_frags(skb, list)
540                 skb_get(list);
541 }
542 
543 static void skb_free_head(struct sk_buff *skb)
544 {
545         unsigned char *head = skb->head;
546 
547         if (skb->head_frag)
548                 skb_free_frag(head);
549         else
550                 kfree(head);
551 }
552 
553 static void skb_release_data(struct sk_buff *skb)
554 {
555         struct skb_shared_info *shinfo = skb_shinfo(skb);
556         int i;
557 
558         if (skb->cloned &&
559             atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
560                               &shinfo->dataref))
561                 return;
562 
563         for (i = 0; i < shinfo->nr_frags; i++)
564                 __skb_frag_unref(&shinfo->frags[i]);
565 
566         if (shinfo->frag_list)
567                 kfree_skb_list(shinfo->frag_list);
568 
569         skb_zcopy_clear(skb, true);
570         skb_free_head(skb);
571 }
572 
573 /*
574  *      Free an skbuff by memory without cleaning the state.
575  */
576 static void kfree_skbmem(struct sk_buff *skb)
577 {
578         struct sk_buff_fclones *fclones;
579 
580         switch (skb->fclone) {
581         case SKB_FCLONE_UNAVAILABLE:
582                 kmem_cache_free(skbuff_head_cache, skb);
583                 return;
584 
585         case SKB_FCLONE_ORIG:
586                 fclones = container_of(skb, struct sk_buff_fclones, skb1);
587 
588                 /* We usually free the clone (TX completion) before original skb
589                  * This test would have no chance to be true for the clone,
590                  * while here, branch prediction will be good.
591                  */
592                 if (refcount_read(&fclones->fclone_ref) == 1)
593                         goto fastpath;
594                 break;
595 
596         default: /* SKB_FCLONE_CLONE */
597                 fclones = container_of(skb, struct sk_buff_fclones, skb2);
598                 break;
599         }
600         if (!refcount_dec_and_test(&fclones->fclone_ref))
601                 return;
602 fastpath:
603         kmem_cache_free(skbuff_fclone_cache, fclones);
604 }
605 
606 void skb_release_head_state(struct sk_buff *skb)
607 {
608         skb_dst_drop(skb);
609         secpath_reset(skb);
610         if (skb->destructor) {
611                 WARN_ON(in_irq());
612                 skb->destructor(skb);
613         }
614 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
615         nf_conntrack_put(skb_nfct(skb));
616 #endif
617 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
618         nf_bridge_put(skb->nf_bridge);
619 #endif
620 }
621 
622 /* Free everything but the sk_buff shell. */
623 static void skb_release_all(struct sk_buff *skb)
624 {
625         skb_release_head_state(skb);
626         if (likely(skb->head))
627                 skb_release_data(skb);
628 }
629 
630 /**
631  *      __kfree_skb - private function
632  *      @skb: buffer
633  *
634  *      Free an sk_buff. Release anything attached to the buffer.
635  *      Clean the state. This is an internal helper function. Users should
636  *      always call kfree_skb
637  */
638 
639 void __kfree_skb(struct sk_buff *skb)
640 {
641         skb_release_all(skb);
642         kfree_skbmem(skb);
643 }
644 EXPORT_SYMBOL(__kfree_skb);
645 
646 /**
647  *      kfree_skb - free an sk_buff
648  *      @skb: buffer to free
649  *
650  *      Drop a reference to the buffer and free it if the usage count has
651  *      hit zero.
652  */
653 void kfree_skb(struct sk_buff *skb)
654 {
655         if (!skb_unref(skb))
656                 return;
657 
658         trace_kfree_skb(skb, __builtin_return_address(0));
659         __kfree_skb(skb);
660 }
661 EXPORT_SYMBOL(kfree_skb);
662 
663 void kfree_skb_list(struct sk_buff *segs)
664 {
665         while (segs) {
666                 struct sk_buff *next = segs->next;
667 
668                 kfree_skb(segs);
669                 segs = next;
670         }
671 }
672 EXPORT_SYMBOL(kfree_skb_list);
673 
674 /**
675  *      skb_tx_error - report an sk_buff xmit error
676  *      @skb: buffer that triggered an error
677  *
678  *      Report xmit error if a device callback is tracking this skb.
679  *      skb must be freed afterwards.
680  */
681 void skb_tx_error(struct sk_buff *skb)
682 {
683         skb_zcopy_clear(skb, true);
684 }
685 EXPORT_SYMBOL(skb_tx_error);
686 
687 /**
688  *      consume_skb - free an skbuff
689  *      @skb: buffer to free
690  *
691  *      Drop a ref to the buffer and free it if the usage count has hit zero
692  *      Functions identically to kfree_skb, but kfree_skb assumes that the frame
693  *      is being dropped after a failure and notes that
694  */
695 void consume_skb(struct sk_buff *skb)
696 {
697         if (!skb_unref(skb))
698                 return;
699 
700         trace_consume_skb(skb);
701         __kfree_skb(skb);
702 }
703 EXPORT_SYMBOL(consume_skb);
704 
705 /**
706  *      consume_stateless_skb - free an skbuff, assuming it is stateless
707  *      @skb: buffer to free
708  *
709  *      Alike consume_skb(), but this variant assumes that this is the last
710  *      skb reference and all the head states have been already dropped
711  */
712 void __consume_stateless_skb(struct sk_buff *skb)
713 {
714         trace_consume_skb(skb);
715         skb_release_data(skb);
716         kfree_skbmem(skb);
717 }
718 
719 void __kfree_skb_flush(void)
720 {
721         struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
722 
723         /* flush skb_cache if containing objects */
724         if (nc->skb_count) {
725                 kmem_cache_free_bulk(skbuff_head_cache, nc->skb_count,
726                                      nc->skb_cache);
727                 nc->skb_count = 0;
728         }
729 }
730 
731 static inline void _kfree_skb_defer(struct sk_buff *skb)
732 {
733         struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
734 
735         /* drop skb->head and call any destructors for packet */
736         skb_release_all(skb);
737 
738         /* record skb to CPU local list */
739         nc->skb_cache[nc->skb_count++] = skb;
740 
741 #ifdef CONFIG_SLUB
742         /* SLUB writes into objects when freeing */
743         prefetchw(skb);
744 #endif
745 
746         /* flush skb_cache if it is filled */
747         if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {
748                 kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_SIZE,
749                                      nc->skb_cache);
750                 nc->skb_count = 0;
751         }
752 }
753 void __kfree_skb_defer(struct sk_buff *skb)
754 {
755         _kfree_skb_defer(skb);
756 }
757 
758 void napi_consume_skb(struct sk_buff *skb, int budget)
759 {
760         if (unlikely(!skb))
761                 return;
762 
763         /* Zero budget indicate non-NAPI context called us, like netpoll */
764         if (unlikely(!budget)) {
765                 dev_consume_skb_any(skb);
766                 return;
767         }
768 
769         if (!skb_unref(skb))
770                 return;
771 
772         /* if reaching here SKB is ready to free */
773         trace_consume_skb(skb);
774 
775         /* if SKB is a clone, don't handle this case */
776         if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
777                 __kfree_skb(skb);
778                 return;
779         }
780 
781         _kfree_skb_defer(skb);
782 }
783 EXPORT_SYMBOL(napi_consume_skb);
784 
785 /* Make sure a field is enclosed inside headers_start/headers_end section */
786 #define CHECK_SKB_FIELD(field) \
787         BUILD_BUG_ON(offsetof(struct sk_buff, field) <          \
788                      offsetof(struct sk_buff, headers_start));  \
789         BUILD_BUG_ON(offsetof(struct sk_buff, field) >          \
790                      offsetof(struct sk_buff, headers_end));    \
791 
792 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
793 {
794         new->tstamp             = old->tstamp;
795         /* We do not copy old->sk */
796         new->dev                = old->dev;
797         memcpy(new->cb, old->cb, sizeof(old->cb));
798         skb_dst_copy(new, old);
799 #ifdef CONFIG_XFRM
800         new->sp                 = secpath_get(old->sp);
801 #endif
802         __nf_copy(new, old, false);
803 
804         /* Note : this field could be in headers_start/headers_end section
805          * It is not yet because we do not want to have a 16 bit hole
806          */
807         new->queue_mapping = old->queue_mapping;
808 
809         memcpy(&new->headers_start, &old->headers_start,
810                offsetof(struct sk_buff, headers_end) -
811                offsetof(struct sk_buff, headers_start));
812         CHECK_SKB_FIELD(protocol);
813         CHECK_SKB_FIELD(csum);
814         CHECK_SKB_FIELD(hash);
815         CHECK_SKB_FIELD(priority);
816         CHECK_SKB_FIELD(skb_iif);
817         CHECK_SKB_FIELD(vlan_proto);
818         CHECK_SKB_FIELD(vlan_tci);
819         CHECK_SKB_FIELD(transport_header);
820         CHECK_SKB_FIELD(network_header);
821         CHECK_SKB_FIELD(mac_header);
822         CHECK_SKB_FIELD(inner_protocol);
823         CHECK_SKB_FIELD(inner_transport_header);
824         CHECK_SKB_FIELD(inner_network_header);
825         CHECK_SKB_FIELD(inner_mac_header);
826         CHECK_SKB_FIELD(mark);
827 #ifdef CONFIG_NETWORK_SECMARK
828         CHECK_SKB_FIELD(secmark);
829 #endif
830 #ifdef CONFIG_NET_RX_BUSY_POLL
831         CHECK_SKB_FIELD(napi_id);
832 #endif
833 #ifdef CONFIG_XPS
834         CHECK_SKB_FIELD(sender_cpu);
835 #endif
836 #ifdef CONFIG_NET_SCHED
837         CHECK_SKB_FIELD(tc_index);
838 #endif
839 
840 }
841 
842 /*
843  * You should not add any new code to this function.  Add it to
844  * __copy_skb_header above instead.
845  */
846 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
847 {
848 #define C(x) n->x = skb->x
849 
850         n->next = n->prev = NULL;
851         n->sk = NULL;
852         __copy_skb_header(n, skb);
853 
854         C(len);
855         C(data_len);
856         C(mac_len);
857         n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
858         n->cloned = 1;
859         n->nohdr = 0;
860         n->peeked = 0;
861         C(pfmemalloc);
862         n->destructor = NULL;
863         C(tail);
864         C(end);
865         C(head);
866         C(head_frag);
867         C(data);
868         C(truesize);
869         refcount_set(&n->users, 1);
870 
871         atomic_inc(&(skb_shinfo(skb)->dataref));
872         skb->cloned = 1;
873 
874         return n;
875 #undef C
876 }
877 
878 /**
879  *      skb_morph       -       morph one skb into another
880  *      @dst: the skb to receive the contents
881  *      @src: the skb to supply the contents
882  *
883  *      This is identical to skb_clone except that the target skb is
884  *      supplied by the user.
885  *
886  *      The target skb is returned upon exit.
887  */
888 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
889 {
890         skb_release_all(dst);
891         return __skb_clone(dst, src);
892 }
893 EXPORT_SYMBOL_GPL(skb_morph);
894 
895 int mm_account_pinned_pages(struct mmpin *mmp, size_t size)
896 {
897         unsigned long max_pg, num_pg, new_pg, old_pg;
898         struct user_struct *user;
899 
900         if (capable(CAP_IPC_LOCK) || !size)
901                 return 0;
902 
903         num_pg = (size >> PAGE_SHIFT) + 2;      /* worst case */
904         max_pg = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
905         user = mmp->user ? : current_user();
906 
907         do {
908                 old_pg = atomic_long_read(&user->locked_vm);
909                 new_pg = old_pg + num_pg;
910                 if (new_pg > max_pg)
911                         return -ENOBUFS;
912         } while (atomic_long_cmpxchg(&user->locked_vm, old_pg, new_pg) !=
913                  old_pg);
914 
915         if (!mmp->user) {
916                 mmp->user = get_uid(user);
917                 mmp->num_pg = num_pg;
918         } else {
919                 mmp->num_pg += num_pg;
920         }
921 
922         return 0;
923 }
924 EXPORT_SYMBOL_GPL(mm_account_pinned_pages);
925 
926 void mm_unaccount_pinned_pages(struct mmpin *mmp)
927 {
928         if (mmp->user) {
929                 atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm);
930                 free_uid(mmp->user);
931         }
932 }
933 EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages);
934 
935 struct ubuf_info *sock_zerocopy_alloc(struct sock *sk, size_t size)
936 {
937         struct ubuf_info *uarg;
938         struct sk_buff *skb;
939 
940         WARN_ON_ONCE(!in_task());
941 
942         if (!sock_flag(sk, SOCK_ZEROCOPY))
943                 return NULL;
944 
945         skb = sock_omalloc(sk, 0, GFP_KERNEL);
946         if (!skb)
947                 return NULL;
948 
949         BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));
950         uarg = (void *)skb->cb;
951         uarg->mmp.user = NULL;
952 
953         if (mm_account_pinned_pages(&uarg->mmp, size)) {
954                 kfree_skb(skb);
955                 return NULL;
956         }
957 
958         uarg->callback = sock_zerocopy_callback;
959         uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
960         uarg->len = 1;
961         uarg->bytelen = size;
962         uarg->zerocopy = 1;
963         refcount_set(&uarg->refcnt, 1);
964         sock_hold(sk);
965 
966         return uarg;
967 }
968 EXPORT_SYMBOL_GPL(sock_zerocopy_alloc);
969 
970 static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg)
971 {
972         return container_of((void *)uarg, struct sk_buff, cb);
973 }
974 
975 struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size,
976                                         struct ubuf_info *uarg)
977 {
978         if (uarg) {
979                 const u32 byte_limit = 1 << 19;         /* limit to a few TSO */
980                 u32 bytelen, next;
981 
982                 /* realloc only when socket is locked (TCP, UDP cork),
983                  * so uarg->len and sk_zckey access is serialized
984                  */
985                 if (!sock_owned_by_user(sk)) {
986                         WARN_ON_ONCE(1);
987                         return NULL;
988                 }
989 
990                 bytelen = uarg->bytelen + size;
991                 if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) {
992                         /* TCP can create new skb to attach new uarg */
993                         if (sk->sk_type == SOCK_STREAM)
994                                 goto new_alloc;
995                         return NULL;
996                 }
997 
998                 next = (u32)atomic_read(&sk->sk_zckey);
999                 if ((u32)(uarg->id + uarg->len) == next) {
1000                         if (mm_account_pinned_pages(&uarg->mmp, size))
1001                                 return NULL;
1002                         uarg->len++;
1003                         uarg->bytelen = bytelen;
1004                         atomic_set(&sk->sk_zckey, ++next);
1005                         sock_zerocopy_get(uarg);
1006                         return uarg;
1007                 }
1008         }
1009 
1010 new_alloc:
1011         return sock_zerocopy_alloc(sk, size);
1012 }
1013 EXPORT_SYMBOL_GPL(sock_zerocopy_realloc);
1014 
1015 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len)
1016 {
1017         struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
1018         u32 old_lo, old_hi;
1019         u64 sum_len;
1020 
1021         old_lo = serr->ee.ee_info;
1022         old_hi = serr->ee.ee_data;
1023         sum_len = old_hi - old_lo + 1ULL + len;
1024 
1025         if (sum_len >= (1ULL << 32))
1026                 return false;
1027 
1028         if (lo != old_hi + 1)
1029                 return false;
1030 
1031         serr->ee.ee_data += len;
1032         return true;
1033 }
1034 
1035 void sock_zerocopy_callback(struct ubuf_info *uarg, bool success)
1036 {
1037         struct sk_buff *tail, *skb = skb_from_uarg(uarg);
1038         struct sock_exterr_skb *serr;
1039         struct sock *sk = skb->sk;
1040         struct sk_buff_head *q;
1041         unsigned long flags;
1042         u32 lo, hi;
1043         u16 len;
1044 
1045         mm_unaccount_pinned_pages(&uarg->mmp);
1046 
1047         /* if !len, there was only 1 call, and it was aborted
1048          * so do not queue a completion notification
1049          */
1050         if (!uarg->len || sock_flag(sk, SOCK_DEAD))
1051                 goto release;
1052 
1053         len = uarg->len;
1054         lo = uarg->id;
1055         hi = uarg->id + len - 1;
1056 
1057         serr = SKB_EXT_ERR(skb);
1058         memset(serr, 0, sizeof(*serr));
1059         serr->ee.ee_errno = 0;
1060         serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY;
1061         serr->ee.ee_data = hi;
1062         serr->ee.ee_info = lo;
1063         if (!success)
1064                 serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED;
1065 
1066         q = &sk->sk_error_queue;
1067         spin_lock_irqsave(&q->lock, flags);
1068         tail = skb_peek_tail(q);
1069         if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY ||
1070             !skb_zerocopy_notify_extend(tail, lo, len)) {
1071                 __skb_queue_tail(q, skb);
1072                 skb = NULL;
1073         }
1074         spin_unlock_irqrestore(&q->lock, flags);
1075 
1076         sk->sk_error_report(sk);
1077 
1078 release:
1079         consume_skb(skb);
1080         sock_put(sk);
1081 }
1082 EXPORT_SYMBOL_GPL(sock_zerocopy_callback);
1083 
1084 void sock_zerocopy_put(struct ubuf_info *uarg)
1085 {
1086         if (uarg && refcount_dec_and_test(&uarg->refcnt)) {
1087                 if (uarg->callback)
1088                         uarg->callback(uarg, uarg->zerocopy);
1089                 else
1090                         consume_skb(skb_from_uarg(uarg));
1091         }
1092 }
1093 EXPORT_SYMBOL_GPL(sock_zerocopy_put);
1094 
1095 void sock_zerocopy_put_abort(struct ubuf_info *uarg)
1096 {
1097         if (uarg) {
1098                 struct sock *sk = skb_from_uarg(uarg)->sk;
1099 
1100                 atomic_dec(&sk->sk_zckey);
1101                 uarg->len--;
1102 
1103                 sock_zerocopy_put(uarg);
1104         }
1105 }
1106 EXPORT_SYMBOL_GPL(sock_zerocopy_put_abort);
1107 
1108 extern int __zerocopy_sg_from_iter(struct sock *sk, struct sk_buff *skb,
1109                                    struct iov_iter *from, size_t length);
1110 
1111 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
1112                              struct msghdr *msg, int len,
1113                              struct ubuf_info *uarg)
1114 {
1115         struct ubuf_info *orig_uarg = skb_zcopy(skb);
1116         struct iov_iter orig_iter = msg->msg_iter;
1117         int err, orig_len = skb->len;
1118 
1119         /* An skb can only point to one uarg. This edge case happens when
1120          * TCP appends to an skb, but zerocopy_realloc triggered a new alloc.
1121          */
1122         if (orig_uarg && uarg != orig_uarg)
1123                 return -EEXIST;
1124 
1125         err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len);
1126         if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
1127                 struct sock *save_sk = skb->sk;
1128 
1129                 /* Streams do not free skb on error. Reset to prev state. */
1130                 msg->msg_iter = orig_iter;
1131                 skb->sk = sk;
1132                 ___pskb_trim(skb, orig_len);
1133                 skb->sk = save_sk;
1134                 return err;
1135         }
1136 
1137         skb_zcopy_set(skb, uarg);
1138         return skb->len - orig_len;
1139 }
1140 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);
1141 
1142 static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig,
1143                               gfp_t gfp_mask)
1144 {
1145         if (skb_zcopy(orig)) {
1146                 if (skb_zcopy(nskb)) {
1147                         /* !gfp_mask callers are verified to !skb_zcopy(nskb) */
1148                         if (!gfp_mask) {
1149                                 WARN_ON_ONCE(1);
1150                                 return -ENOMEM;
1151                         }
1152                         if (skb_uarg(nskb) == skb_uarg(orig))
1153                                 return 0;
1154                         if (skb_copy_ubufs(nskb, GFP_ATOMIC))
1155                                 return -EIO;
1156                 }
1157                 skb_zcopy_set(nskb, skb_uarg(orig));
1158         }
1159         return 0;
1160 }
1161 
1162 /**
1163  *      skb_copy_ubufs  -       copy userspace skb frags buffers to kernel
1164  *      @skb: the skb to modify
1165  *      @gfp_mask: allocation priority
1166  *
1167  *      This must be called on SKBTX_DEV_ZEROCOPY skb.
1168  *      It will copy all frags into kernel and drop the reference
1169  *      to userspace pages.
1170  *
1171  *      If this function is called from an interrupt gfp_mask() must be
1172  *      %GFP_ATOMIC.
1173  *
1174  *      Returns 0 on success or a negative error code on failure
1175  *      to allocate kernel memory to copy to.
1176  */
1177 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
1178 {
1179         int num_frags = skb_shinfo(skb)->nr_frags;
1180         struct page *page, *head = NULL;
1181         int i, new_frags;
1182         u32 d_off;
1183 
1184         if (skb_shared(skb) || skb_unclone(skb, gfp_mask))
1185                 return -EINVAL;
1186 
1187         if (!num_frags)
1188                 goto release;
1189 
1190         new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT;
1191         for (i = 0; i < new_frags; i++) {
1192                 page = alloc_page(gfp_mask);
1193                 if (!page) {
1194                         while (head) {
1195                                 struct page *next = (struct page *)page_private(head);
1196                                 put_page(head);
1197                                 head = next;
1198                         }
1199                         return -ENOMEM;
1200                 }
1201                 set_page_private(page, (unsigned long)head);
1202                 head = page;
1203         }
1204 
1205         page = head;
1206         d_off = 0;
1207         for (i = 0; i < num_frags; i++) {
1208                 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
1209                 u32 p_off, p_len, copied;
1210                 struct page *p;
1211                 u8 *vaddr;
1212 
1213                 skb_frag_foreach_page(f, f->page_offset, skb_frag_size(f),
1214                                       p, p_off, p_len, copied) {
1215                         u32 copy, done = 0;
1216                         vaddr = kmap_atomic(p);
1217 
1218                         while (done < p_len) {
1219                                 if (d_off == PAGE_SIZE) {
1220                                         d_off = 0;
1221                                         page = (struct page *)page_private(page);
1222                                 }
1223                                 copy = min_t(u32, PAGE_SIZE - d_off, p_len - done);
1224                                 memcpy(page_address(page) + d_off,
1225                                        vaddr + p_off + done, copy);
1226                                 done += copy;
1227                                 d_off += copy;
1228                         }
1229                         kunmap_atomic(vaddr);
1230                 }
1231         }
1232 
1233         /* skb frags release userspace buffers */
1234         for (i = 0; i < num_frags; i++)
1235                 skb_frag_unref(skb, i);
1236 
1237         /* skb frags point to kernel buffers */
1238         for (i = 0; i < new_frags - 1; i++) {
1239                 __skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE);
1240                 head = (struct page *)page_private(head);
1241         }
1242         __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off);
1243         skb_shinfo(skb)->nr_frags = new_frags;
1244 
1245 release:
1246         skb_zcopy_clear(skb, false);
1247         return 0;
1248 }
1249 EXPORT_SYMBOL_GPL(skb_copy_ubufs);
1250 
1251 /**
1252  *      skb_clone       -       duplicate an sk_buff
1253  *      @skb: buffer to clone
1254  *      @gfp_mask: allocation priority
1255  *
1256  *      Duplicate an &sk_buff. The new one is not owned by a socket. Both
1257  *      copies share the same packet data but not structure. The new
1258  *      buffer has a reference count of 1. If the allocation fails the
1259  *      function returns %NULL otherwise the new buffer is returned.
1260  *
1261  *      If this function is called from an interrupt gfp_mask() must be
1262  *      %GFP_ATOMIC.
1263  */
1264 
1265 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
1266 {
1267         struct sk_buff_fclones *fclones = container_of(skb,
1268                                                        struct sk_buff_fclones,
1269                                                        skb1);
1270         struct sk_buff *n;
1271 
1272         if (skb_orphan_frags(skb, gfp_mask))
1273                 return NULL;
1274 
1275         if (skb->fclone == SKB_FCLONE_ORIG &&
1276             refcount_read(&fclones->fclone_ref) == 1) {
1277                 n = &fclones->skb2;
1278                 refcount_set(&fclones->fclone_ref, 2);
1279         } else {
1280                 if (skb_pfmemalloc(skb))
1281                         gfp_mask |= __GFP_MEMALLOC;
1282 
1283                 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
1284                 if (!n)
1285                         return NULL;
1286 
1287                 n->fclone = SKB_FCLONE_UNAVAILABLE;
1288         }
1289 
1290         return __skb_clone(n, skb);
1291 }
1292 EXPORT_SYMBOL(skb_clone);
1293 
1294 static void skb_headers_offset_update(struct sk_buff *skb, int off)
1295 {
1296         /* Only adjust this if it actually is csum_start rather than csum */
1297         if (skb->ip_summed == CHECKSUM_PARTIAL)
1298                 skb->csum_start += off;
1299         /* {transport,network,mac}_header and tail are relative to skb->head */
1300         skb->transport_header += off;
1301         skb->network_header   += off;
1302         if (skb_mac_header_was_set(skb))
1303                 skb->mac_header += off;
1304         skb->inner_transport_header += off;
1305         skb->inner_network_header += off;
1306         skb->inner_mac_header += off;
1307 }
1308 
1309 static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
1310 {
1311         __copy_skb_header(new, old);
1312 
1313         skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
1314         skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
1315         skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
1316 }
1317 
1318 static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
1319 {
1320         if (skb_pfmemalloc(skb))
1321                 return SKB_ALLOC_RX;
1322         return 0;
1323 }
1324 
1325 /**
1326  *      skb_copy        -       create private copy of an sk_buff
1327  *      @skb: buffer to copy
1328  *      @gfp_mask: allocation priority
1329  *
1330  *      Make a copy of both an &sk_buff and its data. This is used when the
1331  *      caller wishes to modify the data and needs a private copy of the
1332  *      data to alter. Returns %NULL on failure or the pointer to the buffer
1333  *      on success. The returned buffer has a reference count of 1.
1334  *
1335  *      As by-product this function converts non-linear &sk_buff to linear
1336  *      one, so that &sk_buff becomes completely private and caller is allowed
1337  *      to modify all the data of returned buffer. This means that this
1338  *      function is not recommended for use in circumstances when only
1339  *      header is going to be modified. Use pskb_copy() instead.
1340  */
1341 
1342 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
1343 {
1344         int headerlen = skb_headroom(skb);
1345         unsigned int size = skb_end_offset(skb) + skb->data_len;
1346         struct sk_buff *n = __alloc_skb(size, gfp_mask,
1347                                         skb_alloc_rx_flag(skb), NUMA_NO_NODE);
1348 
1349         if (!n)
1350                 return NULL;
1351 
1352         /* Set the data pointer */
1353         skb_reserve(n, headerlen);
1354         /* Set the tail pointer and length */
1355         skb_put(n, skb->len);
1356 
1357         BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len));
1358 
1359         copy_skb_header(n, skb);
1360         return n;
1361 }
1362 EXPORT_SYMBOL(skb_copy);
1363 
1364 /**
1365  *      __pskb_copy_fclone      -  create copy of an sk_buff with private head.
1366  *      @skb: buffer to copy
1367  *      @headroom: headroom of new skb
1368  *      @gfp_mask: allocation priority
1369  *      @fclone: if true allocate the copy of the skb from the fclone
1370  *      cache instead of the head cache; it is recommended to set this
1371  *      to true for the cases where the copy will likely be cloned
1372  *
1373  *      Make a copy of both an &sk_buff and part of its data, located
1374  *      in header. Fragmented data remain shared. This is used when
1375  *      the caller wishes to modify only header of &sk_buff and needs
1376  *      private copy of the header to alter. Returns %NULL on failure
1377  *      or the pointer to the buffer on success.
1378  *      The returned buffer has a reference count of 1.
1379  */
1380 
1381 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
1382                                    gfp_t gfp_mask, bool fclone)
1383 {
1384         unsigned int size = skb_headlen(skb) + headroom;
1385         int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
1386         struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE);
1387 
1388         if (!n)
1389                 goto out;
1390 
1391         /* Set the data pointer */
1392         skb_reserve(n, headroom);
1393         /* Set the tail pointer and length */
1394         skb_put(n, skb_headlen(skb));
1395         /* Copy the bytes */
1396         skb_copy_from_linear_data(skb, n->data, n->len);
1397 
1398         n->truesize += skb->data_len;
1399         n->data_len  = skb->data_len;
1400         n->len       = skb->len;
1401 
1402         if (skb_shinfo(skb)->nr_frags) {
1403                 int i;
1404 
1405                 if (skb_orphan_frags(skb, gfp_mask) ||
1406                     skb_zerocopy_clone(n, skb, gfp_mask)) {
1407                         kfree_skb(n);
1408                         n = NULL;
1409                         goto out;
1410                 }
1411                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1412                         skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
1413                         skb_frag_ref(skb, i);
1414                 }
1415                 skb_shinfo(n)->nr_frags = i;
1416         }
1417 
1418         if (skb_has_frag_list(skb)) {
1419                 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
1420                 skb_clone_fraglist(n);
1421         }
1422 
1423         copy_skb_header(n, skb);
1424 out:
1425         return n;
1426 }
1427 EXPORT_SYMBOL(__pskb_copy_fclone);
1428 
1429 /**
1430  *      pskb_expand_head - reallocate header of &sk_buff
1431  *      @skb: buffer to reallocate
1432  *      @nhead: room to add at head
1433  *      @ntail: room to add at tail
1434  *      @gfp_mask: allocation priority
1435  *
1436  *      Expands (or creates identical copy, if @nhead and @ntail are zero)
1437  *      header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
1438  *      reference count of 1. Returns zero in the case of success or error,
1439  *      if expansion failed. In the last case, &sk_buff is not changed.
1440  *
1441  *      All the pointers pointing into skb header may change and must be
1442  *      reloaded after call to this function.
1443  */
1444 
1445 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
1446                      gfp_t gfp_mask)
1447 {
1448         int i, osize = skb_end_offset(skb);
1449         int size = osize + nhead + ntail;
1450         long off;
1451         u8 *data;
1452 
1453         BUG_ON(nhead < 0);
1454 
1455         BUG_ON(skb_shared(skb));
1456 
1457         size = SKB_DATA_ALIGN(size);
1458 
1459         if (skb_pfmemalloc(skb))
1460                 gfp_mask |= __GFP_MEMALLOC;
1461         data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
1462                                gfp_mask, NUMA_NO_NODE, NULL);
1463         if (!data)
1464                 goto nodata;
1465         size = SKB_WITH_OVERHEAD(ksize(data));
1466 
1467         /* Copy only real data... and, alas, header. This should be
1468          * optimized for the cases when header is void.
1469          */
1470         memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
1471 
1472         memcpy((struct skb_shared_info *)(data + size),
1473                skb_shinfo(skb),
1474                offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
1475 
1476         /*
1477          * if shinfo is shared we must drop the old head gracefully, but if it
1478          * is not we can just drop the old head and let the existing refcount
1479          * be since all we did is relocate the values
1480          */
1481         if (skb_cloned(skb)) {
1482                 if (skb_orphan_frags(skb, gfp_mask))
1483                         goto nofrags;
1484                 if (skb_zcopy(skb))
1485                         refcount_inc(&skb_uarg(skb)->refcnt);
1486                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1487                         skb_frag_ref(skb, i);
1488 
1489                 if (skb_has_frag_list(skb))
1490                         skb_clone_fraglist(skb);
1491 
1492                 skb_release_data(skb);
1493         } else {
1494                 skb_free_head(skb);
1495         }
1496         off = (data + nhead) - skb->head;
1497 
1498         skb->head     = data;
1499         skb->head_frag = 0;
1500         skb->data    += off;
1501 #ifdef NET_SKBUFF_DATA_USES_OFFSET
1502         skb->end      = size;
1503         off           = nhead;
1504 #else
1505         skb->end      = skb->head + size;
1506 #endif
1507         skb->tail             += off;
1508         skb_headers_offset_update(skb, nhead);
1509         skb->cloned   = 0;
1510         skb->hdr_len  = 0;
1511         skb->nohdr    = 0;
1512         atomic_set(&skb_shinfo(skb)->dataref, 1);
1513 
1514         skb_metadata_clear(skb);
1515 
1516         /* It is not generally safe to change skb->truesize.
1517          * For the moment, we really care of rx path, or
1518          * when skb is orphaned (not attached to a socket).
1519          */
1520         if (!skb->sk || skb->destructor == sock_edemux)
1521                 skb->truesize += size - osize;
1522 
1523         return 0;
1524 
1525 nofrags:
1526         kfree(data);
1527 nodata:
1528         return -ENOMEM;
1529 }
1530 EXPORT_SYMBOL(pskb_expand_head);
1531 
1532 /* Make private copy of skb with writable head and some headroom */
1533 
1534 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
1535 {
1536         struct sk_buff *skb2;
1537         int delta = headroom - skb_headroom(skb);
1538 
1539         if (delta <= 0)
1540                 skb2 = pskb_copy(skb, GFP_ATOMIC);
1541         else {
1542                 skb2 = skb_clone(skb, GFP_ATOMIC);
1543                 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
1544                                              GFP_ATOMIC)) {
1545                         kfree_skb(skb2);
1546                         skb2 = NULL;
1547                 }
1548         }
1549         return skb2;
1550 }
1551 EXPORT_SYMBOL(skb_realloc_headroom);
1552 
1553 /**
1554  *      skb_copy_expand -       copy and expand sk_buff
1555  *      @skb: buffer to copy
1556  *      @newheadroom: new free bytes at head
1557  *      @newtailroom: new free bytes at tail
1558  *      @gfp_mask: allocation priority
1559  *
1560  *      Make a copy of both an &sk_buff and its data and while doing so
1561  *      allocate additional space.
1562  *
1563  *      This is used when the caller wishes to modify the data and needs a
1564  *      private copy of the data to alter as well as more space for new fields.
1565  *      Returns %NULL on failure or the pointer to the buffer
1566  *      on success. The returned buffer has a reference count of 1.
1567  *
1568  *      You must pass %GFP_ATOMIC as the allocation priority if this function
1569  *      is called from an interrupt.
1570  */
1571 struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
1572                                 int newheadroom, int newtailroom,
1573                                 gfp_t gfp_mask)
1574 {
1575         /*
1576          *      Allocate the copy buffer
1577          */
1578         struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
1579                                         gfp_mask, skb_alloc_rx_flag(skb),
1580                                         NUMA_NO_NODE);
1581         int oldheadroom = skb_headroom(skb);
1582         int head_copy_len, head_copy_off;
1583 
1584         if (!n)
1585                 return NULL;
1586 
1587         skb_reserve(n, newheadroom);
1588 
1589         /* Set the tail pointer and length */
1590         skb_put(n, skb->len);
1591 
1592         head_copy_len = oldheadroom;
1593         head_copy_off = 0;
1594         if (newheadroom <= head_copy_len)
1595                 head_copy_len = newheadroom;
1596         else
1597                 head_copy_off = newheadroom - head_copy_len;
1598 
1599         /* Copy the linear header and data. */
1600         BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
1601                              skb->len + head_copy_len));
1602 
1603         copy_skb_header(n, skb);
1604 
1605         skb_headers_offset_update(n, newheadroom - oldheadroom);
1606 
1607         return n;
1608 }
1609 EXPORT_SYMBOL(skb_copy_expand);
1610 
1611 /**
1612  *      __skb_pad               -       zero pad the tail of an skb
1613  *      @skb: buffer to pad
1614  *      @pad: space to pad
1615  *      @free_on_error: free buffer on error
1616  *
1617  *      Ensure that a buffer is followed by a padding area that is zero
1618  *      filled. Used by network drivers which may DMA or transfer data
1619  *      beyond the buffer end onto the wire.
1620  *
1621  *      May return error in out of memory cases. The skb is freed on error
1622  *      if @free_on_error is true.
1623  */
1624 
1625 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error)
1626 {
1627         int err;
1628         int ntail;
1629 
1630         /* If the skbuff is non linear tailroom is always zero.. */
1631         if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
1632                 memset(skb->data+skb->len, 0, pad);
1633                 return 0;
1634         }
1635 
1636         ntail = skb->data_len + pad - (skb->end - skb->tail);
1637         if (likely(skb_cloned(skb) || ntail > 0)) {
1638                 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
1639                 if (unlikely(err))
1640                         goto free_skb;
1641         }
1642 
1643         /* FIXME: The use of this function with non-linear skb's really needs
1644          * to be audited.
1645          */
1646         err = skb_linearize(skb);
1647         if (unlikely(err))
1648                 goto free_skb;
1649 
1650         memset(skb->data + skb->len, 0, pad);
1651         return 0;
1652 
1653 free_skb:
1654         if (free_on_error)
1655                 kfree_skb(skb);
1656         return err;
1657 }
1658 EXPORT_SYMBOL(__skb_pad);
1659 
1660 /**
1661  *      pskb_put - add data to the tail of a potentially fragmented buffer
1662  *      @skb: start of the buffer to use
1663  *      @tail: tail fragment of the buffer to use
1664  *      @len: amount of data to add
1665  *
1666  *      This function extends the used data area of the potentially
1667  *      fragmented buffer. @tail must be the last fragment of @skb -- or
1668  *      @skb itself. If this would exceed the total buffer size the kernel
1669  *      will panic. A pointer to the first byte of the extra data is
1670  *      returned.
1671  */
1672 
1673 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
1674 {
1675         if (tail != skb) {
1676                 skb->data_len += len;
1677                 skb->len += len;
1678         }
1679         return skb_put(tail, len);
1680 }
1681 EXPORT_SYMBOL_GPL(pskb_put);
1682 
1683 /**
1684  *      skb_put - add data to a buffer
1685  *      @skb: buffer to use
1686  *      @len: amount of data to add
1687  *
1688  *      This function extends the used data area of the buffer. If this would
1689  *      exceed the total buffer size the kernel will panic. A pointer to the
1690  *      first byte of the extra data is returned.
1691  */
1692 void *skb_put(struct sk_buff *skb, unsigned int len)
1693 {
1694         void *tmp = skb_tail_pointer(skb);
1695         SKB_LINEAR_ASSERT(skb);
1696         skb->tail += len;
1697         skb->len  += len;
1698         if (unlikely(skb->tail > skb->end))
1699                 skb_over_panic(skb, len, __builtin_return_address(0));
1700         return tmp;
1701 }
1702 EXPORT_SYMBOL(skb_put);
1703 
1704 /**
1705  *      skb_push - add data to the start of a buffer
1706  *      @skb: buffer to use
1707  *      @len: amount of data to add
1708  *
1709  *      This function extends the used data area of the buffer at the buffer
1710  *      start. If this would exceed the total buffer headroom the kernel will
1711  *      panic. A pointer to the first byte of the extra data is returned.
1712  */
1713 void *skb_push(struct sk_buff *skb, unsigned int len)
1714 {
1715         skb->data -= len;
1716         skb->len  += len;
1717         if (unlikely(skb->data<skb->head))
1718                 skb_under_panic(skb, len, __builtin_return_address(0));
1719         return skb->data;
1720 }
1721 EXPORT_SYMBOL(skb_push);
1722 
1723 /**
1724  *      skb_pull - remove data from the start of a buffer
1725  *      @skb: buffer to use
1726  *      @len: amount of data to remove
1727  *
1728  *      This function removes data from the start of a buffer, returning
1729  *      the memory to the headroom. A pointer to the next data in the buffer
1730  *      is returned. Once the data has been pulled future pushes will overwrite
1731  *      the old data.
1732  */
1733 void *skb_pull(struct sk_buff *skb, unsigned int len)
1734 {
1735         return skb_pull_inline(skb, len);
1736 }
1737 EXPORT_SYMBOL(skb_pull);
1738 
1739 /**
1740  *      skb_trim - remove end from a buffer
1741  *      @skb: buffer to alter
1742  *      @len: new length
1743  *
1744  *      Cut the length of a buffer down by removing data from the tail. If
1745  *      the buffer is already under the length specified it is not modified.
1746  *      The skb must be linear.
1747  */
1748 void skb_trim(struct sk_buff *skb, unsigned int len)
1749 {
1750         if (skb->len > len)
1751                 __skb_trim(skb, len);
1752 }
1753 EXPORT_SYMBOL(skb_trim);
1754 
1755 /* Trims skb to length len. It can change skb pointers.
1756  */
1757 
1758 int ___pskb_trim(struct sk_buff *skb, unsigned int len)
1759 {
1760         struct sk_buff **fragp;
1761         struct sk_buff *frag;
1762         int offset = skb_headlen(skb);
1763         int nfrags = skb_shinfo(skb)->nr_frags;
1764         int i;
1765         int err;
1766 
1767         if (skb_cloned(skb) &&
1768             unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
1769                 return err;
1770 
1771         i = 0;
1772         if (offset >= len)
1773                 goto drop_pages;
1774 
1775         for (; i < nfrags; i++) {
1776                 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
1777 
1778                 if (end < len) {
1779                         offset = end;
1780                         continue;
1781                 }
1782 
1783                 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
1784 
1785 drop_pages:
1786                 skb_shinfo(skb)->nr_frags = i;
1787 
1788                 for (; i < nfrags; i++)
1789                         skb_frag_unref(skb, i);
1790 
1791                 if (skb_has_frag_list(skb))
1792                         skb_drop_fraglist(skb);
1793                 goto done;
1794         }
1795 
1796         for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
1797              fragp = &frag->next) {
1798                 int end = offset + frag->len;
1799 
1800                 if (skb_shared(frag)) {
1801                         struct sk_buff *nfrag;
1802 
1803                         nfrag = skb_clone(frag, GFP_ATOMIC);
1804                         if (unlikely(!nfrag))
1805                                 return -ENOMEM;
1806 
1807                         nfrag->next = frag->next;
1808                         consume_skb(frag);
1809                         frag = nfrag;
1810                         *fragp = frag;
1811                 }
1812 
1813                 if (end < len) {
1814                         offset = end;
1815                         continue;
1816                 }
1817 
1818                 if (end > len &&
1819                     unlikely((err = pskb_trim(frag, len - offset))))
1820                         return err;
1821 
1822                 if (frag->next)
1823                         skb_drop_list(&frag->next);
1824                 break;
1825         }
1826 
1827 done:
1828         if (len > skb_headlen(skb)) {
1829                 skb->data_len -= skb->len - len;
1830                 skb->len       = len;
1831         } else {
1832                 skb->len       = len;
1833                 skb->data_len  = 0;
1834                 skb_set_tail_pointer(skb, len);
1835         }
1836 
1837         if (!skb->sk || skb->destructor == sock_edemux)
1838                 skb_condense(skb);
1839         return 0;
1840 }
1841 EXPORT_SYMBOL(___pskb_trim);
1842 
1843 /**
1844  *      __pskb_pull_tail - advance tail of skb header
1845  *      @skb: buffer to reallocate
1846  *      @delta: number of bytes to advance tail
1847  *
1848  *      The function makes a sense only on a fragmented &sk_buff,
1849  *      it expands header moving its tail forward and copying necessary
1850  *      data from fragmented part.
1851  *
1852  *      &sk_buff MUST have reference count of 1.
1853  *
1854  *      Returns %NULL (and &sk_buff does not change) if pull failed
1855  *      or value of new tail of skb in the case of success.
1856  *
1857  *      All the pointers pointing into skb header may change and must be
1858  *      reloaded after call to this function.
1859  */
1860 
1861 /* Moves tail of skb head forward, copying data from fragmented part,
1862  * when it is necessary.
1863  * 1. It may fail due to malloc failure.
1864  * 2. It may change skb pointers.
1865  *
1866  * It is pretty complicated. Luckily, it is called only in exceptional cases.
1867  */
1868 void *__pskb_pull_tail(struct sk_buff *skb, int delta)
1869 {
1870         /* If skb has not enough free space at tail, get new one
1871          * plus 128 bytes for future expansions. If we have enough
1872          * room at tail, reallocate without expansion only if skb is cloned.
1873          */
1874         int i, k, eat = (skb->tail + delta) - skb->end;
1875 
1876         if (eat > 0 || skb_cloned(skb)) {
1877                 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
1878                                      GFP_ATOMIC))
1879                         return NULL;
1880         }
1881 
1882         BUG_ON(skb_copy_bits(skb, skb_headlen(skb),
1883                              skb_tail_pointer(skb), delta));
1884 
1885         /* Optimization: no fragments, no reasons to preestimate
1886          * size of pulled pages. Superb.
1887          */
1888         if (!skb_has_frag_list(skb))
1889                 goto pull_pages;
1890 
1891         /* Estimate size of pulled pages. */
1892         eat = delta;
1893         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1894                 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1895 
1896                 if (size >= eat)
1897                         goto pull_pages;
1898                 eat -= size;
1899         }
1900 
1901         /* If we need update frag list, we are in troubles.
1902          * Certainly, it is possible to add an offset to skb data,
1903          * but taking into account that pulling is expected to
1904          * be very rare operation, it is worth to fight against
1905          * further bloating skb head and crucify ourselves here instead.
1906          * Pure masohism, indeed. 8)8)
1907          */
1908         if (eat) {
1909                 struct sk_buff *list = skb_shinfo(skb)->frag_list;
1910                 struct sk_buff *clone = NULL;
1911                 struct sk_buff *insp = NULL;
1912 
1913                 do {
1914                         BUG_ON(!list);
1915 
1916                         if (list->len <= eat) {
1917                                 /* Eaten as whole. */
1918                                 eat -= list->len;
1919                                 list = list->next;
1920                                 insp = list;
1921                         } else {
1922                                 /* Eaten partially. */
1923 
1924                                 if (skb_shared(list)) {
1925                                         /* Sucks! We need to fork list. :-( */
1926                                         clone = skb_clone(list, GFP_ATOMIC);
1927                                         if (!clone)
1928                                                 return NULL;
1929                                         insp = list->next;
1930                                         list = clone;
1931                                 } else {
1932                                         /* This may be pulled without
1933                                          * problems. */
1934                                         insp = list;
1935                                 }
1936                                 if (!pskb_pull(list, eat)) {
1937                                         kfree_skb(clone);
1938                                         return NULL;
1939                                 }
1940                                 break;
1941                         }
1942                 } while (eat);
1943 
1944                 /* Free pulled out fragments. */
1945                 while ((list = skb_shinfo(skb)->frag_list) != insp) {
1946                         skb_shinfo(skb)->frag_list = list->next;
1947                         kfree_skb(list);
1948                 }
1949                 /* And insert new clone at head. */
1950                 if (clone) {
1951                         clone->next = list;
1952                         skb_shinfo(skb)->frag_list = clone;
1953                 }
1954         }
1955         /* Success! Now we may commit changes to skb data. */
1956 
1957 pull_pages:
1958         eat = delta;
1959         k = 0;
1960         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1961                 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1962 
1963                 if (size <= eat) {
1964                         skb_frag_unref(skb, i);
1965                         eat -= size;
1966                 } else {
1967                         skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
1968                         if (eat) {
1969                                 skb_shinfo(skb)->frags[k].page_offset += eat;
1970                                 skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat);
1971                                 if (!i)
1972                                         goto end;
1973                                 eat = 0;
1974                         }
1975                         k++;
1976                 }
1977         }
1978         skb_shinfo(skb)->nr_frags = k;
1979 
1980 end:
1981         skb->tail     += delta;
1982         skb->data_len -= delta;
1983 
1984         if (!skb->data_len)
1985                 skb_zcopy_clear(skb, false);
1986 
1987         return skb_tail_pointer(skb);
1988 }
1989 EXPORT_SYMBOL(__pskb_pull_tail);
1990 
1991 /**
1992  *      skb_copy_bits - copy bits from skb to kernel buffer
1993  *      @skb: source skb
1994  *      @offset: offset in source
1995  *      @to: destination buffer
1996  *      @len: number of bytes to copy
1997  *
1998  *      Copy the specified number of bytes from the source skb to the
1999  *      destination buffer.
2000  *
2001  *      CAUTION ! :
2002  *              If its prototype is ever changed,
2003  *              check arch/{*}/net/{*}.S files,
2004  *              since it is called from BPF assembly code.
2005  */
2006 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
2007 {
2008         int start = skb_headlen(skb);
2009         struct sk_buff *frag_iter;
2010         int i, copy;
2011 
2012         if (offset > (int)skb->len - len)
2013                 goto fault;
2014 
2015         /* Copy header. */
2016         if ((copy = start - offset) > 0) {
2017                 if (copy > len)
2018                         copy = len;
2019                 skb_copy_from_linear_data_offset(skb, offset, to, copy);
2020                 if ((len -= copy) == 0)
2021                         return 0;
2022                 offset += copy;
2023                 to     += copy;
2024         }
2025 
2026         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2027                 int end;
2028                 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
2029 
2030                 WARN_ON(start > offset + len);
2031 
2032                 end = start + skb_frag_size(f);
2033                 if ((copy = end - offset) > 0) {
2034                         u32 p_off, p_len, copied;
2035                         struct page *p;
2036                         u8 *vaddr;
2037 
2038                         if (copy > len)
2039                                 copy = len;
2040 
2041                         skb_frag_foreach_page(f,
2042                                               f->page_offset + offset - start,
2043                                               copy, p, p_off, p_len, copied) {
2044                                 vaddr = kmap_atomic(p);
2045                                 memcpy(to + copied, vaddr + p_off, p_len);
2046                                 kunmap_atomic(vaddr);
2047                         }
2048 
2049                         if ((len -= copy) == 0)
2050                                 return 0;
2051                         offset += copy;
2052                         to     += copy;
2053                 }
2054                 start = end;
2055         }
2056 
2057         skb_walk_frags(skb, frag_iter) {
2058                 int end;
2059 
2060                 WARN_ON(start > offset + len);
2061 
2062                 end = start + frag_iter->len;
2063                 if ((copy = end - offset) > 0) {
2064                         if (copy > len)
2065                                 copy = len;
2066                         if (skb_copy_bits(frag_iter, offset - start, to, copy))
2067                                 goto fault;
2068                         if ((len -= copy) == 0)
2069                                 return 0;
2070                         offset += copy;
2071                         to     += copy;
2072                 }
2073                 start = end;
2074         }
2075 
2076         if (!len)
2077                 return 0;
2078 
2079 fault:
2080         return -EFAULT;
2081 }
2082 EXPORT_SYMBOL(skb_copy_bits);
2083 
2084 /*
2085  * Callback from splice_to_pipe(), if we need to release some pages
2086  * at the end of the spd in case we error'ed out in filling the pipe.
2087  */
2088 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
2089 {
2090         put_page(spd->pages[i]);
2091 }
2092 
2093 static struct page *linear_to_page(struct page *page, unsigned int *len,
2094                                    unsigned int *offset,
2095                                    struct sock *sk)
2096 {
2097         struct page_frag *pfrag = sk_page_frag(sk);
2098 
2099         if (!sk_page_frag_refill(sk, pfrag))
2100                 return NULL;
2101 
2102         *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
2103 
2104         memcpy(page_address(pfrag->page) + pfrag->offset,
2105                page_address(page) + *offset, *len);
2106         *offset = pfrag->offset;
2107         pfrag->offset += *len;
2108 
2109         return pfrag->page;
2110 }
2111 
2112 static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
2113                              struct page *page,
2114                              unsigned int offset)
2115 {
2116         return  spd->nr_pages &&
2117                 spd->pages[spd->nr_pages - 1] == page &&
2118                 (spd->partial[spd->nr_pages - 1].offset +
2119                  spd->partial[spd->nr_pages - 1].len == offset);
2120 }
2121 
2122 /*
2123  * Fill page/offset/length into spd, if it can hold more pages.
2124  */
2125 static bool spd_fill_page(struct splice_pipe_desc *spd,
2126                           struct pipe_inode_info *pipe, struct page *page,
2127                           unsigned int *len, unsigned int offset,
2128                           bool linear,
2129                           struct sock *sk)
2130 {
2131         if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
2132                 return true;
2133 
2134         if (linear) {
2135                 page = linear_to_page(page, len, &offset, sk);
2136                 if (!page)
2137                         return true;
2138         }
2139         if (spd_can_coalesce(spd, page, offset)) {
2140                 spd->partial[spd->nr_pages - 1].len += *len;
2141                 return false;
2142         }
2143         get_page(page);
2144         spd->pages[spd->nr_pages] = page;
2145         spd->partial[spd->nr_pages].len = *len;
2146         spd->partial[spd->nr_pages].offset = offset;
2147         spd->nr_pages++;
2148 
2149         return false;
2150 }
2151 
2152 static bool __splice_segment(struct page *page, unsigned int poff,
2153                              unsigned int plen, unsigned int *off,
2154                              unsigned int *len,
2155                              struct splice_pipe_desc *spd, bool linear,
2156                              struct sock *sk,
2157                              struct pipe_inode_info *pipe)
2158 {
2159         if (!*len)
2160                 return true;
2161 
2162         /* skip this segment if already processed */
2163         if (*off >= plen) {
2164                 *off -= plen;
2165                 return false;
2166         }
2167 
2168         /* ignore any bits we already processed */
2169         poff += *off;
2170         plen -= *off;
2171         *off = 0;
2172 
2173         do {
2174                 unsigned int flen = min(*len, plen);
2175 
2176                 if (spd_fill_page(spd, pipe, page, &flen, poff,
2177                                   linear, sk))
2178                         return true;
2179                 poff += flen;
2180                 plen -= flen;
2181                 *len -= flen;
2182         } while (*len && plen);
2183 
2184         return false;
2185 }
2186 
2187 /*
2188  * Map linear and fragment data from the skb to spd. It reports true if the
2189  * pipe is full or if we already spliced the requested length.
2190  */
2191 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
2192                               unsigned int *offset, unsigned int *len,
2193                               struct splice_pipe_desc *spd, struct sock *sk)
2194 {
2195         int seg;
2196         struct sk_buff *iter;
2197 
2198         /* map the linear part :
2199          * If skb->head_frag is set, this 'linear' part is backed by a
2200          * fragment, and if the head is not shared with any clones then
2201          * we can avoid a copy since we own the head portion of this page.
2202          */
2203         if (__splice_segment(virt_to_page(skb->data),
2204                              (unsigned long) skb->data & (PAGE_SIZE - 1),
2205                              skb_headlen(skb),
2206                              offset, len, spd,
2207                              skb_head_is_locked(skb),
2208                              sk, pipe))
2209                 return true;
2210 
2211         /*
2212          * then map the fragments
2213          */
2214         for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
2215                 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
2216 
2217                 if (__splice_segment(skb_frag_page(f),
2218                                      f->page_offset, skb_frag_size(f),
2219                                      offset, len, spd, false, sk, pipe))
2220                         return true;
2221         }
2222 
2223         skb_walk_frags(skb, iter) {
2224                 if (*offset >= iter->len) {
2225                         *offset -= iter->len;
2226                         continue;
2227                 }
2228                 /* __skb_splice_bits() only fails if the output has no room
2229                  * left, so no point in going over the frag_list for the error
2230                  * case.
2231                  */
2232                 if (__skb_splice_bits(iter, pipe, offset, len, spd, sk))
2233                         return true;
2234         }
2235 
2236         return false;
2237 }
2238 
2239 /*
2240  * Map data from the skb to a pipe. Should handle both the linear part,
2241  * the fragments, and the frag list.
2242  */
2243 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
2244                     struct pipe_inode_info *pipe, unsigned int tlen,
2245                     unsigned int flags)
2246 {
2247         struct partial_page partial[MAX_SKB_FRAGS];
2248         struct page *pages[MAX_SKB_FRAGS];
2249         struct splice_pipe_desc spd = {
2250                 .pages = pages,
2251                 .partial = partial,
2252                 .nr_pages_max = MAX_SKB_FRAGS,
2253                 .ops = &nosteal_pipe_buf_ops,
2254                 .spd_release = sock_spd_release,
2255         };
2256         int ret = 0;
2257 
2258         __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk);
2259 
2260         if (spd.nr_pages)
2261                 ret = splice_to_pipe(pipe, &spd);
2262 
2263         return ret;
2264 }
2265 EXPORT_SYMBOL_GPL(skb_splice_bits);
2266 
2267 /* Send skb data on a socket. Socket must be locked. */
2268 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
2269                          int len)
2270 {
2271         unsigned int orig_len = len;
2272         struct sk_buff *head = skb;
2273         unsigned short fragidx;
2274         int slen, ret;
2275 
2276 do_frag_list:
2277 
2278         /* Deal with head data */
2279         while (offset < skb_headlen(skb) && len) {
2280                 struct kvec kv;
2281                 struct msghdr msg;
2282 
2283                 slen = min_t(int, len, skb_headlen(skb) - offset);
2284                 kv.iov_base = skb->data + offset;
2285                 kv.iov_len = slen;
2286                 memset(&msg, 0, sizeof(msg));
2287 
2288                 ret = kernel_sendmsg_locked(sk, &msg, &kv, 1, slen);
2289                 if (ret <= 0)
2290                         goto error;
2291 
2292                 offset += ret;
2293                 len -= ret;
2294         }
2295 
2296         /* All the data was skb head? */
2297         if (!len)
2298                 goto out;
2299 
2300         /* Make offset relative to start of frags */
2301         offset -= skb_headlen(skb);
2302 
2303         /* Find where we are in frag list */
2304         for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2305                 skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];
2306 
2307                 if (offset < frag->size)
2308                         break;
2309 
2310                 offset -= frag->size;
2311         }
2312 
2313         for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2314                 skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];
2315 
2316                 slen = min_t(size_t, len, frag->size - offset);
2317 
2318                 while (slen) {
2319                         ret = kernel_sendpage_locked(sk, frag->page.p,
2320                                                      frag->page_offset + offset,
2321                                                      slen, MSG_DONTWAIT);
2322                         if (ret <= 0)
2323                                 goto error;
2324 
2325                         len -= ret;
2326                         offset += ret;
2327                         slen -= ret;
2328                 }
2329 
2330                 offset = 0;
2331         }
2332 
2333         if (len) {
2334                 /* Process any frag lists */
2335 
2336                 if (skb == head) {
2337                         if (skb_has_frag_list(skb)) {
2338                                 skb = skb_shinfo(skb)->frag_list;
2339                                 goto do_frag_list;
2340                         }
2341                 } else if (skb->next) {
2342                         skb = skb->next;
2343                         goto do_frag_list;
2344                 }
2345         }
2346 
2347 out:
2348         return orig_len - len;
2349 
2350 error:
2351         return orig_len == len ? ret : orig_len - len;
2352 }
2353 EXPORT_SYMBOL_GPL(skb_send_sock_locked);
2354 
2355 /* Send skb data on a socket. */
2356 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len)
2357 {
2358         int ret = 0;
2359 
2360         lock_sock(sk);
2361         ret = skb_send_sock_locked(sk, skb, offset, len);
2362         release_sock(sk);
2363 
2364         return ret;
2365 }
2366 EXPORT_SYMBOL_GPL(skb_send_sock);
2367 
2368 /**
2369  *      skb_store_bits - store bits from kernel buffer to skb
2370  *      @skb: destination buffer
2371  *      @offset: offset in destination
2372  *      @from: source buffer
2373  *      @len: number of bytes to copy
2374  *
2375  *      Copy the specified number of bytes from the source buffer to the
2376  *      destination skb.  This function handles all the messy bits of
2377  *      traversing fragment lists and such.
2378  */
2379 
2380 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
2381 {
2382         int start = skb_headlen(skb);
2383         struct sk_buff *frag_iter;
2384         int i, copy;
2385 
2386         if (offset > (int)skb->len - len)
2387                 goto fault;
2388 
2389         if ((copy = start - offset) > 0) {
2390                 if (copy > len)
2391                         copy = len;
2392                 skb_copy_to_linear_data_offset(skb, offset, from, copy);
2393                 if ((len -= copy) == 0)
2394                         return 0;
2395                 offset += copy;
2396                 from += copy;
2397         }
2398 
2399         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2400                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2401                 int end;
2402 
2403                 WARN_ON(start > offset + len);
2404 
2405                 end = start + skb_frag_size(frag);
2406                 if ((copy = end - offset) > 0) {
2407                         u32 p_off, p_len, copied;
2408                         struct page *p;
2409                         u8 *vaddr;
2410 
2411                         if (copy > len)
2412                                 copy = len;
2413 
2414                         skb_frag_foreach_page(frag,
2415                                               frag->page_offset + offset - start,
2416                                               copy, p, p_off, p_len, copied) {
2417                                 vaddr = kmap_atomic(p);
2418                                 memcpy(vaddr + p_off, from + copied, p_len);
2419                                 kunmap_atomic(vaddr);
2420                         }
2421 
2422                         if ((len -= copy) == 0)
2423                                 return 0;
2424                         offset += copy;
2425                         from += copy;
2426                 }
2427                 start = end;
2428         }
2429 
2430         skb_walk_frags(skb, frag_iter) {
2431                 int end;
2432 
2433                 WARN_ON(start > offset + len);
2434 
2435                 end = start + frag_iter->len;
2436                 if ((copy = end - offset) > 0) {
2437                         if (copy > len)
2438                                 copy = len;
2439                         if (skb_store_bits(frag_iter, offset - start,
2440                                            from, copy))
2441                                 goto fault;
2442                         if ((len -= copy) == 0)
2443                                 return 0;
2444                         offset += copy;
2445                         from += copy;
2446                 }
2447                 start = end;
2448         }
2449         if (!len)
2450                 return 0;
2451 
2452 fault:
2453         return -EFAULT;
2454 }
2455 EXPORT_SYMBOL(skb_store_bits);
2456 
2457 /* Checksum skb data. */
2458 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
2459                       __wsum csum, const struct skb_checksum_ops *ops)
2460 {
2461         int start = skb_headlen(skb);
2462         int i, copy = start - offset;
2463         struct sk_buff *frag_iter;
2464         int pos = 0;
2465 
2466         /* Checksum header. */
2467         if (copy > 0) {
2468                 if (copy > len)
2469                         copy = len;
2470                 csum = ops->update(skb->data + offset, copy, csum);
2471                 if ((len -= copy) == 0)
2472                         return csum;
2473                 offset += copy;
2474                 pos     = copy;
2475         }
2476 
2477         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2478                 int end;
2479                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2480 
2481                 WARN_ON(start > offset + len);
2482 
2483                 end = start + skb_frag_size(frag);
2484                 if ((copy = end - offset) > 0) {
2485                         u32 p_off, p_len, copied;
2486                         struct page *p;
2487                         __wsum csum2;
2488                         u8 *vaddr;
2489 
2490                         if (copy > len)
2491                                 copy = len;
2492 
2493                         skb_frag_foreach_page(frag,
2494                                               frag->page_offset + offset - start,
2495                                               copy, p, p_off, p_len, copied) {
2496                                 vaddr = kmap_atomic(p);
2497                                 csum2 = ops->update(vaddr + p_off, p_len, 0);
2498                                 kunmap_atomic(vaddr);
2499                                 csum = ops->combine(csum, csum2, pos, p_len);
2500                                 pos += p_len;
2501                         }
2502 
2503                         if (!(len -= copy))
2504                                 return csum;
2505                         offset += copy;
2506                 }
2507                 start = end;
2508         }
2509 
2510         skb_walk_frags(skb, frag_iter) {
2511                 int end;
2512 
2513                 WARN_ON(start > offset + len);
2514 
2515                 end = start + frag_iter->len;
2516                 if ((copy = end - offset) > 0) {
2517                         __wsum csum2;
2518                         if (copy > len)
2519                                 copy = len;
2520                         csum2 = __skb_checksum(frag_iter, offset - start,
2521                                                copy, 0, ops);
2522                         csum = ops->combine(csum, csum2, pos, copy);
2523                         if ((len -= copy) == 0)
2524                                 return csum;
2525                         offset += copy;
2526                         pos    += copy;
2527                 }
2528                 start = end;
2529         }
2530         BUG_ON(len);
2531 
2532         return csum;
2533 }
2534 EXPORT_SYMBOL(__skb_checksum);
2535 
2536 __wsum skb_checksum(const struct sk_buff *skb, int offset,
2537                     int len, __wsum csum)
2538 {
2539         const struct skb_checksum_ops ops = {
2540                 .update  = csum_partial_ext,
2541                 .combine = csum_block_add_ext,
2542         };
2543 
2544         return __skb_checksum(skb, offset, len, csum, &ops);
2545 }
2546 EXPORT_SYMBOL(skb_checksum);
2547 
2548 /* Both of above in one bottle. */
2549 
2550 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
2551                                     u8 *to, int len, __wsum csum)
2552 {
2553         int start = skb_headlen(skb);
2554         int i, copy = start - offset;
2555         struct sk_buff *frag_iter;
2556         int pos = 0;
2557 
2558         /* Copy header. */
2559         if (copy > 0) {
2560                 if (copy > len)
2561                         copy = len;
2562                 csum = csum_partial_copy_nocheck(skb->data + offset, to,
2563                                                  copy, csum);
2564                 if ((len -= copy) == 0)
2565                         return csum;
2566                 offset += copy;
2567                 to     += copy;
2568                 pos     = copy;
2569         }
2570 
2571         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2572                 int end;
2573 
2574                 WARN_ON(start > offset + len);
2575 
2576                 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
2577                 if ((copy = end - offset) > 0) {
2578                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2579                         u32 p_off, p_len, copied;
2580                         struct page *p;
2581                         __wsum csum2;
2582                         u8 *vaddr;
2583 
2584                         if (copy > len)
2585                                 copy = len;
2586 
2587                         skb_frag_foreach_page(frag,
2588                                               frag->page_offset + offset - start,
2589                                               copy, p, p_off, p_len, copied) {
2590                                 vaddr = kmap_atomic(p);
2591                                 csum2 = csum_partial_copy_nocheck(vaddr + p_off,
2592                                                                   to + copied,
2593                                                                   p_len, 0);
2594                                 kunmap_atomic(vaddr);
2595                                 csum = csum_block_add(csum, csum2, pos);
2596                                 pos += p_len;
2597                         }
2598 
2599                         if (!(len -= copy))
2600                                 return csum;
2601                         offset += copy;
2602                         to     += copy;
2603                 }
2604                 start = end;
2605         }
2606 
2607         skb_walk_frags(skb, frag_iter) {
2608                 __wsum csum2;
2609                 int end;
2610 
2611                 WARN_ON(start > offset + len);
2612 
2613                 end = start + frag_iter->len;
2614                 if ((copy = end - offset) > 0) {
2615                         if (copy > len)
2616                                 copy = len;
2617                         csum2 = skb_copy_and_csum_bits(frag_iter,
2618                                                        offset - start,
2619                                                        to, copy, 0);
2620                         csum = csum_block_add(csum, csum2, pos);
2621                         if ((len -= copy) == 0)
2622                                 return csum;
2623                         offset += copy;
2624                         to     += copy;
2625                         pos    += copy;
2626                 }
2627                 start = end;
2628         }
2629         BUG_ON(len);
2630         return csum;
2631 }
2632 EXPORT_SYMBOL(skb_copy_and_csum_bits);
2633 
2634 static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum)
2635 {
2636         net_warn_ratelimited(
2637                 "%s: attempt to compute crc32c without libcrc32c.ko\n",
2638                 __func__);
2639         return 0;
2640 }
2641 
2642 static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2,
2643                                        int offset, int len)
2644 {
2645         net_warn_ratelimited(
2646                 "%s: attempt to compute crc32c without libcrc32c.ko\n",
2647                 __func__);
2648         return 0;
2649 }
2650 
2651 static const struct skb_checksum_ops default_crc32c_ops = {
2652         .update  = warn_crc32c_csum_update,
2653         .combine = warn_crc32c_csum_combine,
2654 };
2655 
2656 const struct skb_checksum_ops *crc32c_csum_stub __read_mostly =
2657         &default_crc32c_ops;
2658 EXPORT_SYMBOL(crc32c_csum_stub);
2659 
2660  /**
2661  *      skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy()
2662  *      @from: source buffer
2663  *
2664  *      Calculates the amount of linear headroom needed in the 'to' skb passed
2665  *      into skb_zerocopy().
2666  */
2667 unsigned int
2668 skb_zerocopy_headlen(const struct sk_buff *from)
2669 {
2670         unsigned int hlen = 0;
2671 
2672         if (!from->head_frag ||
2673             skb_headlen(from) < L1_CACHE_BYTES ||
2674             skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS)
2675                 hlen = skb_headlen(from);
2676 
2677         if (skb_has_frag_list(from))
2678                 hlen = from->len;
2679 
2680         return hlen;
2681 }
2682 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen);
2683 
2684 /**
2685  *      skb_zerocopy - Zero copy skb to skb
2686  *      @to: destination buffer
2687  *      @from: source buffer
2688  *      @len: number of bytes to copy from source buffer
2689  *      @hlen: size of linear headroom in destination buffer
2690  *
2691  *      Copies up to `len` bytes from `from` to `to` by creating references
2692  *      to the frags in the source buffer.
2693  *
2694  *      The `hlen` as calculated by skb_zerocopy_headlen() specifies the
2695  *      headroom in the `to` buffer.
2696  *
2697  *      Return value:
2698  *      0: everything is OK
2699  *      -ENOMEM: couldn't orphan frags of @from due to lack of memory
2700  *      -EFAULT: skb_copy_bits() found some problem with skb geometry
2701  */
2702 int
2703 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
2704 {
2705         int i, j = 0;
2706         int plen = 0; /* length of skb->head fragment */
2707         int ret;
2708         struct page *page;
2709         unsigned int offset;
2710 
2711         BUG_ON(!from->head_frag && !hlen);
2712 
2713         /* dont bother with small payloads */
2714         if (len <= skb_tailroom(to))
2715                 return skb_copy_bits(from, 0, skb_put(to, len), len);
2716 
2717         if (hlen) {
2718                 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen);
2719                 if (unlikely(ret))
2720                         return ret;
2721                 len -= hlen;
2722         } else {
2723                 plen = min_t(int, skb_headlen(from), len);
2724                 if (plen) {
2725                         page = virt_to_head_page(from->head);
2726                         offset = from->data - (unsigned char *)page_address(page);
2727                         __skb_fill_page_desc(to, 0, page, offset, plen);
2728                         get_page(page);
2729                         j = 1;
2730                         len -= plen;
2731                 }
2732         }
2733 
2734         to->truesize += len + plen;
2735         to->len += len + plen;
2736         to->data_len += len + plen;
2737 
2738         if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) {
2739                 skb_tx_error(from);
2740                 return -ENOMEM;
2741         }
2742         skb_zerocopy_clone(to, from, GFP_ATOMIC);
2743 
2744         for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
2745                 if (!len)
2746                         break;
2747                 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
2748                 skb_shinfo(to)->frags[j].size = min_t(int, skb_shinfo(to)->frags[j].size, len);
2749                 len -= skb_shinfo(to)->frags[j].size;
2750                 skb_frag_ref(to, j);
2751                 j++;
2752         }
2753         skb_shinfo(to)->nr_frags = j;
2754 
2755         return 0;
2756 }
2757 EXPORT_SYMBOL_GPL(skb_zerocopy);
2758 
2759 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
2760 {
2761         __wsum csum;
2762         long csstart;
2763 
2764         if (skb->ip_summed == CHECKSUM_PARTIAL)
2765                 csstart = skb_checksum_start_offset(skb);
2766         else
2767                 csstart = skb_headlen(skb);
2768 
2769         BUG_ON(csstart > skb_headlen(skb));
2770 
2771         skb_copy_from_linear_data(skb, to, csstart);
2772 
2773         csum = 0;
2774         if (csstart != skb->len)
2775                 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
2776                                               skb->len - csstart, 0);
2777 
2778         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2779                 long csstuff = csstart + skb->csum_offset;
2780 
2781                 *((__sum16 *)(to + csstuff)) = csum_fold(csum);
2782         }
2783 }
2784 EXPORT_SYMBOL(skb_copy_and_csum_dev);
2785 
2786 /**
2787  *      skb_dequeue - remove from the head of the queue
2788  *      @list: list to dequeue from
2789  *
2790  *      Remove the head of the list. The list lock is taken so the function
2791  *      may be used safely with other locking list functions. The head item is
2792  *      returned or %NULL if the list is empty.
2793  */
2794 
2795 struct sk_buff *skb_dequeue(struct sk_buff_head *list)
2796 {
2797         unsigned long flags;
2798         struct sk_buff *result;
2799 
2800         spin_lock_irqsave(&list->lock, flags);
2801         result = __skb_dequeue(list);
2802         spin_unlock_irqrestore(&list->lock, flags);
2803         return result;
2804 }
2805 EXPORT_SYMBOL(skb_dequeue);
2806 
2807 /**
2808  *      skb_dequeue_tail - remove from the tail of the queue
2809  *      @list: list to dequeue from
2810  *
2811  *      Remove the tail of the list. The list lock is taken so the function
2812  *      may be used safely with other locking list functions. The tail item is
2813  *      returned or %NULL if the list is empty.
2814  */
2815 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
2816 {
2817         unsigned long flags;
2818         struct sk_buff *result;
2819 
2820         spin_lock_irqsave(&list->lock, flags);
2821         result = __skb_dequeue_tail(list);
2822         spin_unlock_irqrestore(&list->lock, flags);
2823         return result;
2824 }
2825 EXPORT_SYMBOL(skb_dequeue_tail);
2826 
2827 /**
2828  *      skb_queue_purge - empty a list
2829  *      @list: list to empty
2830  *
2831  *      Delete all buffers on an &sk_buff list. Each buffer is removed from
2832  *      the list and one reference dropped. This function takes the list
2833  *      lock and is atomic with respect to other list locking functions.
2834  */
2835 void skb_queue_purge(struct sk_buff_head *list)
2836 {
2837         struct sk_buff *skb;
2838         while ((skb = skb_dequeue(list)) != NULL)
2839                 kfree_skb(skb);
2840 }
2841 EXPORT_SYMBOL(skb_queue_purge);
2842 
2843 /**
2844  *      skb_rbtree_purge - empty a skb rbtree
2845  *      @root: root of the rbtree to empty
2846  *
2847  *      Delete all buffers on an &sk_buff rbtree. Each buffer is removed from
2848  *      the list and one reference dropped. This function does not take
2849  *      any lock. Synchronization should be handled by the caller (e.g., TCP
2850  *      out-of-order queue is protected by the socket lock).
2851  */
2852 void skb_rbtree_purge(struct rb_root *root)
2853 {
2854         struct rb_node *p = rb_first(root);
2855 
2856         while (p) {
2857                 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode);
2858 
2859                 p = rb_next(p);
2860                 rb_erase(&skb->rbnode, root);
2861                 kfree_skb(skb);
2862         }
2863 }
2864 
2865 /**
2866  *      skb_queue_head - queue a buffer at the list head
2867  *      @list: list to use
2868  *      @newsk: buffer to queue
2869  *
2870  *      Queue a buffer at the start of the list. This function takes the
2871  *      list lock and can be used safely with other locking &sk_buff functions
2872  *      safely.
2873  *
2874  *      A buffer cannot be placed on two lists at the same time.
2875  */
2876 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
2877 {
2878         unsigned long flags;
2879 
2880         spin_lock_irqsave(&list->lock, flags);
2881         __skb_queue_head(list, newsk);
2882         spin_unlock_irqrestore(&list->lock, flags);
2883 }
2884 EXPORT_SYMBOL(skb_queue_head);
2885 
2886 /**
2887  *      skb_queue_tail - queue a buffer at the list tail
2888  *      @list: list to use
2889  *      @newsk: buffer to queue
2890  *
2891  *      Queue a buffer at the tail of the list. This function takes the
2892  *      list lock and can be used safely with other locking &sk_buff functions
2893  *      safely.
2894  *
2895  *      A buffer cannot be placed on two lists at the same time.
2896  */
2897 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
2898 {
2899         unsigned long flags;
2900 
2901         spin_lock_irqsave(&list->lock, flags);
2902         __skb_queue_tail(list, newsk);
2903         spin_unlock_irqrestore(&list->lock, flags);
2904 }
2905 EXPORT_SYMBOL(skb_queue_tail);
2906 
2907 /**
2908  *      skb_unlink      -       remove a buffer from a list
2909  *      @skb: buffer to remove
2910  *      @list: list to use
2911  *
2912  *      Remove a packet from a list. The list locks are taken and this
2913  *      function is atomic with respect to other list locked calls
2914  *
2915  *      You must know what list the SKB is on.
2916  */
2917 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
2918 {
2919         unsigned long flags;
2920 
2921         spin_lock_irqsave(&list->lock, flags);
2922         __skb_unlink(skb, list);
2923         spin_unlock_irqrestore(&list->lock, flags);
2924 }
2925 EXPORT_SYMBOL(skb_unlink);
2926 
2927 /**
2928  *      skb_append      -       append a buffer
2929  *      @old: buffer to insert after
2930  *      @newsk: buffer to insert
2931  *      @list: list to use
2932  *
2933  *      Place a packet after a given packet in a list. The list locks are taken
2934  *      and this function is atomic with respect to other list locked calls.
2935  *      A buffer cannot be placed on two lists at the same time.
2936  */
2937 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
2938 {
2939         unsigned long flags;
2940 
2941         spin_lock_irqsave(&list->lock, flags);
2942         __skb_queue_after(list, old, newsk);
2943         spin_unlock_irqrestore(&list->lock, flags);
2944 }
2945 EXPORT_SYMBOL(skb_append);
2946 
2947 /**
2948  *      skb_insert      -       insert a buffer
2949  *      @old: buffer to insert before
2950  *      @newsk: buffer to insert
2951  *      @list: list to use
2952  *
2953  *      Place a packet before a given packet in a list. The list locks are
2954  *      taken and this function is atomic with respect to other list locked
2955  *      calls.
2956  *
2957  *      A buffer cannot be placed on two lists at the same time.
2958  */
2959 void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
2960 {
2961         unsigned long flags;
2962 
2963         spin_lock_irqsave(&list->lock, flags);
2964         __skb_insert(newsk, old->prev, old, list);
2965         spin_unlock_irqrestore(&list->lock, flags);
2966 }
2967 EXPORT_SYMBOL(skb_insert);
2968 
2969 static inline void skb_split_inside_header(struct sk_buff *skb,
2970                                            struct sk_buff* skb1,
2971                                            const u32 len, const int pos)
2972 {
2973         int i;
2974 
2975         skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
2976                                          pos - len);
2977         /* And move data appendix as is. */
2978         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
2979                 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
2980 
2981         skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
2982         skb_shinfo(skb)->nr_frags  = 0;
2983         skb1->data_len             = skb->data_len;
2984         skb1->len                  += skb1->data_len;
2985         skb->data_len              = 0;
2986         skb->len                   = len;
2987         skb_set_tail_pointer(skb, len);
2988 }
2989 
2990 static inline void skb_split_no_header(struct sk_buff *skb,
2991                                        struct sk_buff* skb1,
2992                                        const u32 len, int pos)
2993 {
2994         int i, k = 0;
2995         const int nfrags = skb_shinfo(skb)->nr_frags;
2996 
2997         skb_shinfo(skb)->nr_frags = 0;
2998         skb1->len                 = skb1->data_len = skb->len - len;
2999         skb->len                  = len;
3000         skb->data_len             = len - pos;
3001 
3002         for (i = 0; i < nfrags; i++) {
3003                 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
3004 
3005                 if (pos + size > len) {
3006                         skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
3007 
3008                         if (pos < len) {
3009                                 /* Split frag.
3010                                  * We have two variants in this case:
3011                                  * 1. Move all the frag to the second
3012                                  *    part, if it is possible. F.e.
3013                                  *    this approach is mandatory for TUX,
3014                                  *    where splitting is expensive.
3015                                  * 2. Split is accurately. We make this.
3016                                  */
3017                                 skb_frag_ref(skb, i);
3018                                 skb_shinfo(skb1)->frags[0].page_offset += len - pos;
3019                                 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
3020                                 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
3021                                 skb_shinfo(skb)->nr_frags++;
3022                         }
3023                         k++;
3024                 } else
3025                         skb_shinfo(skb)->nr_frags++;
3026                 pos += size;
3027         }
3028         skb_shinfo(skb1)->nr_frags = k;
3029 }
3030 
3031 /**
3032  * skb_split - Split fragmented skb to two parts at length len.
3033  * @skb: the buffer to split
3034  * @skb1: the buffer to receive the second part
3035  * @len: new length for skb
3036  */
3037 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
3038 {
3039         int pos = skb_headlen(skb);
3040 
3041         skb_shinfo(skb1)->tx_flags |= skb_shinfo(skb)->tx_flags &
3042                                       SKBTX_SHARED_FRAG;
3043         skb_zerocopy_clone(skb1, skb, 0);
3044         if (len < pos)  /* Split line is inside header. */
3045                 skb_split_inside_header(skb, skb1, len, pos);
3046         else            /* Second chunk has no header, nothing to copy. */
3047                 skb_split_no_header(skb, skb1, len, pos);
3048 }
3049 EXPORT_SYMBOL(skb_split);
3050 
3051 /* Shifting from/to a cloned skb is a no-go.
3052  *
3053  * Caller cannot keep skb_shinfo related pointers past calling here!
3054  */
3055 static int skb_prepare_for_shift(struct sk_buff *skb)
3056 {
3057         return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3058 }
3059 
3060 /**
3061  * skb_shift - Shifts paged data partially from skb to another
3062  * @tgt: buffer into which tail data gets added
3063  * @skb: buffer from which the paged data comes from
3064  * @shiftlen: shift up to this many bytes
3065  *
3066  * Attempts to shift up to shiftlen worth of bytes, which may be less than
3067  * the length of the skb, from skb to tgt. Returns number bytes shifted.
3068  * It's up to caller to free skb if everything was shifted.
3069  *
3070  * If @tgt runs out of frags, the whole operation is aborted.
3071  *
3072  * Skb cannot include anything else but paged data while tgt is allowed
3073  * to have non-paged data as well.
3074  *
3075  * TODO: full sized shift could be optimized but that would need
3076  * specialized skb free'er to handle frags without up-to-date nr_frags.
3077  */
3078 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
3079 {
3080         int from, to, merge, todo;
3081         struct skb_frag_struct *fragfrom, *fragto;
3082 
3083         BUG_ON(shiftlen > skb->len);
3084 
3085         if (skb_headlen(skb))
3086                 return 0;
3087         if (skb_zcopy(tgt) || skb_zcopy(skb))
3088                 return 0;
3089 
3090         todo = shiftlen;
3091         from = 0;
3092         to = skb_shinfo(tgt)->nr_frags;
3093         fragfrom = &skb_shinfo(skb)->frags[from];
3094 
3095         /* Actual merge is delayed until the point when we know we can
3096          * commit all, so that we don't have to undo partial changes
3097          */
3098         if (!to ||
3099             !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
3100                               fragfrom->page_offset)) {
3101                 merge = -1;
3102         } else {
3103                 merge = to - 1;
3104 
3105                 todo -= skb_frag_size(fragfrom);
3106                 if (todo < 0) {
3107                         if (skb_prepare_for_shift(skb) ||
3108                             skb_prepare_for_shift(tgt))
3109                                 return 0;
3110 
3111                         /* All previous frag pointers might be stale! */
3112                         fragfrom = &skb_shinfo(skb)->frags[from];
3113                         fragto = &skb_shinfo(tgt)->frags[merge];
3114 
3115                         skb_frag_size_add(fragto, shiftlen);
3116                         skb_frag_size_sub(fragfrom, shiftlen);
3117                         fragfrom->page_offset += shiftlen;
3118 
3119                         goto onlymerged;
3120                 }
3121 
3122                 from++;
3123         }
3124 
3125         /* Skip full, not-fitting skb to avoid expensive operations */
3126         if ((shiftlen == skb->len) &&
3127             (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
3128                 return 0;
3129 
3130         if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
3131                 return 0;
3132 
3133         while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
3134                 if (to == MAX_SKB_FRAGS)
3135                         return 0;
3136 
3137                 fragfrom = &skb_shinfo(skb)->frags[from];
3138                 fragto = &skb_shinfo(tgt)->frags[to];
3139 
3140                 if (todo >= skb_frag_size(fragfrom)) {
3141                         *fragto = *fragfrom;
3142                         todo -= skb_frag_size(fragfrom);
3143                         from++;
3144                         to++;
3145 
3146                 } else {
3147                         __skb_frag_ref(fragfrom);
3148                         fragto->page = fragfrom->page;
3149                         fragto->page_offset = fragfrom->page_offset;
3150                         skb_frag_size_set(fragto, todo);
3151 
3152                         fragfrom->page_offset += todo;
3153                         skb_frag_size_sub(fragfrom, todo);
3154                         todo = 0;
3155 
3156                         to++;
3157                         break;
3158                 }
3159         }
3160 
3161         /* Ready to "commit" this state change to tgt */
3162         skb_shinfo(tgt)->nr_frags = to;
3163 
3164         if (merge >= 0) {
3165                 fragfrom = &skb_shinfo(skb)->frags[0];
3166                 fragto = &skb_shinfo(tgt)->frags[merge];
3167 
3168                 skb_frag_size_add(fragto, skb_frag_size(fragfrom));
3169                 __skb_frag_unref(fragfrom);
3170         }
3171 
3172         /* Reposition in the original skb */
3173         to = 0;
3174         while (from < skb_shinfo(skb)->nr_frags)
3175                 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
3176         skb_shinfo(skb)->nr_frags = to;
3177 
3178         BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
3179 
3180 onlymerged:
3181         /* Most likely the tgt won't ever need its checksum anymore, skb on
3182          * the other hand might need it if it needs to be resent
3183          */
3184         tgt->ip_summed = CHECKSUM_PARTIAL;
3185         skb->ip_summed = CHECKSUM_PARTIAL;
3186 
3187         /* Yak, is it really working this way? Some helper please? */
3188         skb->len -= shiftlen;
3189         skb->data_len -= shiftlen;
3190         skb->truesize -= shiftlen;
3191         tgt->len += shiftlen;
3192         tgt->data_len += shiftlen;
3193         tgt->truesize += shiftlen;
3194 
3195         return shiftlen;
3196 }
3197 
3198 /**
3199  * skb_prepare_seq_read - Prepare a sequential read of skb data
3200  * @skb: the buffer to read
3201  * @from: lower offset of data to be read
3202  * @to: upper offset of data to be read
3203  * @st: state variable
3204  *
3205  * Initializes the specified state variable. Must be called before
3206  * invoking skb_seq_read() for the first time.
3207  */
3208 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
3209                           unsigned int to, struct skb_seq_state *st)
3210 {
3211         st->lower_offset = from;
3212         st->upper_offset = to;
3213         st->root_skb = st->cur_skb = skb;
3214         st->frag_idx = st->stepped_offset = 0;
3215         st->frag_data = NULL;
3216 }
3217 EXPORT_SYMBOL(skb_prepare_seq_read);
3218 
3219 /**
3220  * skb_seq_read - Sequentially read skb data
3221  * @consumed: number of bytes consumed by the caller so far
3222  * @data: destination pointer for data to be returned
3223  * @st: state variable
3224  *
3225  * Reads a block of skb data at @consumed relative to the
3226  * lower offset specified to skb_prepare_seq_read(). Assigns
3227  * the head of the data block to @data and returns the length
3228  * of the block or 0 if the end of the skb data or the upper
3229  * offset has been reached.
3230  *
3231  * The caller is not required to consume all of the data
3232  * returned, i.e. @consumed is typically set to the number
3233  * of bytes already consumed and the next call to
3234  * skb_seq_read() will return the remaining part of the block.
3235  *
3236  * Note 1: The size of each block of data returned can be arbitrary,
3237  *       this limitation is the cost for zerocopy sequential
3238  *       reads of potentially non linear data.
3239  *
3240  * Note 2: Fragment lists within fragments are not implemented
3241  *       at the moment, state->root_skb could be replaced with
3242  *       a stack for this purpose.
3243  */
3244 unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
3245                           struct skb_seq_state *st)
3246 {
3247         unsigned int block_limit, abs_offset = consumed + st->lower_offset;
3248         skb_frag_t *frag;
3249 
3250         if (unlikely(abs_offset >= st->upper_offset)) {
3251                 if (st->frag_data) {
3252                         kunmap_atomic(st->frag_data);
3253                         st->frag_data = NULL;
3254                 }
3255                 return 0;
3256         }
3257 
3258 next_skb:
3259         block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
3260 
3261         if (abs_offset < block_limit && !st->frag_data) {
3262                 *data = st->cur_skb->data + (abs_offset - st->stepped_offset);
3263                 return block_limit - abs_offset;
3264         }
3265 
3266         if (st->frag_idx == 0 && !st->frag_data)
3267                 st->stepped_offset += skb_headlen(st->cur_skb);
3268 
3269         while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
3270                 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
3271                 block_limit = skb_frag_size(frag) + st->stepped_offset;
3272 
3273                 if (abs_offset < block_limit) {
3274                         if (!st->frag_data)
3275                                 st->frag_data = kmap_atomic(skb_frag_page(frag));
3276 
3277                         *data = (u8 *) st->frag_data + frag->page_offset +
3278                                 (abs_offset - st->stepped_offset);
3279 
3280                         return block_limit - abs_offset;
3281                 }
3282 
3283                 if (st->frag_data) {
3284                         kunmap_atomic(st->frag_data);
3285                         st->frag_data = NULL;
3286                 }
3287 
3288                 st->frag_idx++;
3289                 st->stepped_offset += skb_frag_size(frag);
3290         }
3291 
3292         if (st->frag_data) {
3293                 kunmap_atomic(st->frag_data);
3294                 st->frag_data = NULL;
3295         }
3296 
3297         if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
3298                 st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
3299                 st->frag_idx = 0;
3300                 goto next_skb;
3301         } else if (st->cur_skb->next) {
3302                 st->cur_skb = st->cur_skb->next;
3303                 st->frag_idx = 0;
3304                 goto next_skb;
3305         }
3306 
3307         return 0;
3308 }
3309 EXPORT_SYMBOL(skb_seq_read);
3310 
3311 /**
3312  * skb_abort_seq_read - Abort a sequential read of skb data
3313  * @st: state variable
3314  *
3315  * Must be called if skb_seq_read() was not called until it
3316  * returned 0.
3317  */
3318 void skb_abort_seq_read(struct skb_seq_state *st)
3319 {
3320         if (st->frag_data)
3321                 kunmap_atomic(st->frag_data);
3322 }
3323 EXPORT_SYMBOL(skb_abort_seq_read);
3324 
3325 #define TS_SKB_CB(state)        ((struct skb_seq_state *) &((state)->cb))
3326 
3327 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
3328                                           struct ts_config *conf,
3329                                           struct ts_state *state)
3330 {
3331         return skb_seq_read(offset, text, TS_SKB_CB(state));
3332 }
3333 
3334 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
3335 {
3336         skb_abort_seq_read(TS_SKB_CB(state));
3337 }
3338 
3339 /**
3340  * skb_find_text - Find a text pattern in skb data
3341  * @skb: the buffer to look in
3342  * @from: search offset
3343  * @to: search limit
3344  * @config: textsearch configuration
3345  *
3346  * Finds a pattern in the skb data according to the specified
3347  * textsearch configuration. Use textsearch_next() to retrieve
3348  * subsequent occurrences of the pattern. Returns the offset
3349  * to the first occurrence or UINT_MAX if no match was found.
3350  */
3351 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
3352                            unsigned int to, struct ts_config *config)
3353 {
3354         struct ts_state state;
3355         unsigned int ret;
3356 
3357         config->get_next_block = skb_ts_get_next_block;
3358         config->finish = skb_ts_finish;
3359 
3360         skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state));
3361 
3362         ret = textsearch_find(config, &state);
3363         return (ret <= to - from ? ret : UINT_MAX);
3364 }
3365 EXPORT_SYMBOL(skb_find_text);
3366 
3367 /**
3368  * skb_append_datato_frags - append the user data to a skb
3369  * @sk: sock  structure
3370  * @skb: skb structure to be appended with user data.
3371  * @getfrag: call back function to be used for getting the user data
3372  * @from: pointer to user message iov
3373  * @length: length of the iov message
3374  *
3375  * Description: This procedure append the user data in the fragment part
3376  * of the skb if any page alloc fails user this procedure returns  -ENOMEM
3377  */
3378 int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
3379                         int (*getfrag)(void *from, char *to, int offset,
3380                                         int len, int odd, struct sk_buff *skb),
3381                         void *from, int length)
3382 {
3383         int frg_cnt = skb_shinfo(skb)->nr_frags;
3384         int copy;
3385         int offset = 0;
3386         int ret;
3387         struct page_frag *pfrag = &current->task_frag;
3388 
3389         do {
3390                 /* Return error if we don't have space for new frag */
3391                 if (frg_cnt >= MAX_SKB_FRAGS)
3392                         return -EMSGSIZE;
3393 
3394                 if (!sk_page_frag_refill(sk, pfrag))
3395                         return -ENOMEM;
3396 
3397                 /* copy the user data to page */
3398                 copy = min_t(int, length, pfrag->size - pfrag->offset);
3399 
3400                 ret = getfrag(from, page_address(pfrag->page) + pfrag->offset,
3401                               offset, copy, 0, skb);
3402                 if (ret < 0)
3403                         return -EFAULT;
3404 
3405                 /* copy was successful so update the size parameters */
3406                 skb_fill_page_desc(skb, frg_cnt, pfrag->page, pfrag->offset,
3407                                    copy);
3408                 frg_cnt++;
3409                 pfrag->offset += copy;
3410                 get_page(pfrag->page);
3411 
3412                 skb->truesize += copy;
3413                 refcount_add(copy, &sk->sk_wmem_alloc);
3414                 skb->len += copy;
3415                 skb->data_len += copy;
3416                 offset += copy;
3417                 length -= copy;
3418 
3419         } while (length > 0);
3420 
3421         return 0;
3422 }
3423 EXPORT_SYMBOL(skb_append_datato_frags);
3424 
3425 int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
3426                          int offset, size_t size)
3427 {
3428         int i = skb_shinfo(skb)->nr_frags;
3429 
3430         if (skb_can_coalesce(skb, i, page, offset)) {
3431                 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
3432         } else if (i < MAX_SKB_FRAGS) {
3433                 get_page(page);
3434                 skb_fill_page_desc(skb, i, page, offset, size);
3435         } else {
3436                 return -EMSGSIZE;
3437         }
3438 
3439         return 0;
3440 }
3441 EXPORT_SYMBOL_GPL(skb_append_pagefrags);
3442 
3443 /**
3444  *      skb_pull_rcsum - pull skb and update receive checksum
3445  *      @skb: buffer to update
3446  *      @len: length of data pulled
3447  *
3448  *      This function performs an skb_pull on the packet and updates
3449  *      the CHECKSUM_COMPLETE checksum.  It should be used on
3450  *      receive path processing instead of skb_pull unless you know
3451  *      that the checksum difference is zero (e.g., a valid IP header)
3452  *      or you are setting ip_summed to CHECKSUM_NONE.
3453  */
3454 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
3455 {
3456         unsigned char *data = skb->data;
3457 
3458         BUG_ON(len > skb->len);
3459         __skb_pull(skb, len);
3460         skb_postpull_rcsum(skb, data, len);
3461         return skb->data;
3462 }
3463 EXPORT_SYMBOL_GPL(skb_pull_rcsum);
3464 
3465 static inline skb_frag_t skb_head_frag_to_page_desc(struct sk_buff *frag_skb)
3466 {
3467         skb_frag_t head_frag;
3468         struct page *page;
3469 
3470         page = virt_to_head_page(frag_skb->head);
3471         head_frag.page.p = page;
3472         head_frag.page_offset = frag_skb->data -
3473                 (unsigned char *)page_address(page);
3474         head_frag.size = skb_headlen(frag_skb);
3475         return head_frag;
3476 }
3477 
3478 /**
3479  *      skb_segment - Perform protocol segmentation on skb.
3480  *      @head_skb: buffer to segment
3481  *      @features: features for the output path (see dev->features)
3482  *
3483  *      This function performs segmentation on the given skb.  It returns
3484  *      a pointer to the first in a list of new skbs for the segments.
3485  *      In case of error it returns ERR_PTR(err).
3486  */
3487 struct sk_buff *skb_segment(struct sk_buff *head_skb,
3488                             netdev_features_t features)
3489 {
3490         struct sk_buff *segs = NULL;
3491         struct sk_buff *tail = NULL;
3492         struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list;
3493         skb_frag_t *frag = skb_shinfo(head_skb)->frags;
3494         unsigned int mss = skb_shinfo(head_skb)->gso_size;
3495         unsigned int doffset = head_skb->data - skb_mac_header(head_skb);
3496         struct sk_buff *frag_skb = head_skb;
3497         unsigned int offset = doffset;
3498         unsigned int tnl_hlen = skb_tnl_header_len(head_skb);
3499         unsigned int partial_segs = 0;
3500         unsigned int headroom;
3501         unsigned int len = head_skb->len;
3502         __be16 proto;
3503         bool csum, sg;
3504         int nfrags = skb_shinfo(head_skb)->nr_frags;
3505         int err = -ENOMEM;
3506         int i = 0;
3507         int pos;
3508         int dummy;
3509 
3510         __skb_push(head_skb, doffset);
3511         proto = skb_network_protocol(head_skb, &dummy);
3512         if (unlikely(!proto))
3513                 return ERR_PTR(-EINVAL);
3514 
3515         sg = !!(features & NETIF_F_SG);
3516         csum = !!can_checksum_protocol(features, proto);
3517 
3518         if (sg && csum && (mss != GSO_BY_FRAGS))  {
3519                 if (!(features & NETIF_F_GSO_PARTIAL)) {
3520                         struct sk_buff *iter;
3521                         unsigned int frag_len;
3522 
3523                         if (!list_skb ||
3524                             !net_gso_ok(features, skb_shinfo(head_skb)->gso_type))
3525                                 goto normal;
3526 
3527                         /* If we get here then all the required
3528                          * GSO features except frag_list are supported.
3529                          * Try to split the SKB to multiple GSO SKBs
3530                          * with no frag_list.
3531                          * Currently we can do that only when the buffers don't
3532                          * have a linear part and all the buffers except
3533                          * the last are of the same length.
3534                          */
3535                         frag_len = list_skb->len;
3536                         skb_walk_frags(head_skb, iter) {
3537                                 if (frag_len != iter->len && iter->next)
3538                                         goto normal;
3539                                 if (skb_headlen(iter) && !iter->head_frag)
3540                                         goto normal;
3541 
3542                                 len -= iter->len;
3543                         }
3544 
3545                         if (len != frag_len)
3546                                 goto normal;
3547                 }
3548 
3549                 /* GSO partial only requires that we trim off any excess that
3550                  * doesn't fit into an MSS sized block, so take care of that
3551                  * now.
3552                  */
3553                 partial_segs = len / mss;
3554                 if (partial_segs > 1)
3555                         mss *= partial_segs;
3556                 else
3557                         partial_segs = 0;
3558         }
3559 
3560 normal:
3561         headroom = skb_headroom(head_skb);
3562         pos = skb_headlen(head_skb);
3563 
3564         do {
3565                 struct sk_buff *nskb;
3566                 skb_frag_t *nskb_frag;
3567                 int hsize;
3568                 int size;
3569 
3570                 if (unlikely(mss == GSO_BY_FRAGS)) {
3571                         len = list_skb->len;
3572                 } else {
3573                         len = head_skb->len - offset;
3574                         if (len > mss)
3575                                 len = mss;
3576                 }
3577 
3578                 hsize = skb_headlen(head_skb) - offset;
3579                 if (hsize < 0)
3580                         hsize = 0;
3581                 if (hsize > len || !sg)
3582                         hsize = len;
3583 
3584                 if (!hsize && i >= nfrags && skb_headlen(list_skb) &&
3585                     (skb_headlen(list_skb) == len || sg)) {
3586                         BUG_ON(skb_headlen(list_skb) > len);
3587 
3588                         i = 0;
3589                         nfrags = skb_shinfo(list_skb)->nr_frags;
3590                         frag = skb_shinfo(list_skb)->frags;
3591                         frag_skb = list_skb;
3592                         pos += skb_headlen(list_skb);
3593 
3594                         while (pos < offset + len) {
3595                                 BUG_ON(i >= nfrags);
3596 
3597                                 size = skb_frag_size(frag);
3598                                 if (pos + size > offset + len)
3599                                         break;
3600 
3601                                 i++;
3602                                 pos += size;
3603                                 frag++;
3604                         }
3605 
3606                         nskb = skb_clone(list_skb, GFP_ATOMIC);
3607                         list_skb = list_skb->next;
3608 
3609                         if (unlikely(!nskb))
3610                                 goto err;
3611 
3612                         if (unlikely(pskb_trim(nskb, len))) {
3613                                 kfree_skb(nskb);
3614                                 goto err;
3615                         }
3616 
3617                         hsize = skb_end_offset(nskb);
3618                         if (skb_cow_head(nskb, doffset + headroom)) {
3619                                 kfree_skb(nskb);
3620                                 goto err;
3621                         }
3622 
3623                         nskb->truesize += skb_end_offset(nskb) - hsize;
3624                         skb_release_head_state(nskb);
3625                         __skb_push(nskb, doffset);
3626                 } else {
3627                         nskb = __alloc_skb(hsize + doffset + headroom,
3628                                            GFP_ATOMIC, skb_alloc_rx_flag(head_skb),
3629                                            NUMA_NO_NODE);
3630 
3631                         if (unlikely(!nskb))
3632                                 goto err;
3633 
3634                         skb_reserve(nskb, headroom);
3635                         __skb_put(nskb, doffset);
3636                 }
3637 
3638                 if (segs)
3639                         tail->next = nskb;
3640                 else
3641                         segs = nskb;
3642                 tail = nskb;
3643 
3644                 __copy_skb_header(nskb, head_skb);
3645 
3646                 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom);
3647                 skb_reset_mac_len(nskb);
3648 
3649                 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen,
3650                                                  nskb->data - tnl_hlen,
3651                                                  doffset + tnl_hlen);
3652 
3653                 if (nskb->len == len + doffset)
3654                         goto perform_csum_check;
3655 
3656                 if (!sg) {
3657                         if (!nskb->remcsum_offload)
3658                                 nskb->ip_summed = CHECKSUM_NONE;
3659                         SKB_GSO_CB(nskb)->csum =
3660                                 skb_copy_and_csum_bits(head_skb, offset,
3661                                                        skb_put(nskb, len),
3662                                                        len, 0);
3663                         SKB_GSO_CB(nskb)->csum_start =
3664                                 skb_headroom(nskb) + doffset;
3665                         continue;
3666                 }
3667 
3668                 nskb_frag = skb_shinfo(nskb)->frags;
3669 
3670                 skb_copy_from_linear_data_offset(head_skb, offset,
3671                                                  skb_put(nskb, hsize), hsize);
3672 
3673                 skb_shinfo(nskb)->tx_flags |= skb_shinfo(head_skb)->tx_flags &
3674                                               SKBTX_SHARED_FRAG;
3675 
3676                 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) ||
3677                     skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC))
3678                         goto err;
3679 
3680                 while (pos < offset + len) {
3681                         if (i >= nfrags) {
3682                                 i = 0;
3683                                 nfrags = skb_shinfo(list_skb)->nr_frags;
3684                                 frag = skb_shinfo(list_skb)->frags;
3685                                 frag_skb = list_skb;
3686                                 if (!skb_headlen(list_skb)) {
3687                                         BUG_ON(!nfrags);
3688                                 } else {
3689                                         BUG_ON(!list_skb->head_frag);
3690 
3691                                         /* to make room for head_frag. */
3692                                         i--;
3693                                         frag--;
3694                                 }
3695                                 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) ||
3696                                     skb_zerocopy_clone(nskb, frag_skb,
3697                                                        GFP_ATOMIC))
3698                                         goto err;
3699 
3700                                 list_skb = list_skb->next;
3701                         }
3702 
3703                         if (unlikely(skb_shinfo(nskb)->nr_frags >=
3704                                      MAX_SKB_FRAGS)) {
3705                                 net_warn_ratelimited(
3706                                         "skb_segment: too many frags: %u %u\n",
3707                                         pos, mss);
3708                                 err = -EINVAL;
3709                                 goto err;
3710                         }
3711 
3712                         *nskb_frag = (i < 0) ? skb_head_frag_to_page_desc(frag_skb) : *frag;
3713                         __skb_frag_ref(nskb_frag);
3714                         size = skb_frag_size(nskb_frag);
3715 
3716                         if (pos < offset) {
3717                                 nskb_frag->page_offset += offset - pos;
3718                                 skb_frag_size_sub(nskb_frag, offset - pos);
3719                         }
3720 
3721                         skb_shinfo(nskb)->nr_frags++;
3722 
3723                         if (pos + size <= offset + len) {
3724                                 i++;
3725                                 frag++;
3726                                 pos += size;
3727                         } else {
3728                                 skb_frag_size_sub(nskb_frag, pos + size - (offset + len));
3729                                 goto skip_fraglist;
3730                         }
3731 
3732                         nskb_frag++;
3733                 }
3734 
3735 skip_fraglist:
3736                 nskb->data_len = len - hsize;
3737                 nskb->len += nskb->data_len;
3738                 nskb->truesize += nskb->data_len;
3739 
3740 perform_csum_check:
3741                 if (!csum) {
3742                         if (skb_has_shared_frag(nskb) &&
3743                             __skb_linearize(nskb))
3744                                 goto err;
3745 
3746                         if (!nskb->remcsum_offload)
3747                                 nskb->ip_summed = CHECKSUM_NONE;
3748                         SKB_GSO_CB(nskb)->csum =
3749                                 skb_checksum(nskb, doffset,
3750                                              nskb->len - doffset, 0);
3751                         SKB_GSO_CB(nskb)->csum_start =
3752                                 skb_headroom(nskb) + doffset;
3753                 }
3754         } while ((offset += len) < head_skb->len);
3755 
3756         /* Some callers want to get the end of the list.
3757          * Put it in segs->prev to avoid walking the list.
3758          * (see validate_xmit_skb_list() for example)
3759          */
3760         segs->prev = tail;
3761 
3762         if (partial_segs) {
3763                 struct sk_buff *iter;
3764                 int type = skb_shinfo(head_skb)->gso_type;
3765                 unsigned short gso_size = skb_shinfo(head_skb)->gso_size;
3766 
3767                 /* Update type to add partial and then remove dodgy if set */
3768                 type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL;
3769                 type &= ~SKB_GSO_DODGY;
3770 
3771                 /* Update GSO info and prepare to start updating headers on
3772                  * our way back down the stack of protocols.
3773                  */
3774                 for (iter = segs; iter; iter = iter->next) {
3775                         skb_shinfo(iter)->gso_size = gso_size;
3776                         skb_shinfo(iter)->gso_segs = partial_segs;
3777                         skb_shinfo(iter)->gso_type = type;
3778                         SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset;
3779                 }
3780 
3781                 if (tail->len - doffset <= gso_size)
3782                         skb_shinfo(tail)->gso_size = 0;
3783                 else if (tail != segs)
3784                         skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size);
3785         }
3786 
3787         /* Following permits correct backpressure, for protocols
3788          * using skb_set_owner_w().
3789          * Idea is to tranfert ownership from head_skb to last segment.
3790          */
3791         if (head_skb->destructor == sock_wfree) {
3792                 swap(tail->truesize, head_skb->truesize);
3793                 swap(tail->destructor, head_skb->destructor);
3794                 swap(tail->sk, head_skb->sk);
3795         }
3796         return segs;
3797 
3798 err:
3799         kfree_skb_list(segs);
3800         return ERR_PTR(err);
3801 }
3802 EXPORT_SYMBOL_GPL(skb_segment);
3803 
3804 int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
3805 {
3806         struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb);
3807         unsigned int offset = skb_gro_offset(skb);
3808         unsigned int headlen = skb_headlen(skb);
3809         unsigned int len = skb_gro_len(skb);
3810         struct sk_buff *lp, *p = *head;
3811         unsigned int delta_truesize;
3812 
3813         if (unlikely(p->len + len >= 65536))
3814                 return -E2BIG;
3815 
3816         lp = NAPI_GRO_CB(p)->last;
3817         pinfo = skb_shinfo(lp);
3818 
3819         if (headlen <= offset) {
3820                 skb_frag_t *frag;
3821                 skb_frag_t *frag2;
3822                 int i = skbinfo->nr_frags;
3823                 int nr_frags = pinfo->nr_frags + i;
3824 
3825                 if (nr_frags > MAX_SKB_FRAGS)
3826                         goto merge;
3827 
3828                 offset -= headlen;
3829                 pinfo->nr_frags = nr_frags;
3830                 skbinfo->nr_frags = 0;
3831 
3832                 frag = pinfo->frags + nr_frags;
3833                 frag2 = skbinfo->frags + i;
3834                 do {
3835                         *--frag = *--frag2;
3836                 } while (--i);
3837 
3838                 frag->page_offset += offset;
3839                 skb_frag_size_sub(frag, offset);
3840 
3841                 /* all fragments truesize : remove (head size + sk_buff) */
3842                 delta_truesize = skb->truesize -
3843                                  SKB_TRUESIZE(skb_end_offset(skb));
3844 
3845                 skb->truesize -= skb->data_len;
3846                 skb->len -= skb->data_len;
3847                 skb->data_len = 0;
3848 
3849                 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
3850                 goto done;
3851         } else if (skb->head_frag) {
3852                 int nr_frags = pinfo->nr_frags;
3853                 skb_frag_t *frag = pinfo->frags + nr_frags;
3854                 struct page *page = virt_to_head_page(skb->head);
3855                 unsigned int first_size = headlen - offset;
3856                 unsigned int first_offset;
3857 
3858                 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
3859                         goto merge;
3860 
3861                 first_offset = skb->data -
3862                                (unsigned char *)page_address(page) +
3863                                offset;
3864 
3865                 pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
3866 
3867                 frag->page.p      = page;
3868                 frag->page_offset = first_offset;
3869                 skb_frag_size_set(frag, first_size);
3870 
3871                 memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
3872                 /* We dont need to clear skbinfo->nr_frags here */
3873 
3874                 delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
3875                 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
3876                 goto done;
3877         }
3878 
3879 merge:
3880         delta_truesize = skb->truesize;
3881         if (offset > headlen) {
3882                 unsigned int eat = offset - headlen;
3883 
3884                 skbinfo->frags[0].page_offset += eat;
3885                 skb_frag_size_sub(&skbinfo->frags[0], eat);
3886                 skb->data_len -= eat;
3887                 skb->len -= eat;
3888                 offset = headlen;
3889         }
3890 
3891         __skb_pull(skb, offset);
3892 
3893         if (NAPI_GRO_CB(p)->last == p)
3894                 skb_shinfo(p)->frag_list = skb;
3895         else
3896                 NAPI_GRO_CB(p)->last->next = skb;
3897         NAPI_GRO_CB(p)->last = skb;
3898         __skb_header_release(skb);
3899         lp = p;
3900 
3901 done:
3902         NAPI_GRO_CB(p)->count++;
3903         p->data_len += len;
3904         p->truesize += delta_truesize;
3905         p->len += len;
3906         if (lp != p) {
3907                 lp->data_len += len;
3908                 lp->truesize += delta_truesize;
3909                 lp->len += len;
3910         }
3911         NAPI_GRO_CB(skb)->same_flow = 1;
3912         return 0;
3913 }
3914 EXPORT_SYMBOL_GPL(skb_gro_receive);
3915 
3916 void __init skb_init(void)
3917 {
3918         skbuff_head_cache = kmem_cache_create_usercopy("skbuff_head_cache",
3919                                               sizeof(struct sk_buff),
3920                                               0,
3921                                               SLAB_HWCACHE_ALIGN|SLAB_PANIC,
3922                                               offsetof(struct sk_buff, cb),
3923                                               sizeof_field(struct sk_buff, cb),
3924                                               NULL);
3925         skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
3926                                                 sizeof(struct sk_buff_fclones),
3927                                                 0,
3928                                                 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
3929                                                 NULL);
3930 }
3931 
3932 static int
3933 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len,
3934                unsigned int recursion_level)
3935 {
3936         int start = skb_headlen(skb);
3937         int i, copy = start - offset;
3938         struct sk_buff *frag_iter;
3939         int elt = 0;
3940 
3941         if (unlikely(recursion_level >= 24))
3942                 return -EMSGSIZE;
3943 
3944         if (copy > 0) {
3945                 if (copy > len)
3946                         copy = len;
3947                 sg_set_buf(sg, skb->data + offset, copy);
3948                 elt++;
3949                 if ((len -= copy) == 0)
3950                         return elt;
3951                 offset += copy;
3952         }
3953 
3954         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3955                 int end;
3956 
3957                 WARN_ON(start > offset + len);
3958 
3959                 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
3960                 if ((copy = end - offset) > 0) {
3961                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3962                         if (unlikely(elt && sg_is_last(&sg[elt - 1])))
3963                                 return -EMSGSIZE;
3964 
3965                         if (copy > len)
3966                                 copy = len;
3967                         sg_set_page(&sg[elt], skb_frag_page(frag), copy,
3968                                         frag->page_offset+offset-start);
3969                         elt++;
3970                         if (!(len -= copy))
3971                                 return elt;
3972                         offset += copy;
3973                 }
3974                 start = end;
3975         }
3976 
3977         skb_walk_frags(skb, frag_iter) {
3978                 int end, ret;
3979 
3980                 WARN_ON(start > offset + len);
3981 
3982                 end = start + frag_iter->len;
3983                 if ((copy = end - offset) > 0) {
3984                         if (unlikely(elt && sg_is_last(&sg[elt - 1])))
3985                                 return -EMSGSIZE;
3986 
3987                         if (copy > len)
3988                                 copy = len;
3989                         ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start,
3990                                               copy, recursion_level + 1);
3991                         if (unlikely(ret < 0))
3992                                 return ret;
3993                         elt += ret;
3994                         if ((len -= copy) == 0)
3995                                 return elt;
3996                         offset += copy;
3997                 }
3998                 start = end;
3999         }
4000         BUG_ON(len);
4001         return elt;
4002 }
4003 
4004 /**
4005  *      skb_to_sgvec - Fill a scatter-gather list from a socket buffer
4006  *      @skb: Socket buffer containing the buffers to be mapped
4007  *      @sg: The scatter-gather list to map into
4008  *      @offset: The offset into the buffer's contents to start mapping
4009  *      @len: Length of buffer space to be mapped
4010  *
4011  *      Fill the specified scatter-gather list with mappings/pointers into a
4012  *      region of the buffer space attached to a socket buffer. Returns either
4013  *      the number of scatterlist items used, or -EMSGSIZE if the contents
4014  *      could not fit.
4015  */
4016 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
4017 {
4018         int nsg = __skb_to_sgvec(skb, sg, offset, len, 0);
4019 
4020         if (nsg <= 0)
4021                 return nsg;
4022 
4023         sg_mark_end(&sg[nsg - 1]);
4024 
4025         return nsg;
4026 }
4027 EXPORT_SYMBOL_GPL(skb_to_sgvec);
4028 
4029 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given
4030  * sglist without mark the sg which contain last skb data as the end.
4031  * So the caller can mannipulate sg list as will when padding new data after
4032  * the first call without calling sg_unmark_end to expend sg list.
4033  *
4034  * Scenario to use skb_to_sgvec_nomark:
4035  * 1. sg_init_table
4036  * 2. skb_to_sgvec_nomark(payload1)
4037  * 3. skb_to_sgvec_nomark(payload2)
4038  *
4039  * This is equivalent to:
4040  * 1. sg_init_table
4041  * 2. skb_to_sgvec(payload1)
4042  * 3. sg_unmark_end
4043  * 4. skb_to_sgvec(payload2)
4044  *
4045  * When mapping mutilple payload conditionally, skb_to_sgvec_nomark
4046  * is more preferable.
4047  */
4048 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
4049                         int offset, int len)
4050 {
4051         return __skb_to_sgvec(skb, sg, offset, len, 0);
4052 }
4053 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark);
4054 
4055 
4056 
4057 /**
4058  *      skb_cow_data - Check that a socket buffer's data buffers are writable
4059  *      @skb: The socket buffer to check.
4060  *      @tailbits: Amount of trailing space to be added
4061  *      @trailer: Returned pointer to the skb where the @tailbits space begins
4062  *
4063  *      Make sure that the data buffers attached to a socket buffer are
4064  *      writable. If they are not, private copies are made of the data buffers
4065  *      and the socket buffer is set to use these instead.
4066  *
4067  *      If @tailbits is given, make sure that there is space to write @tailbits
4068  *      bytes of data beyond current end of socket buffer.  @trailer will be
4069  *      set to point to the skb in which this space begins.
4070  *
4071  *      The number of scatterlist elements required to completely map the
4072  *      COW'd and extended socket buffer will be returned.
4073  */
4074 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
4075 {
4076         int copyflag;
4077         int elt;
4078         struct sk_buff *skb1, **skb_p;
4079 
4080         /* If skb is cloned or its head is paged, reallocate
4081          * head pulling out all the pages (pages are considered not writable
4082          * at the moment even if they are anonymous).
4083          */
4084         if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
4085             __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
4086                 return -ENOMEM;
4087 
4088         /* Easy case. Most of packets will go this way. */
4089         if (!skb_has_frag_list(skb)) {
4090                 /* A little of trouble, not enough of space for trailer.
4091                  * This should not happen, when stack is tuned to generate
4092                  * good frames. OK, on miss we reallocate and reserve even more
4093                  * space, 128 bytes is fair. */
4094 
4095                 if (skb_tailroom(skb) < tailbits &&
4096                     pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
4097                         return -ENOMEM;
4098 
4099                 /* Voila! */
4100                 *trailer = skb;
4101                 return 1;
4102         }
4103 
4104         /* Misery. We are in troubles, going to mincer fragments... */
4105 
4106         elt = 1;
4107         skb_p = &skb_shinfo(skb)->frag_list;
4108         copyflag = 0;
4109 
4110         while ((skb1 = *skb_p) != NULL) {
4111                 int ntail = 0;
4112 
4113                 /* The fragment is partially pulled by someone,
4114                  * this can happen on input. Copy it and everything
4115                  * after it. */
4116 
4117                 if (skb_shared(skb1))
4118                         copyflag = 1;
4119 
4120                 /* If the skb is the last, worry about trailer. */
4121 
4122                 if (skb1->next == NULL && tailbits) {
4123                         if (skb_shinfo(skb1)->nr_frags ||
4124                             skb_has_frag_list(skb1) ||
4125                             skb_tailroom(skb1) < tailbits)
4126                                 ntail = tailbits + 128;
4127                 }
4128 
4129                 if (copyflag ||
4130                     skb_cloned(skb1) ||
4131                     ntail ||
4132                     skb_shinfo(skb1)->nr_frags ||
4133                     skb_has_frag_list(skb1)) {
4134                         struct sk_buff *skb2;
4135 
4136                         /* Fuck, we are miserable poor guys... */
4137                         if (ntail == 0)
4138                                 skb2 = skb_copy(skb1, GFP_ATOMIC);
4139                         else
4140                                 skb2 = skb_copy_expand(skb1,
4141                                                        skb_headroom(skb1),
4142                                                        ntail,
4143                                                        GFP_ATOMIC);
4144                         if (unlikely(skb2 == NULL))
4145                                 return -ENOMEM;
4146 
4147                         if (skb1->sk)
4148                                 skb_set_owner_w(skb2, skb1->sk);
4149 
4150                         /* Looking around. Are we still alive?
4151                          * OK, link new skb, drop old one */
4152 
4153                         skb2->next = skb1->next;
4154                         *skb_p = skb2;
4155                         kfree_skb(skb1);
4156                         skb1 = skb2;
4157                 }
4158                 elt++;
4159                 *trailer = skb1;
4160                 skb_p = &skb1->next;
4161         }
4162 
4163         return elt;
4164 }
4165 EXPORT_SYMBOL_GPL(skb_cow_data);
4166 
4167 static void sock_rmem_free(struct sk_buff *skb)
4168 {
4169         struct sock *sk = skb->sk;
4170 
4171         atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
4172 }
4173 
4174 static void skb_set_err_queue(struct sk_buff *skb)
4175 {
4176         /* pkt_type of skbs received on local sockets is never PACKET_OUTGOING.
4177          * So, it is safe to (mis)use it to mark skbs on the error queue.
4178          */
4179         skb->pkt_type = PACKET_OUTGOING;
4180         BUILD_BUG_ON(PACKET_OUTGOING == 0);
4181 }
4182 
4183 /*
4184  * Note: We dont mem charge error packets (no sk_forward_alloc changes)
4185  */
4186 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
4187 {
4188         if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
4189             (unsigned int)sk->sk_rcvbuf)
4190                 return -ENOMEM;
4191 
4192         skb_orphan(skb);
4193         skb->sk = sk;
4194         skb->destructor = sock_rmem_free;
4195         atomic_add(skb->truesize, &sk->sk_rmem_alloc);
4196         skb_set_err_queue(skb);
4197 
4198         /* before exiting rcu section, make sure dst is refcounted */
4199         skb_dst_force(skb);
4200 
4201         skb_queue_tail(&sk->sk_error_queue, skb);
4202         if (!sock_flag(sk, SOCK_DEAD))
4203                 sk->sk_error_report(sk);
4204         return 0;
4205 }
4206 EXPORT_SYMBOL(sock_queue_err_skb);
4207 
4208 static bool is_icmp_err_skb(const struct sk_buff *skb)
4209 {
4210         return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP ||
4211                        SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6);
4212 }
4213 
4214 struct sk_buff *sock_dequeue_err_skb(struct sock *sk)
4215 {
4216         struct sk_buff_head *q = &sk->sk_error_queue;
4217         struct sk_buff *skb, *skb_next = NULL;
4218         bool icmp_next = false;
4219         unsigned long flags;
4220 
4221         spin_lock_irqsave(&q->lock, flags);
4222         skb = __skb_dequeue(q);
4223         if (skb && (skb_next = skb_peek(q))) {
4224                 icmp_next = is_icmp_err_skb(skb_next);
4225                 if (icmp_next)
4226                         sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_origin;
4227         }
4228         spin_unlock_irqrestore(&q->lock, flags);
4229 
4230         if (is_icmp_err_skb(skb) && !icmp_next)
4231                 sk->sk_err = 0;
4232 
4233         if (skb_next)
4234                 sk->sk_error_report(sk);
4235 
4236         return skb;
4237 }
4238 EXPORT_SYMBOL(sock_dequeue_err_skb);
4239 
4240 /**
4241  * skb_clone_sk - create clone of skb, and take reference to socket
4242  * @skb: the skb to clone
4243  *
4244  * This function creates a clone of a buffer that holds a reference on
4245  * sk_refcnt.  Buffers created via this function are meant to be
4246  * returned using sock_queue_err_skb, or free via kfree_skb.
4247  *
4248  * When passing buffers allocated with this function to sock_queue_err_skb
4249  * it is necessary to wrap the call with sock_hold/sock_put in order to
4250  * prevent the socket from being released prior to being enqueued on
4251  * the sk_error_queue.
4252  */
4253 struct sk_buff *skb_clone_sk(struct sk_buff *skb)
4254 {
4255         struct sock *sk = skb->sk;
4256         struct sk_buff *clone;
4257 
4258         if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
4259                 return NULL;
4260 
4261         clone = skb_clone(skb, GFP_ATOMIC);
4262         if (!clone) {
4263                 sock_put(sk);
4264                 return NULL;
4265         }
4266 
4267         clone->sk = sk;
4268         clone->destructor = sock_efree;
4269 
4270         return clone;
4271 }
4272 EXPORT_SYMBOL(skb_clone_sk);
4273 
4274 static void __skb_complete_tx_timestamp(struct sk_buff *skb,
4275                                         struct sock *sk,
4276                                         int tstype,
4277                                         bool opt_stats)
4278 {
4279         struct sock_exterr_skb *serr;
4280         int err;
4281 
4282         BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb));
4283 
4284         serr = SKB_EXT_ERR(skb);
4285         memset(serr, 0, sizeof(*serr));
4286         serr->ee.ee_errno = ENOMSG;
4287         serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
4288         serr->ee.ee_info = tstype;
4289         serr->opt_stats = opt_stats;
4290         serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0;
4291         if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) {
4292                 serr->ee.ee_data = skb_shinfo(skb)->tskey;
4293                 if (sk->sk_protocol == IPPROTO_TCP &&
4294                     sk->sk_type == SOCK_STREAM)
4295                         serr->ee.ee_data -= sk->sk_tskey;
4296         }
4297 
4298         err = sock_queue_err_skb(sk, skb);
4299 
4300         if (err)
4301                 kfree_skb(skb);
4302 }
4303 
4304 static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly)
4305 {
4306         bool ret;
4307 
4308         if (likely(sysctl_tstamp_allow_data || tsonly))
4309                 return true;
4310 
4311         read_lock_bh(&sk->sk_callback_lock);
4312         ret = sk->sk_socket && sk->sk_socket->file &&
4313               file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW);
4314         read_unlock_bh(&sk->sk_callback_lock);
4315         return ret;
4316 }
4317 
4318 void skb_complete_tx_timestamp(struct sk_buff *skb,
4319                                struct skb_shared_hwtstamps *hwtstamps)
4320 {
4321         struct sock *sk = skb->sk;
4322 
4323         if (!skb_may_tx_timestamp(sk, false))
4324                 goto err;
4325 
4326         /* Take a reference to prevent skb_orphan() from freeing the socket,
4327          * but only if the socket refcount is not zero.
4328          */
4329         if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
4330                 *skb_hwtstamps(skb) = *hwtstamps;
4331                 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false);
4332                 sock_put(sk);
4333                 return;
4334         }
4335 
4336 err:
4337         kfree_skb(skb);
4338 }
4339 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp);
4340 
4341 void __skb_tstamp_tx(struct sk_buff *orig_skb,
4342                      struct skb_shared_hwtstamps *hwtstamps,
4343                      struct sock *sk, int tstype)
4344 {
4345         struct sk_buff *skb;
4346         bool tsonly, opt_stats = false;
4347 
4348         if (!sk)
4349                 return;
4350 
4351         if (!hwtstamps && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) &&
4352             skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS)
4353                 return;
4354 
4355         tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY;
4356         if (!skb_may_tx_timestamp(sk, tsonly))
4357                 return;
4358 
4359         if (tsonly) {
4360 #ifdef CONFIG_INET
4361                 if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) &&
4362                     sk->sk_protocol == IPPROTO_TCP &&
4363                     sk->sk_type == SOCK_STREAM) {
4364                         skb = tcp_get_timestamping_opt_stats(sk);
4365                         opt_stats = true;
4366                 } else
4367 #endif
4368                         skb = alloc_skb(0, GFP_ATOMIC);
4369         } else {
4370                 skb = skb_clone(orig_skb, GFP_ATOMIC);
4371         }
4372         if (!skb)
4373                 return;
4374 
4375         if (tsonly) {
4376                 skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags &
4377                                              SKBTX_ANY_TSTAMP;
4378                 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey;
4379         }
4380 
4381         if (hwtstamps)
4382                 *skb_hwtstamps(skb) = *hwtstamps;
4383         else
4384                 skb->tstamp = ktime_get_real();
4385 
4386         __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats);
4387 }
4388 EXPORT_SYMBOL_GPL(__skb_tstamp_tx);
4389 
4390 void skb_tstamp_tx(struct sk_buff *orig_skb,
4391                    struct skb_shared_hwtstamps *hwtstamps)
4392 {
4393         return __skb_tstamp_tx(orig_skb, hwtstamps, orig_skb->sk,
4394                                SCM_TSTAMP_SND);
4395 }
4396 EXPORT_SYMBOL_GPL(skb_tstamp_tx);
4397 
4398 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
4399 {
4400         struct sock *sk = skb->sk;
4401         struct sock_exterr_skb *serr;
4402         int err = 1;
4403 
4404         skb->wifi_acked_valid = 1;
4405         skb->wifi_acked = acked;
4406 
4407         serr = SKB_EXT_ERR(skb);
4408         memset(serr, 0, sizeof(*serr));
4409         serr->ee.ee_errno = ENOMSG;
4410         serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
4411 
4412         /* Take a reference to prevent skb_orphan() from freeing the socket,
4413          * but only if the socket refcount is not zero.
4414          */
4415         if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
4416                 err = sock_queue_err_skb(sk, skb);
4417                 sock_put(sk);
4418         }
4419         if (err)
4420                 kfree_skb(skb);
4421 }
4422 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
4423 
4424 /**
4425  * skb_partial_csum_set - set up and verify partial csum values for packet
4426  * @skb: the skb to set
4427  * @start: the number of bytes after skb->data to start checksumming.
4428  * @off: the offset from start to place the checksum.
4429  *
4430  * For untrusted partially-checksummed packets, we need to make sure the values
4431  * for skb->csum_start and skb->csum_offset are valid so we don't oops.
4432  *
4433  * This function checks and sets those values and skb->ip_summed: if this
4434  * returns false you should drop the packet.
4435  */
4436 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
4437 {
4438         if (unlikely(start > skb_headlen(skb)) ||
4439             unlikely((int)start + off > skb_headlen(skb) - 2)) {
4440                 net_warn_ratelimited("bad partial csum: csum=%u/%u len=%u\n",
4441                                      start, off, skb_headlen(skb));
4442                 return false;
4443         }
4444         skb->ip_summed = CHECKSUM_PARTIAL;
4445         skb->csum_start = skb_headroom(skb) + start;
4446         skb->csum_offset = off;
4447         skb_set_transport_header(skb, start);
4448         return true;
4449 }
4450 EXPORT_SYMBOL_GPL(skb_partial_csum_set);
4451 
4452 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len,
4453                                unsigned int max)
4454 {
4455         if (skb_headlen(skb) >= len)
4456                 return 0;
4457 
4458         /* If we need to pullup then pullup to the max, so we
4459          * won't need to do it again.
4460          */
4461         if (max > skb->len)
4462                 max = skb->len;
4463 
4464         if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL)
4465                 return -ENOMEM;
4466 
4467         if (skb_headlen(skb) < len)
4468                 return -EPROTO;
4469 
4470         return 0;
4471 }
4472 
4473 #define MAX_TCP_HDR_LEN (15 * 4)
4474 
4475 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb,
4476                                       typeof(IPPROTO_IP) proto,
4477                                       unsigned int off)
4478 {
4479         switch (proto) {
4480                 int err;
4481 
4482         case IPPROTO_TCP:
4483                 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
4484                                           off + MAX_TCP_HDR_LEN);
4485                 if (!err && !skb_partial_csum_set(skb, off,
4486                                                   offsetof(struct tcphdr,
4487                                                            check)))
4488                         err = -EPROTO;
4489                 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check;
4490 
4491         case IPPROTO_UDP:
4492                 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr),
4493                                           off + sizeof(struct udphdr));
4494                 if (!err && !skb_partial_csum_set(skb, off,
4495                                                   offsetof(struct udphdr,
4496                                                            check)))
4497                         err = -EPROTO;
4498                 return err ? ERR_PTR(err) : &udp_hdr(skb)->check;
4499         }
4500 
4501         return ERR_PTR(-EPROTO);
4502 }
4503 
4504 /* This value should be large enough to cover a tagged ethernet header plus
4505  * maximally sized IP and TCP or UDP headers.
4506  */
4507 #define MAX_IP_HDR_LEN 128
4508 
4509 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate)
4510 {
4511         unsigned int off;
4512         bool fragment;
4513         __sum16 *csum;
4514         int err;
4515 
4516         fragment = false;
4517 
4518         err = skb_maybe_pull_tail(skb,
4519                                   sizeof(struct iphdr),
4520                                   MAX_IP_HDR_LEN);
4521         if (err < 0)
4522                 goto out;
4523 
4524         if (ip_hdr(skb)->frag_off & htons(IP_OFFSET | IP_MF))
4525                 fragment = true;
4526 
4527         off = ip_hdrlen(skb);
4528 
4529         err = -EPROTO;
4530 
4531         if (fragment)
4532                 goto out;
4533 
4534         csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off);
4535         if (IS_ERR(csum))
4536                 return PTR_ERR(csum);
4537 
4538         if (recalculate)
4539                 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
4540                                            ip_hdr(skb)->daddr,
4541                                            skb->len - off,
4542                                            ip_hdr(skb)->protocol, 0);
4543         err = 0;
4544 
4545 out:
4546         return err;
4547 }
4548 
4549 /* This value should be large enough to cover a tagged ethernet header plus
4550  * an IPv6 header, all options, and a maximal TCP or UDP header.
4551  */
4552 #define MAX_IPV6_HDR_LEN 256
4553 
4554 #define OPT_HDR(type, skb, off) \
4555         (type *)(skb_network_header(skb) + (off))
4556 
4557 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
4558 {
4559         int err;
4560         u8 nexthdr;
4561         unsigned int off;
4562         unsigned int len;
4563         bool fragment;
4564         bool done;
4565         __sum16 *csum;
4566 
4567         fragment = false;
4568         done = false;
4569 
4570         off = sizeof(struct ipv6hdr);
4571 
4572         err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN);
4573         if (err < 0)
4574                 goto out;
4575 
4576         nexthdr = ipv6_hdr(skb)->nexthdr;
4577 
4578         len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len);
4579         while (off <= len && !done) {
4580                 switch (nexthdr) {
4581                 case IPPROTO_DSTOPTS:
4582                 case IPPROTO_HOPOPTS:
4583                 case IPPROTO_ROUTING: {
4584                         struct ipv6_opt_hdr *hp;
4585 
4586                         err = skb_maybe_pull_tail(skb,
4587                                                   off +
4588                                                   sizeof(struct ipv6_opt_hdr),
4589                                                   MAX_IPV6_HDR_LEN);
4590                         if (err < 0)
4591                                 goto out;
4592 
4593                         hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
4594                         nexthdr = hp->nexthdr;
4595                         off += ipv6_optlen(hp);
4596                         break;
4597                 }
4598                 case IPPROTO_AH: {
4599                         struct ip_auth_hdr *hp;
4600 
4601                         err = skb_maybe_pull_tail(skb,
4602                                                   off +
4603                                                   sizeof(struct ip_auth_hdr),
4604                                                   MAX_IPV6_HDR_LEN);
4605                         if (err < 0)
4606                                 goto out;
4607 
4608                         hp = OPT_HDR(struct ip_auth_hdr, skb, off);
4609                         nexthdr = hp->nexthdr;
4610                         off += ipv6_authlen(hp);
4611                         break;
4612                 }
4613                 case IPPROTO_FRAGMENT: {
4614                         struct frag_hdr *hp;
4615 
4616                         err = skb_maybe_pull_tail(skb,
4617                                                   off +
4618                                                   sizeof(struct frag_hdr),
4619                                                   MAX_IPV6_HDR_LEN);
4620                         if (err < 0)
4621                                 goto out;
4622 
4623                         hp = OPT_HDR(struct frag_hdr, skb, off);
4624 
4625                         if (hp->frag_off & htons(IP6_OFFSET | IP6_MF))
4626                                 fragment = true;
4627 
4628                         nexthdr = hp->nexthdr;
4629                         off += sizeof(struct frag_hdr);
4630                         break;
4631                 }
4632                 default:
4633                         done = true;
4634                         break;
4635                 }
4636         }
4637 
4638         err = -EPROTO;
4639 
4640         if (!done || fragment)
4641                 goto out;
4642 
4643         csum = skb_checksum_setup_ip(skb, nexthdr, off);
4644         if (IS_ERR(csum))
4645                 return PTR_ERR(csum);
4646 
4647         if (recalculate)
4648                 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4649                                          &ipv6_hdr(skb)->daddr,
4650                                          skb->len - off, nexthdr, 0);
4651         err = 0;
4652 
4653 out:
4654         return err;
4655 }
4656 
4657 /**
4658  * skb_checksum_setup - set up partial checksum offset
4659  * @skb: the skb to set up
4660  * @recalculate: if true the pseudo-header checksum will be recalculated
4661  */
4662 int skb_checksum_setup(struct sk_buff *skb, bool recalculate)
4663 {
4664         int err;
4665 
4666         switch (skb->protocol) {
4667         case htons(ETH_P_IP):
4668                 err = skb_checksum_setup_ipv4(skb, recalculate);
4669                 break;
4670 
4671         case htons(ETH_P_IPV6):
4672                 err = skb_checksum_setup_ipv6(skb, recalculate);
4673                 break;
4674 
4675         default:
4676                 err = -EPROTO;
4677                 break;
4678         }
4679 
4680         return err;
4681 }
4682 EXPORT_SYMBOL(skb_checksum_setup);
4683 
4684 /**
4685  * skb_checksum_maybe_trim - maybe trims the given skb
4686  * @skb: the skb to check
4687  * @transport_len: the data length beyond the network header
4688  *
4689  * Checks whether the given skb has data beyond the given transport length.
4690  * If so, returns a cloned skb trimmed to this transport length.
4691  * Otherwise returns the provided skb. Returns NULL in error cases
4692  * (e.g. transport_len exceeds skb length or out-of-memory).
4693  *
4694  * Caller needs to set the skb transport header and free any returned skb if it
4695  * differs from the provided skb.
4696  */
4697 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb,
4698                                                unsigned int transport_len)
4699 {
4700         struct sk_buff *skb_chk;
4701         unsigned int len = skb_transport_offset(skb) + transport_len;
4702         int ret;
4703 
4704         if (skb->len < len)
4705                 return NULL;
4706         else if (skb->len == len)
4707                 return skb;
4708 
4709         skb_chk = skb_clone(skb, GFP_ATOMIC);
4710         if (!skb_chk)
4711                 return NULL;
4712 
4713         ret = pskb_trim_rcsum(skb_chk, len);
4714         if (ret) {
4715                 kfree_skb(skb_chk);
4716                 return NULL;
4717         }
4718 
4719         return skb_chk;
4720 }
4721 
4722 /**
4723  * skb_checksum_trimmed - validate checksum of an skb
4724  * @skb: the skb to check
4725  * @transport_len: the data length beyond the network header
4726  * @skb_chkf: checksum function to use
4727  *
4728  * Applies the given checksum function skb_chkf to the provided skb.
4729  * Returns a checked and maybe trimmed skb. Returns NULL on error.
4730  *
4731  * If the skb has data beyond the given transport length, then a
4732  * trimmed & cloned skb is checked and returned.
4733  *
4734  * Caller needs to set the skb transport header and free any returned skb if it
4735  * differs from the provided skb.
4736  */
4737 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
4738                                      unsigned int transport_len,
4739                                      __sum16(*skb_chkf)(struct sk_buff *skb))
4740 {
4741         struct sk_buff *skb_chk;
4742         unsigned int offset = skb_transport_offset(skb);
4743         __sum16 ret;
4744 
4745         skb_chk = skb_checksum_maybe_trim(skb, transport_len);
4746         if (!skb_chk)
4747                 goto err;
4748 
4749         if (!pskb_may_pull(skb_chk, offset))
4750                 goto err;
4751 
4752         skb_pull_rcsum(skb_chk, offset);
4753         ret = skb_chkf(skb_chk);
4754         skb_push_rcsum(skb_chk, offset);
4755 
4756         if (ret)
4757                 goto err;
4758 
4759         return skb_chk;
4760 
4761 err:
4762         if (skb_chk && skb_chk != skb)
4763                 kfree_skb(skb_chk);
4764 
4765         return NULL;
4766 
4767 }
4768 EXPORT_SYMBOL(skb_checksum_trimmed);
4769 
4770 void __skb_warn_lro_forwarding(const struct sk_buff *skb)
4771 {
4772         net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
4773                              skb->dev->name);
4774 }
4775 EXPORT_SYMBOL(__skb_warn_lro_forwarding);
4776 
4777 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
4778 {
4779         if (head_stolen) {
4780                 skb_release_head_state(skb);
4781                 kmem_cache_free(skbuff_head_cache, skb);
4782         } else {
4783                 __kfree_skb(skb);
4784         }
4785 }
4786 EXPORT_SYMBOL(kfree_skb_partial);
4787 
4788 /**
4789  * skb_try_coalesce - try to merge skb to prior one
4790  * @to: prior buffer
4791  * @from: buffer to add
4792  * @fragstolen: pointer to boolean
4793  * @delta_truesize: how much more was allocated than was requested
4794  */
4795 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
4796                       bool *fragstolen, int *delta_truesize)
4797 {
4798         struct skb_shared_info *to_shinfo, *from_shinfo;
4799         int i, delta, len = from->len;
4800 
4801         *fragstolen = false;
4802 
4803         if (skb_cloned(to))
4804                 return false;
4805 
4806         if (len <= skb_tailroom(to)) {
4807                 if (len)
4808                         BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
4809                 *delta_truesize = 0;
4810                 return true;
4811         }
4812 
4813         to_shinfo = skb_shinfo(to);
4814         from_shinfo = skb_shinfo(from);
4815         if (to_shinfo->frag_list || from_shinfo->frag_list)
4816                 return false;
4817         if (skb_zcopy(to) || skb_zcopy(from))
4818                 return false;
4819 
4820         if (skb_headlen(from) != 0) {
4821                 struct page *page;
4822                 unsigned int offset;
4823 
4824                 if (to_shinfo->nr_frags +
4825                     from_shinfo->nr_frags >= MAX_SKB_FRAGS)
4826                         return false;
4827 
4828                 if (skb_head_is_locked(from))
4829                         return false;
4830 
4831                 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
4832 
4833                 page = virt_to_head_page(from->head);
4834                 offset = from->data - (unsigned char *)page_address(page);
4835 
4836                 skb_fill_page_desc(to, to_shinfo->nr_frags,
4837                                    page, offset, skb_headlen(from));
4838                 *fragstolen = true;
4839         } else {
4840                 if (to_shinfo->nr_frags +
4841                     from_shinfo->nr_frags > MAX_SKB_FRAGS)
4842                         return false;
4843 
4844                 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
4845         }
4846 
4847         WARN_ON_ONCE(delta < len);
4848 
4849         memcpy(to_shinfo->frags + to_shinfo->nr_frags,
4850                from_shinfo->frags,
4851                from_shinfo->nr_frags * sizeof(skb_frag_t));
4852         to_shinfo->nr_frags += from_shinfo->nr_frags;
4853 
4854         if (!skb_cloned(from))
4855                 from_shinfo->nr_frags = 0;
4856 
4857         /* if the skb is not cloned this does nothing
4858          * since we set nr_frags to 0.
4859          */
4860         for (i = 0; i < from_shinfo->nr_frags; i++)
4861                 __skb_frag_ref(&from_shinfo->frags[i]);
4862 
4863         to->truesize += delta;
4864         to->len += len;
4865         to->data_len += len;
4866 
4867         *delta_truesize = delta;
4868         return true;
4869 }
4870 EXPORT_SYMBOL(skb_try_coalesce);
4871 
4872 /**
4873  * skb_scrub_packet - scrub an skb
4874  *
4875  * @skb: buffer to clean
4876  * @xnet: packet is crossing netns
4877  *
4878  * skb_scrub_packet can be used after encapsulating or decapsulting a packet
4879  * into/from a tunnel. Some information have to be cleared during these
4880  * operations.
4881  * skb_scrub_packet can also be used to clean a skb before injecting it in
4882  * another namespace (@xnet == true). We have to clear all information in the
4883  * skb that could impact namespace isolation.
4884  */
4885 void skb_scrub_packet(struct sk_buff *skb, bool xnet)
4886 {
4887         skb->tstamp = 0;
4888         skb->pkt_type = PACKET_HOST;
4889         skb->skb_iif = 0;
4890         skb->ignore_df = 0;
4891         skb_dst_drop(skb);
4892         secpath_reset(skb);
4893         nf_reset(skb);
4894         nf_reset_trace(skb);
4895 
4896         if (!xnet)
4897                 return;
4898 
4899         ipvs_reset(skb);
4900         skb_orphan(skb);
4901         skb->mark = 0;
4902 }
4903 EXPORT_SYMBOL_GPL(skb_scrub_packet);
4904 
4905 /**
4906  * skb_gso_transport_seglen - Return length of individual segments of a gso packet
4907  *
4908  * @skb: GSO skb
4909  *
4910  * skb_gso_transport_seglen is used to determine the real size of the
4911  * individual segments, including Layer4 headers (TCP/UDP).
4912  *
4913  * The MAC/L2 or network (IP, IPv6) headers are not accounted for.
4914  */
4915 static unsigned int skb_gso_transport_seglen(const struct sk_buff *skb)
4916 {
4917         const struct skb_shared_info *shinfo = skb_shinfo(skb);
4918         unsigned int thlen = 0;
4919 
4920         if (skb->encapsulation) {
4921                 thlen = skb_inner_transport_header(skb) -
4922                         skb_transport_header(skb);
4923 
4924                 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
4925                         thlen += inner_tcp_hdrlen(skb);
4926         } else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
4927                 thlen = tcp_hdrlen(skb);
4928         } else if (unlikely(skb_is_gso_sctp(skb))) {
4929                 thlen = sizeof(struct sctphdr);
4930         }
4931         /* UFO sets gso_size to the size of the fragmentation
4932          * payload, i.e. the size of the L4 (UDP) header is already
4933          * accounted for.
4934          */
4935         return thlen + shinfo->gso_size;
4936 }
4937 
4938 /**
4939  * skb_gso_network_seglen - Return length of individual segments of a gso packet
4940  *
4941  * @skb: GSO skb
4942  *
4943  * skb_gso_network_seglen is used to determine the real size of the
4944  * individual segments, including Layer3 (IP, IPv6) and L4 headers (TCP/UDP).
4945  *
4946  * The MAC/L2 header is not accounted for.
4947  */
4948 static unsigned int skb_gso_network_seglen(const struct sk_buff *skb)
4949 {
4950         unsigned int hdr_len = skb_transport_header(skb) -
4951                                skb_network_header(skb);
4952 
4953         return hdr_len + skb_gso_transport_seglen(skb);
4954 }
4955 
4956 /**
4957  * skb_gso_mac_seglen - Return length of individual segments of a gso packet
4958  *
4959  * @skb: GSO skb
4960  *
4961  * skb_gso_mac_seglen is used to determine the real size of the
4962  * individual segments, including MAC/L2, Layer3 (IP, IPv6) and L4
4963  * headers (TCP/UDP).
4964  */
4965 static unsigned int skb_gso_mac_seglen(const struct sk_buff *skb)
4966 {
4967         unsigned int hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
4968 
4969         return hdr_len + skb_gso_transport_seglen(skb);
4970 }
4971 
4972 /**
4973  * skb_gso_size_check - check the skb size, considering GSO_BY_FRAGS
4974  *
4975  * There are a couple of instances where we have a GSO skb, and we
4976  * want to determine what size it would be after it is segmented.
4977  *
4978  * We might want to check:
4979  * -    L3+L4+payload size (e.g. IP forwarding)
4980  * - L2+L3+L4+payload size (e.g. sanity check before passing to driver)
4981  *
4982  * This is a helper to do that correctly considering GSO_BY_FRAGS.
4983  *
4984  * @seg_len: The segmented length (from skb_gso_*_seglen). In the
4985  *           GSO_BY_FRAGS case this will be [header sizes + GSO_BY_FRAGS].
4986  *
4987  * @max_len: The maximum permissible length.
4988  *
4989  * Returns true if the segmented length <= max length.
4990  */
4991 static inline bool skb_gso_size_check(const struct sk_buff *skb,
4992                                       unsigned int seg_len,
4993                                       unsigned int max_len) {
4994         const struct skb_shared_info *shinfo = skb_shinfo(skb);
4995         const struct sk_buff *iter;
4996 
4997         if (shinfo->gso_size != GSO_BY_FRAGS)
4998                 return seg_len <= max_len;
4999 
5000         /* Undo this so we can re-use header sizes */
5001         seg_len -= GSO_BY_FRAGS;
5002 
5003         skb_walk_frags(skb, iter) {
5004                 if (seg_len + skb_headlen(iter) > max_len)
5005                         return false;
5006         }
5007 
5008         return true;
5009 }
5010 
5011 /**
5012  * skb_gso_validate_network_len - Will a split GSO skb fit into a given MTU?
5013  *
5014  * @skb: GSO skb
5015  * @mtu: MTU to validate against
5016  *
5017  * skb_gso_validate_network_len validates if a given skb will fit a
5018  * wanted MTU once split. It considers L3 headers, L4 headers, and the
5019  * payload.
5020  */
5021 bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu)
5022 {
5023         return skb_gso_size_check(skb, skb_gso_network_seglen(skb), mtu);
5024 }
5025 EXPORT_SYMBOL_GPL(skb_gso_validate_network_len);
5026 
5027 /**
5028  * skb_gso_validate_mac_len - Will a split GSO skb fit in a given length?
5029  *
5030  * @skb: GSO skb
5031  * @len: length to validate against
5032  *
5033  * skb_gso_validate_mac_len validates if a given skb will fit a wanted
5034  * length once split, including L2, L3 and L4 headers and the payload.
5035  */
5036 bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len)
5037 {
5038         return skb_gso_size_check(skb, skb_gso_mac_seglen(skb), len);
5039 }
5040 EXPORT_SYMBOL_GPL(skb_gso_validate_mac_len);
5041 
5042 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb)
5043 {
5044         int mac_len;
5045 
5046         if (skb_cow(skb, skb_headroom(skb)) < 0) {
5047                 kfree_skb(skb);
5048                 return NULL;
5049         }
5050 
5051         mac_len = skb->data - skb_mac_header(skb);
5052         if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) {
5053                 memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb),
5054                         mac_len - VLAN_HLEN - ETH_TLEN);
5055         }
5056         skb->mac_header += VLAN_HLEN;
5057         return skb;
5058 }
5059 
5060 struct sk_buff *skb_vlan_untag(struct sk_buff *skb)
5061 {
5062         struct vlan_hdr *vhdr;
5063         u16 vlan_tci;
5064 
5065         if (unlikely(skb_vlan_tag_present(skb))) {
5066                 /* vlan_tci is already set-up so leave this for another time */
5067                 return skb;
5068         }
5069 
5070         skb = skb_share_check(skb, GFP_ATOMIC);
5071         if (unlikely(!skb))
5072                 goto err_free;
5073 
5074         if (unlikely(!pskb_may_pull(skb, VLAN_HLEN)))
5075                 goto err_free;
5076 
5077         vhdr = (struct vlan_hdr *)skb->data;
5078         vlan_tci = ntohs(vhdr->h_vlan_TCI);
5079         __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci);
5080 
5081         skb_pull_rcsum(skb, VLAN_HLEN);
5082         vlan_set_encap_proto(skb, vhdr);
5083 
5084         skb = skb_reorder_vlan_header(skb);
5085         if (unlikely(!skb))
5086                 goto err_free;
5087 
5088         skb_reset_network_header(skb);
5089         skb_reset_transport_header(skb);
5090         skb_reset_mac_len(skb);
5091 
5092         return skb;
5093 
5094 err_free:
5095         kfree_skb(skb);
5096         return NULL;
5097 }
5098 EXPORT_SYMBOL(skb_vlan_untag);
5099 
5100 int skb_ensure_writable(struct sk_buff *skb, int write_len)
5101 {
5102         if (!pskb_may_pull(skb, write_len))
5103                 return -ENOMEM;
5104 
5105         if (!skb_cloned(skb) || skb_clone_writable(skb, write_len))
5106                 return 0;
5107 
5108         return pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
5109 }
5110 EXPORT_SYMBOL(skb_ensure_writable);
5111 
5112 /* remove VLAN header from packet and update csum accordingly.
5113  * expects a non skb_vlan_tag_present skb with a vlan tag payload
5114  */
5115 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci)
5116 {
5117         struct vlan_hdr *vhdr;
5118         int offset = skb->data - skb_mac_header(skb);
5119         int err;
5120 
5121         if (WARN_ONCE(offset,
5122                       "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n",
5123                       offset)) {
5124                 return -EINVAL;
5125         }
5126 
5127         err = skb_ensure_writable(skb, VLAN_ETH_HLEN);
5128         if (unlikely(err))
5129                 return err;
5130 
5131         skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
5132 
5133         vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN);
5134         *vlan_tci = ntohs(vhdr->h_vlan_TCI);
5135 
5136         memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN);
5137         __skb_pull(skb, VLAN_HLEN);
5138 
5139         vlan_set_encap_proto(skb, vhdr);
5140         skb->mac_header += VLAN_HLEN;
5141 
5142         if (skb_network_offset(skb) < ETH_HLEN)
5143                 skb_set_network_header(skb, ETH_HLEN);
5144 
5145         skb_reset_mac_len(skb);
5146 
5147         return err;
5148 }
5149 EXPORT_SYMBOL(__skb_vlan_pop);
5150 
5151 /* Pop a vlan tag either from hwaccel or from payload.
5152  * Expects skb->data at mac header.
5153  */
5154 int skb_vlan_pop(struct sk_buff *skb)
5155 {
5156         u16 vlan_tci;
5157         __be16 vlan_proto;
5158         int err;
5159 
5160         if (likely(skb_vlan_tag_present(skb))) {
5161                 skb->vlan_tci = 0;
5162         } else {
5163                 if (unlikely(!eth_type_vlan(skb->protocol)))
5164                         return 0;
5165 
5166                 err = __skb_vlan_pop(skb, &vlan_tci);
5167                 if (err)
5168                         return err;
5169         }
5170         /* move next vlan tag to hw accel tag */
5171         if (likely(!eth_type_vlan(skb->protocol)))
5172                 return 0;
5173 
5174         vlan_proto = skb->protocol;
5175         err = __skb_vlan_pop(skb, &vlan_tci);
5176         if (unlikely(err))
5177                 return err;
5178 
5179         __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
5180         return 0;
5181 }
5182 EXPORT_SYMBOL(skb_vlan_pop);
5183 
5184 /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present).
5185  * Expects skb->data at mac header.
5186  */
5187 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci)
5188 {
5189         if (skb_vlan_tag_present(skb)) {
5190                 int offset = skb->data - skb_mac_header(skb);
5191                 int err;
5192 
5193                 if (WARN_ONCE(offset,
5194                               "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n",
5195                               offset)) {
5196                         return -EINVAL;
5197                 }
5198 
5199                 err = __vlan_insert_tag(skb, skb->vlan_proto,
5200                                         skb_vlan_tag_get(skb));
5201                 if (err)
5202                         return err;
5203 
5204                 skb->protocol = skb->vlan_proto;
5205                 skb->mac_len += VLAN_HLEN;
5206 
5207                 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
5208         }
5209         __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
5210         return 0;
5211 }
5212 EXPORT_SYMBOL(skb_vlan_push);
5213 
5214 /**
5215  * alloc_skb_with_frags - allocate skb with page frags
5216  *
5217  * @header_len: size of linear part
5218  * @data_len: needed length in frags
5219  * @max_page_order: max page order desired.
5220  * @errcode: pointer to error code if any
5221  * @gfp_mask: allocation mask
5222  *
5223  * This can be used to allocate a paged skb, given a maximal order for frags.
5224  */
5225 struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
5226                                      unsigned long data_len,
5227                                      int max_page_order,
5228                                      int *errcode,
5229                                      gfp_t gfp_mask)
5230 {
5231         int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
5232         unsigned long chunk;
5233         struct sk_buff *skb;
5234         struct page *page;
5235         gfp_t gfp_head;
5236         int i;
5237 
5238         *errcode = -EMSGSIZE;
5239         /* Note this test could be relaxed, if we succeed to allocate
5240          * high order pages...
5241          */
5242         if (npages > MAX_SKB_FRAGS)
5243                 return NULL;
5244 
5245         gfp_head = gfp_mask;
5246         if (gfp_head & __GFP_DIRECT_RECLAIM)
5247                 gfp_head |= __GFP_RETRY_MAYFAIL;
5248 
5249         *errcode = -ENOBUFS;
5250         skb = alloc_skb(header_len, gfp_head);
5251         if (!skb)
5252                 return NULL;
5253 
5254         skb->truesize += npages << PAGE_SHIFT;
5255 
5256         for (i = 0; npages > 0; i++) {
5257                 int order = max_page_order;
5258 
5259                 while (order) {
5260                         if (npages >= 1 << order) {
5261                                 page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) |
5262                                                    __GFP_COMP |
5263                                                    __GFP_NOWARN |
5264                                                    __GFP_NORETRY,
5265                                                    order);
5266                                 if (page)
5267                                         goto fill_page;
5268                                 /* Do not retry other high order allocations */
5269                                 order = 1;
5270                                 max_page_order = 0;
5271                         }
5272                         order--;
5273                 }
5274                 page = alloc_page(gfp_mask);
5275                 if (!page)
5276                         goto failure;
5277 fill_page:
5278                 chunk = min_t(unsigned long, data_len,
5279                               PAGE_SIZE << order);
5280                 skb_fill_page_desc(skb, i, page, 0, chunk);
5281                 data_len -= chunk;
5282                 npages -= 1 << order;
5283         }
5284         return skb;
5285 
5286 failure:
5287         kfree_skb(skb);
5288         return NULL;
5289 }
5290 EXPORT_SYMBOL(alloc_skb_with_frags);
5291 
5292 /* carve out the first off bytes from skb when off < headlen */
5293 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
5294                                     const int headlen, gfp_t gfp_mask)
5295 {
5296         int i;
5297         int size = skb_end_offset(skb);
5298         int new_hlen = headlen - off;
5299         u8 *data;
5300 
5301         size = SKB_DATA_ALIGN(size);
5302 
5303         if (skb_pfmemalloc(skb))
5304                 gfp_mask |= __GFP_MEMALLOC;
5305         data = kmalloc_reserve(size +
5306                                SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
5307                                gfp_mask, NUMA_NO_NODE, NULL);
5308         if (!data)
5309                 return -ENOMEM;
5310 
5311         size = SKB_WITH_OVERHEAD(ksize(data));
5312 
5313         /* Copy real data, and all frags */
5314         skb_copy_from_linear_data_offset(skb, off, data, new_hlen);
5315         skb->len -= off;
5316 
5317         memcpy((struct skb_shared_info *)(data + size),
5318                skb_shinfo(skb),
5319                offsetof(struct skb_shared_info,
5320                         frags[skb_shinfo(skb)->nr_frags]));
5321         if (skb_cloned(skb)) {
5322                 /* drop the old head gracefully */
5323                 if (skb_orphan_frags(skb, gfp_mask)) {
5324                         kfree(data);
5325                         return -ENOMEM;
5326                 }
5327                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
5328                         skb_frag_ref(skb, i);
5329                 if (skb_has_frag_list(skb))
5330                         skb_clone_fraglist(skb);
5331                 skb_release_data(skb);
5332         } else {
5333                 /* we can reuse existing recount- all we did was
5334                  * relocate values
5335                  */
5336                 skb_free_head(skb);
5337         }
5338 
5339         skb->head = data;
5340         skb->data = data;
5341         skb->head_frag = 0;
5342 #ifdef NET_SKBUFF_DATA_USES_OFFSET
5343         skb->end = size;
5344 #else
5345         skb->end = skb->head + size;
5346 #endif
5347         skb_set_tail_pointer(skb, skb_headlen(skb));
5348         skb_headers_offset_update(skb, 0);
5349         skb->cloned = 0;
5350         skb->hdr_len = 0;
5351         skb->nohdr = 0;
5352         atomic_set(&skb_shinfo(skb)->dataref, 1);
5353 
5354         return 0;
5355 }
5356 
5357 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp);
5358 
5359 /* carve out the first eat bytes from skb's frag_list. May recurse into
5360  * pskb_carve()
5361  */
5362 static int pskb_carve_frag_list(struct sk_buff *skb,
5363                                 struct skb_shared_info *shinfo, int eat,
5364                                 gfp_t gfp_mask)
5365 {
5366         struct sk_buff *list = shinfo->frag_list;
5367         struct sk_buff *clone = NULL;
5368         struct sk_buff *insp = NULL;
5369 
5370         do {
5371                 if (!list) {
5372                         pr_err("Not enough bytes to eat. Want %d\n", eat);
5373                         return -EFAULT;
5374                 }
5375                 if (list->len <= eat) {
5376                         /* Eaten as whole. */
5377                         eat -= list->len;
5378                         list = list->next;
5379                         insp = list;
5380                 } else {
5381                         /* Eaten partially. */
5382                         if (skb_shared(list)) {
5383                                 clone = skb_clone(list, gfp_mask);
5384                                 if (!clone)
5385                                         return -ENOMEM;
5386                                 insp = list->next;
5387                                 list = clone;
5388                         } else {
5389                                 /* This may be pulled without problems. */
5390                                 insp = list;
5391                         }
5392                         if (pskb_carve(list, eat, gfp_mask) < 0) {
5393                                 kfree_skb(clone);
5394                                 return -ENOMEM;
5395                         }
5396                         break;
5397                 }
5398         } while (eat);
5399 
5400         /* Free pulled out fragments. */
5401         while ((list = shinfo->frag_list) != insp) {
5402                 shinfo->frag_list = list->next;
5403                 kfree_skb(list);
5404         }
5405         /* And insert new clone at head. */
5406         if (clone) {
5407                 clone->next = list;
5408                 shinfo->frag_list = clone;
5409         }
5410         return 0;
5411 }
5412 
5413 /* carve off first len bytes from skb. Split line (off) is in the
5414  * non-linear part of skb
5415  */
5416 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
5417                                        int pos, gfp_t gfp_mask)
5418 {
5419         int i, k = 0;
5420         int size = skb_end_offset(skb);
5421         u8 *data;
5422         const int nfrags = skb_shinfo(skb)->nr_frags;
5423         struct skb_shared_info *shinfo;
5424 
5425         size = SKB_DATA_ALIGN(size);
5426 
5427         if (skb_pfmemalloc(skb))
5428                 gfp_mask |= __GFP_MEMALLOC;
5429         data = kmalloc_reserve(size +
5430                                SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
5431                                gfp_mask, NUMA_NO_NODE, NULL);
5432         if (!data)
5433                 return -ENOMEM;
5434 
5435         size = SKB_WITH_OVERHEAD(ksize(data));
5436 
5437         memcpy((struct skb_shared_info *)(data + size),
5438                skb_shinfo(skb), offsetof(struct skb_shared_info,
5439                                          frags[skb_shinfo(skb)->nr_frags]));
5440         if (skb_orphan_frags(skb, gfp_mask)) {
5441                 kfree(data);
5442                 return -ENOMEM;
5443         }
5444         shinfo = (struct skb_shared_info *)(data + size);
5445         for (i = 0; i < nfrags; i++) {
5446                 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]);
5447 
5448                 if (pos + fsize > off) {
5449                         shinfo->frags[k] = skb_shinfo(skb)->frags[i];
5450 
5451                         if (pos < off) {
5452                                 /* Split frag.
5453                                  * We have two variants in this case:
5454                                  * 1. Move all the frag to the second
5455                                  *    part, if it is possible. F.e.
5456                                  *    this approach is mandatory for TUX,
5457                                  *    where splitting is expensive.
5458                                  * 2. Split is accurately. We make this.
5459                                  */
5460                                 shinfo->frags[0].page_offset += off - pos;
5461                                 skb_frag_size_sub(&shinfo->frags[0], off - pos);
5462                         }
5463                         skb_frag_ref(skb, i);
5464                         k++;
5465                 }
5466                 pos += fsize;
5467         }
5468         shinfo->nr_frags = k;
5469         if (skb_has_frag_list(skb))
5470                 skb_clone_fraglist(skb);
5471 
5472         if (k == 0) {
5473                 /* split line is in frag list */
5474                 pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask);
5475         }
5476         skb_release_data(skb);
5477 
5478         skb->head = data;
5479         skb->head_frag = 0;
5480         skb->data = data;
5481 #ifdef NET_SKBUFF_DATA_USES_OFFSET
5482         skb->end = size;
5483 #else
5484         skb->end = skb->head + size;
5485 #endif
5486         skb_reset_tail_pointer(skb);
5487         skb_headers_offset_update(skb, 0);
5488         skb->cloned   = 0;
5489         skb->hdr_len  = 0;
5490         skb->nohdr    = 0;
5491         skb->len -= off;
5492         skb->data_len = skb->len;
5493         atomic_set(&skb_shinfo(skb)->dataref, 1);
5494         return 0;
5495 }
5496 
5497 /* remove len bytes from the beginning of the skb */
5498 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp)
5499 {
5500         int headlen = skb_headlen(skb);
5501 
5502         if (len < headlen)
5503                 return pskb_carve_inside_header(skb, len, headlen, gfp);
5504         else
5505                 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp);
5506 }
5507 
5508 /* Extract to_copy bytes starting at off from skb, and return this in
5509  * a new skb
5510  */
5511 struct sk_buff *pskb_extract(struct sk_buff *skb, int off,
5512                              int to_copy, gfp_t gfp)
5513 {
5514         struct sk_buff  *clone = skb_clone(skb, gfp);
5515 
5516         if (!clone)
5517                 return NULL;
5518 
5519         if (pskb_carve(clone, off, gfp) < 0 ||
5520             pskb_trim(clone, to_copy)) {
5521                 kfree_skb(clone);
5522                 return NULL;
5523         }
5524         return clone;
5525 }
5526 EXPORT_SYMBOL(pskb_extract);
5527 
5528 /**
5529  * skb_condense - try to get rid of fragments/frag_list if possible
5530  * @skb: buffer
5531  *
5532  * Can be used to save memory before skb is added to a busy queue.
5533  * If packet has bytes in frags and enough tail room in skb->head,
5534  * pull all of them, so that we can free the frags right now and adjust
5535  * truesize.
5536  * Notes:
5537  *      We do not reallocate skb->head thus can not fail.
5538  *      Caller must re-evaluate skb->truesize if needed.
5539  */
5540 void skb_condense(struct sk_buff *skb)
5541 {
5542         if (skb->data_len) {
5543                 if (skb->data_len > skb->end - skb->tail ||
5544                     skb_cloned(skb))
5545                         return;
5546 
5547                 /* Nice, we can free page frag(s) right now */
5548                 __pskb_pull_tail(skb, skb->data_len);
5549         }
5550         /* At this point, skb->truesize might be over estimated,
5551          * because skb had a fragment, and fragments do not tell
5552          * their truesize.
5553          * When we pulled its content into skb->head, fragment
5554          * was freed, but __pskb_pull_tail() could not possibly
5555          * adjust skb->truesize, not knowing the frag truesize.
5556          */
5557         skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
5558 }
5559 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp