~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/core/skbuff.c

Version: ~ [ linux-5.19-rc8 ] ~ [ linux-5.18.14 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.57 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.133 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.207 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.253 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.289 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.324 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *      Routines having to do with the 'struct sk_buff' memory handlers.
  3  *
  4  *      Authors:        Alan Cox <alan@lxorguk.ukuu.org.uk>
  5  *                      Florian La Roche <rzsfl@rz.uni-sb.de>
  6  *
  7  *      Fixes:
  8  *              Alan Cox        :       Fixed the worst of the load
  9  *                                      balancer bugs.
 10  *              Dave Platt      :       Interrupt stacking fix.
 11  *      Richard Kooijman        :       Timestamp fixes.
 12  *              Alan Cox        :       Changed buffer format.
 13  *              Alan Cox        :       destructor hook for AF_UNIX etc.
 14  *              Linus Torvalds  :       Better skb_clone.
 15  *              Alan Cox        :       Added skb_copy.
 16  *              Alan Cox        :       Added all the changed routines Linus
 17  *                                      only put in the headers
 18  *              Ray VanTassle   :       Fixed --skb->lock in free
 19  *              Alan Cox        :       skb_copy copy arp field
 20  *              Andi Kleen      :       slabified it.
 21  *              Robert Olsson   :       Removed skb_head_pool
 22  *
 23  *      NOTE:
 24  *              The __skb_ routines should be called with interrupts
 25  *      disabled, or you better be *real* sure that the operation is atomic
 26  *      with respect to whatever list is being frobbed (e.g. via lock_sock()
 27  *      or via disabling bottom half handlers, etc).
 28  *
 29  *      This program is free software; you can redistribute it and/or
 30  *      modify it under the terms of the GNU General Public License
 31  *      as published by the Free Software Foundation; either version
 32  *      2 of the License, or (at your option) any later version.
 33  */
 34 
 35 /*
 36  *      The functions in this file will not compile correctly with gcc 2.4.x
 37  */
 38 
 39 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 40 
 41 #include <linux/module.h>
 42 #include <linux/types.h>
 43 #include <linux/kernel.h>
 44 #include <linux/mm.h>
 45 #include <linux/interrupt.h>
 46 #include <linux/in.h>
 47 #include <linux/inet.h>
 48 #include <linux/slab.h>
 49 #include <linux/tcp.h>
 50 #include <linux/udp.h>
 51 #include <linux/sctp.h>
 52 #include <linux/netdevice.h>
 53 #ifdef CONFIG_NET_CLS_ACT
 54 #include <net/pkt_sched.h>
 55 #endif
 56 #include <linux/string.h>
 57 #include <linux/skbuff.h>
 58 #include <linux/splice.h>
 59 #include <linux/cache.h>
 60 #include <linux/rtnetlink.h>
 61 #include <linux/init.h>
 62 #include <linux/scatterlist.h>
 63 #include <linux/errqueue.h>
 64 #include <linux/prefetch.h>
 65 #include <linux/if_vlan.h>
 66 
 67 #include <net/protocol.h>
 68 #include <net/dst.h>
 69 #include <net/sock.h>
 70 #include <net/checksum.h>
 71 #include <net/ip6_checksum.h>
 72 #include <net/xfrm.h>
 73 
 74 #include <linux/uaccess.h>
 75 #include <trace/events/skb.h>
 76 #include <linux/highmem.h>
 77 #include <linux/capability.h>
 78 #include <linux/user_namespace.h>
 79 
 80 struct kmem_cache *skbuff_head_cache __read_mostly;
 81 static struct kmem_cache *skbuff_fclone_cache __read_mostly;
 82 int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
 83 EXPORT_SYMBOL(sysctl_max_skb_frags);
 84 
 85 /**
 86  *      skb_panic - private function for out-of-line support
 87  *      @skb:   buffer
 88  *      @sz:    size
 89  *      @addr:  address
 90  *      @msg:   skb_over_panic or skb_under_panic
 91  *
 92  *      Out-of-line support for skb_put() and skb_push().
 93  *      Called via the wrapper skb_over_panic() or skb_under_panic().
 94  *      Keep out of line to prevent kernel bloat.
 95  *      __builtin_return_address is not used because it is not always reliable.
 96  */
 97 static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
 98                       const char msg[])
 99 {
100         pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n",
101                  msg, addr, skb->len, sz, skb->head, skb->data,
102                  (unsigned long)skb->tail, (unsigned long)skb->end,
103                  skb->dev ? skb->dev->name : "<NULL>");
104         BUG();
105 }
106 
107 static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
108 {
109         skb_panic(skb, sz, addr, __func__);
110 }
111 
112 static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
113 {
114         skb_panic(skb, sz, addr, __func__);
115 }
116 
117 /*
118  * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
119  * the caller if emergency pfmemalloc reserves are being used. If it is and
120  * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves
121  * may be used. Otherwise, the packet data may be discarded until enough
122  * memory is free
123  */
124 #define kmalloc_reserve(size, gfp, node, pfmemalloc) \
125          __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc)
126 
127 static void *__kmalloc_reserve(size_t size, gfp_t flags, int node,
128                                unsigned long ip, bool *pfmemalloc)
129 {
130         void *obj;
131         bool ret_pfmemalloc = false;
132 
133         /*
134          * Try a regular allocation, when that fails and we're not entitled
135          * to the reserves, fail.
136          */
137         obj = kmalloc_node_track_caller(size,
138                                         flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
139                                         node);
140         if (obj || !(gfp_pfmemalloc_allowed(flags)))
141                 goto out;
142 
143         /* Try again but now we are using pfmemalloc reserves */
144         ret_pfmemalloc = true;
145         obj = kmalloc_node_track_caller(size, flags, node);
146 
147 out:
148         if (pfmemalloc)
149                 *pfmemalloc = ret_pfmemalloc;
150 
151         return obj;
152 }
153 
154 /*      Allocate a new skbuff. We do this ourselves so we can fill in a few
155  *      'private' fields and also do memory statistics to find all the
156  *      [BEEP] leaks.
157  *
158  */
159 
160 /**
161  *      __alloc_skb     -       allocate a network buffer
162  *      @size: size to allocate
163  *      @gfp_mask: allocation mask
164  *      @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache
165  *              instead of head cache and allocate a cloned (child) skb.
166  *              If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
167  *              allocations in case the data is required for writeback
168  *      @node: numa node to allocate memory on
169  *
170  *      Allocate a new &sk_buff. The returned buffer has no headroom and a
171  *      tail room of at least size bytes. The object has a reference count
172  *      of one. The return is the buffer. On a failure the return is %NULL.
173  *
174  *      Buffers may only be allocated from interrupts using a @gfp_mask of
175  *      %GFP_ATOMIC.
176  */
177 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
178                             int flags, int node)
179 {
180         struct kmem_cache *cache;
181         struct skb_shared_info *shinfo;
182         struct sk_buff *skb;
183         u8 *data;
184         bool pfmemalloc;
185 
186         cache = (flags & SKB_ALLOC_FCLONE)
187                 ? skbuff_fclone_cache : skbuff_head_cache;
188 
189         if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
190                 gfp_mask |= __GFP_MEMALLOC;
191 
192         /* Get the HEAD */
193         skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
194         if (!skb)
195                 goto out;
196         prefetchw(skb);
197 
198         /* We do our best to align skb_shared_info on a separate cache
199          * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
200          * aligned memory blocks, unless SLUB/SLAB debug is enabled.
201          * Both skb->head and skb_shared_info are cache line aligned.
202          */
203         size = SKB_DATA_ALIGN(size);
204         size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
205         data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
206         if (!data)
207                 goto nodata;
208         /* kmalloc(size) might give us more room than requested.
209          * Put skb_shared_info exactly at the end of allocated zone,
210          * to allow max possible filling before reallocation.
211          */
212         size = SKB_WITH_OVERHEAD(ksize(data));
213         prefetchw(data + size);
214 
215         /*
216          * Only clear those fields we need to clear, not those that we will
217          * actually initialise below. Hence, don't put any more fields after
218          * the tail pointer in struct sk_buff!
219          */
220         memset(skb, 0, offsetof(struct sk_buff, tail));
221         /* Account for allocated memory : skb + skb->head */
222         skb->truesize = SKB_TRUESIZE(size);
223         skb->pfmemalloc = pfmemalloc;
224         refcount_set(&skb->users, 1);
225         skb->head = data;
226         skb->data = data;
227         skb_reset_tail_pointer(skb);
228         skb->end = skb->tail + size;
229         skb->mac_header = (typeof(skb->mac_header))~0U;
230         skb->transport_header = (typeof(skb->transport_header))~0U;
231 
232         /* make sure we initialize shinfo sequentially */
233         shinfo = skb_shinfo(skb);
234         memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
235         atomic_set(&shinfo->dataref, 1);
236 
237         if (flags & SKB_ALLOC_FCLONE) {
238                 struct sk_buff_fclones *fclones;
239 
240                 fclones = container_of(skb, struct sk_buff_fclones, skb1);
241 
242                 skb->fclone = SKB_FCLONE_ORIG;
243                 refcount_set(&fclones->fclone_ref, 1);
244 
245                 fclones->skb2.fclone = SKB_FCLONE_CLONE;
246         }
247 out:
248         return skb;
249 nodata:
250         kmem_cache_free(cache, skb);
251         skb = NULL;
252         goto out;
253 }
254 EXPORT_SYMBOL(__alloc_skb);
255 
256 /**
257  * __build_skb - build a network buffer
258  * @data: data buffer provided by caller
259  * @frag_size: size of data, or 0 if head was kmalloced
260  *
261  * Allocate a new &sk_buff. Caller provides space holding head and
262  * skb_shared_info. @data must have been allocated by kmalloc() only if
263  * @frag_size is 0, otherwise data should come from the page allocator
264  *  or vmalloc()
265  * The return is the new skb buffer.
266  * On a failure the return is %NULL, and @data is not freed.
267  * Notes :
268  *  Before IO, driver allocates only data buffer where NIC put incoming frame
269  *  Driver should add room at head (NET_SKB_PAD) and
270  *  MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info))
271  *  After IO, driver calls build_skb(), to allocate sk_buff and populate it
272  *  before giving packet to stack.
273  *  RX rings only contains data buffers, not full skbs.
274  */
275 struct sk_buff *__build_skb(void *data, unsigned int frag_size)
276 {
277         struct skb_shared_info *shinfo;
278         struct sk_buff *skb;
279         unsigned int size = frag_size ? : ksize(data);
280 
281         skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
282         if (!skb)
283                 return NULL;
284 
285         size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
286 
287         memset(skb, 0, offsetof(struct sk_buff, tail));
288         skb->truesize = SKB_TRUESIZE(size);
289         refcount_set(&skb->users, 1);
290         skb->head = data;
291         skb->data = data;
292         skb_reset_tail_pointer(skb);
293         skb->end = skb->tail + size;
294         skb->mac_header = (typeof(skb->mac_header))~0U;
295         skb->transport_header = (typeof(skb->transport_header))~0U;
296 
297         /* make sure we initialize shinfo sequentially */
298         shinfo = skb_shinfo(skb);
299         memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
300         atomic_set(&shinfo->dataref, 1);
301 
302         return skb;
303 }
304 
305 /* build_skb() is wrapper over __build_skb(), that specifically
306  * takes care of skb->head and skb->pfmemalloc
307  * This means that if @frag_size is not zero, then @data must be backed
308  * by a page fragment, not kmalloc() or vmalloc()
309  */
310 struct sk_buff *build_skb(void *data, unsigned int frag_size)
311 {
312         struct sk_buff *skb = __build_skb(data, frag_size);
313 
314         if (skb && frag_size) {
315                 skb->head_frag = 1;
316                 if (page_is_pfmemalloc(virt_to_head_page(data)))
317                         skb->pfmemalloc = 1;
318         }
319         return skb;
320 }
321 EXPORT_SYMBOL(build_skb);
322 
323 #define NAPI_SKB_CACHE_SIZE     64
324 
325 struct napi_alloc_cache {
326         struct page_frag_cache page;
327         unsigned int skb_count;
328         void *skb_cache[NAPI_SKB_CACHE_SIZE];
329 };
330 
331 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
332 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
333 
334 static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
335 {
336         struct page_frag_cache *nc;
337         unsigned long flags;
338         void *data;
339 
340         local_irq_save(flags);
341         nc = this_cpu_ptr(&netdev_alloc_cache);
342         data = page_frag_alloc(nc, fragsz, gfp_mask);
343         local_irq_restore(flags);
344         return data;
345 }
346 
347 /**
348  * netdev_alloc_frag - allocate a page fragment
349  * @fragsz: fragment size
350  *
351  * Allocates a frag from a page for receive buffer.
352  * Uses GFP_ATOMIC allocations.
353  */
354 void *netdev_alloc_frag(unsigned int fragsz)
355 {
356         return __netdev_alloc_frag(fragsz, GFP_ATOMIC);
357 }
358 EXPORT_SYMBOL(netdev_alloc_frag);
359 
360 static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
361 {
362         struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
363 
364         return page_frag_alloc(&nc->page, fragsz, gfp_mask);
365 }
366 
367 void *napi_alloc_frag(unsigned int fragsz)
368 {
369         return __napi_alloc_frag(fragsz, GFP_ATOMIC);
370 }
371 EXPORT_SYMBOL(napi_alloc_frag);
372 
373 /**
374  *      __netdev_alloc_skb - allocate an skbuff for rx on a specific device
375  *      @dev: network device to receive on
376  *      @len: length to allocate
377  *      @gfp_mask: get_free_pages mask, passed to alloc_skb
378  *
379  *      Allocate a new &sk_buff and assign it a usage count of one. The
380  *      buffer has NET_SKB_PAD headroom built in. Users should allocate
381  *      the headroom they think they need without accounting for the
382  *      built in space. The built in space is used for optimisations.
383  *
384  *      %NULL is returned if there is no free memory.
385  */
386 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len,
387                                    gfp_t gfp_mask)
388 {
389         struct page_frag_cache *nc;
390         unsigned long flags;
391         struct sk_buff *skb;
392         bool pfmemalloc;
393         void *data;
394 
395         len += NET_SKB_PAD;
396 
397         if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
398             (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
399                 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
400                 if (!skb)
401                         goto skb_fail;
402                 goto skb_success;
403         }
404 
405         len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
406         len = SKB_DATA_ALIGN(len);
407 
408         if (sk_memalloc_socks())
409                 gfp_mask |= __GFP_MEMALLOC;
410 
411         local_irq_save(flags);
412 
413         nc = this_cpu_ptr(&netdev_alloc_cache);
414         data = page_frag_alloc(nc, len, gfp_mask);
415         pfmemalloc = nc->pfmemalloc;
416 
417         local_irq_restore(flags);
418 
419         if (unlikely(!data))
420                 return NULL;
421 
422         skb = __build_skb(data, len);
423         if (unlikely(!skb)) {
424                 skb_free_frag(data);
425                 return NULL;
426         }
427 
428         /* use OR instead of assignment to avoid clearing of bits in mask */
429         if (pfmemalloc)
430                 skb->pfmemalloc = 1;
431         skb->head_frag = 1;
432 
433 skb_success:
434         skb_reserve(skb, NET_SKB_PAD);
435         skb->dev = dev;
436 
437 skb_fail:
438         return skb;
439 }
440 EXPORT_SYMBOL(__netdev_alloc_skb);
441 
442 /**
443  *      __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance
444  *      @napi: napi instance this buffer was allocated for
445  *      @len: length to allocate
446  *      @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages
447  *
448  *      Allocate a new sk_buff for use in NAPI receive.  This buffer will
449  *      attempt to allocate the head from a special reserved region used
450  *      only for NAPI Rx allocation.  By doing this we can save several
451  *      CPU cycles by avoiding having to disable and re-enable IRQs.
452  *
453  *      %NULL is returned if there is no free memory.
454  */
455 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
456                                  gfp_t gfp_mask)
457 {
458         struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
459         struct sk_buff *skb;
460         void *data;
461 
462         len += NET_SKB_PAD + NET_IP_ALIGN;
463 
464         if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
465             (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
466                 skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
467                 if (!skb)
468                         goto skb_fail;
469                 goto skb_success;
470         }
471 
472         len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
473         len = SKB_DATA_ALIGN(len);
474 
475         if (sk_memalloc_socks())
476                 gfp_mask |= __GFP_MEMALLOC;
477 
478         data = page_frag_alloc(&nc->page, len, gfp_mask);
479         if (unlikely(!data))
480                 return NULL;
481 
482         skb = __build_skb(data, len);
483         if (unlikely(!skb)) {
484                 skb_free_frag(data);
485                 return NULL;
486         }
487 
488         /* use OR instead of assignment to avoid clearing of bits in mask */
489         if (nc->page.pfmemalloc)
490                 skb->pfmemalloc = 1;
491         skb->head_frag = 1;
492 
493 skb_success:
494         skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
495         skb->dev = napi->dev;
496 
497 skb_fail:
498         return skb;
499 }
500 EXPORT_SYMBOL(__napi_alloc_skb);
501 
502 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
503                      int size, unsigned int truesize)
504 {
505         skb_fill_page_desc(skb, i, page, off, size);
506         skb->len += size;
507         skb->data_len += size;
508         skb->truesize += truesize;
509 }
510 EXPORT_SYMBOL(skb_add_rx_frag);
511 
512 void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
513                           unsigned int truesize)
514 {
515         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
516 
517         skb_frag_size_add(frag, size);
518         skb->len += size;
519         skb->data_len += size;
520         skb->truesize += truesize;
521 }
522 EXPORT_SYMBOL(skb_coalesce_rx_frag);
523 
524 static void skb_drop_list(struct sk_buff **listp)
525 {
526         kfree_skb_list(*listp);
527         *listp = NULL;
528 }
529 
530 static inline void skb_drop_fraglist(struct sk_buff *skb)
531 {
532         skb_drop_list(&skb_shinfo(skb)->frag_list);
533 }
534 
535 static void skb_clone_fraglist(struct sk_buff *skb)
536 {
537         struct sk_buff *list;
538 
539         skb_walk_frags(skb, list)
540                 skb_get(list);
541 }
542 
543 static void skb_free_head(struct sk_buff *skb)
544 {
545         unsigned char *head = skb->head;
546 
547         if (skb->head_frag)
548                 skb_free_frag(head);
549         else
550                 kfree(head);
551 }
552 
553 static void skb_release_data(struct sk_buff *skb)
554 {
555         struct skb_shared_info *shinfo = skb_shinfo(skb);
556         int i;
557 
558         if (skb->cloned &&
559             atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
560                               &shinfo->dataref))
561                 return;
562 
563         for (i = 0; i < shinfo->nr_frags; i++)
564                 __skb_frag_unref(&shinfo->frags[i]);
565 
566         if (shinfo->frag_list)
567                 kfree_skb_list(shinfo->frag_list);
568 
569         skb_zcopy_clear(skb, true);
570         skb_free_head(skb);
571 }
572 
573 /*
574  *      Free an skbuff by memory without cleaning the state.
575  */
576 static void kfree_skbmem(struct sk_buff *skb)
577 {
578         struct sk_buff_fclones *fclones;
579 
580         switch (skb->fclone) {
581         case SKB_FCLONE_UNAVAILABLE:
582                 kmem_cache_free(skbuff_head_cache, skb);
583                 return;
584 
585         case SKB_FCLONE_ORIG:
586                 fclones = container_of(skb, struct sk_buff_fclones, skb1);
587 
588                 /* We usually free the clone (TX completion) before original skb
589                  * This test would have no chance to be true for the clone,
590                  * while here, branch prediction will be good.
591                  */
592                 if (refcount_read(&fclones->fclone_ref) == 1)
593                         goto fastpath;
594                 break;
595 
596         default: /* SKB_FCLONE_CLONE */
597                 fclones = container_of(skb, struct sk_buff_fclones, skb2);
598                 break;
599         }
600         if (!refcount_dec_and_test(&fclones->fclone_ref))
601                 return;
602 fastpath:
603         kmem_cache_free(skbuff_fclone_cache, fclones);
604 }
605 
606 void skb_release_head_state(struct sk_buff *skb)
607 {
608         skb_dst_drop(skb);
609         secpath_reset(skb);
610         if (skb->destructor) {
611                 WARN_ON(in_irq());
612                 skb->destructor(skb);
613         }
614 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
615         nf_conntrack_put(skb_nfct(skb));
616 #endif
617 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
618         nf_bridge_put(skb->nf_bridge);
619 #endif
620 }
621 
622 /* Free everything but the sk_buff shell. */
623 static void skb_release_all(struct sk_buff *skb)
624 {
625         skb_release_head_state(skb);
626         if (likely(skb->head))
627                 skb_release_data(skb);
628 }
629 
630 /**
631  *      __kfree_skb - private function
632  *      @skb: buffer
633  *
634  *      Free an sk_buff. Release anything attached to the buffer.
635  *      Clean the state. This is an internal helper function. Users should
636  *      always call kfree_skb
637  */
638 
639 void __kfree_skb(struct sk_buff *skb)
640 {
641         skb_release_all(skb);
642         kfree_skbmem(skb);
643 }
644 EXPORT_SYMBOL(__kfree_skb);
645 
646 /**
647  *      kfree_skb - free an sk_buff
648  *      @skb: buffer to free
649  *
650  *      Drop a reference to the buffer and free it if the usage count has
651  *      hit zero.
652  */
653 void kfree_skb(struct sk_buff *skb)
654 {
655         if (!skb_unref(skb))
656                 return;
657 
658         trace_kfree_skb(skb, __builtin_return_address(0));
659         __kfree_skb(skb);
660 }
661 EXPORT_SYMBOL(kfree_skb);
662 
663 void kfree_skb_list(struct sk_buff *segs)
664 {
665         while (segs) {
666                 struct sk_buff *next = segs->next;
667 
668                 kfree_skb(segs);
669                 segs = next;
670         }
671 }
672 EXPORT_SYMBOL(kfree_skb_list);
673 
674 /**
675  *      skb_tx_error - report an sk_buff xmit error
676  *      @skb: buffer that triggered an error
677  *
678  *      Report xmit error if a device callback is tracking this skb.
679  *      skb must be freed afterwards.
680  */
681 void skb_tx_error(struct sk_buff *skb)
682 {
683         skb_zcopy_clear(skb, true);
684 }
685 EXPORT_SYMBOL(skb_tx_error);
686 
687 /**
688  *      consume_skb - free an skbuff
689  *      @skb: buffer to free
690  *
691  *      Drop a ref to the buffer and free it if the usage count has hit zero
692  *      Functions identically to kfree_skb, but kfree_skb assumes that the frame
693  *      is being dropped after a failure and notes that
694  */
695 void consume_skb(struct sk_buff *skb)
696 {
697         if (!skb_unref(skb))
698                 return;
699 
700         trace_consume_skb(skb);
701         __kfree_skb(skb);
702 }
703 EXPORT_SYMBOL(consume_skb);
704 
705 /**
706  *      consume_stateless_skb - free an skbuff, assuming it is stateless
707  *      @skb: buffer to free
708  *
709  *      Alike consume_skb(), but this variant assumes that this is the last
710  *      skb reference and all the head states have been already dropped
711  */
712 void __consume_stateless_skb(struct sk_buff *skb)
713 {
714         trace_consume_skb(skb);
715         skb_release_data(skb);
716         kfree_skbmem(skb);
717 }
718 
719 void __kfree_skb_flush(void)
720 {
721         struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
722 
723         /* flush skb_cache if containing objects */
724         if (nc->skb_count) {
725                 kmem_cache_free_bulk(skbuff_head_cache, nc->skb_count,
726                                      nc->skb_cache);
727                 nc->skb_count = 0;
728         }
729 }
730 
731 static inline void _kfree_skb_defer(struct sk_buff *skb)
732 {
733         struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
734 
735         /* drop skb->head and call any destructors for packet */
736         skb_release_all(skb);
737 
738         /* record skb to CPU local list */
739         nc->skb_cache[nc->skb_count++] = skb;
740 
741 #ifdef CONFIG_SLUB
742         /* SLUB writes into objects when freeing */
743         prefetchw(skb);
744 #endif
745 
746         /* flush skb_cache if it is filled */
747         if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {
748                 kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_SIZE,
749                                      nc->skb_cache);
750                 nc->skb_count = 0;
751         }
752 }
753 void __kfree_skb_defer(struct sk_buff *skb)
754 {
755         _kfree_skb_defer(skb);
756 }
757 
758 void napi_consume_skb(struct sk_buff *skb, int budget)
759 {
760         if (unlikely(!skb))
761                 return;
762 
763         /* Zero budget indicate non-NAPI context called us, like netpoll */
764         if (unlikely(!budget)) {
765                 dev_consume_skb_any(skb);
766                 return;
767         }
768 
769         if (!skb_unref(skb))
770                 return;
771 
772         /* if reaching here SKB is ready to free */
773         trace_consume_skb(skb);
774 
775         /* if SKB is a clone, don't handle this case */
776         if (skb->fclone != SKB_FCLONE_UNAVAILABLE) {
777                 __kfree_skb(skb);
778                 return;
779         }
780 
781         _kfree_skb_defer(skb);
782 }
783 EXPORT_SYMBOL(napi_consume_skb);
784 
785 /* Make sure a field is enclosed inside headers_start/headers_end section */
786 #define CHECK_SKB_FIELD(field) \
787         BUILD_BUG_ON(offsetof(struct sk_buff, field) <          \
788                      offsetof(struct sk_buff, headers_start));  \
789         BUILD_BUG_ON(offsetof(struct sk_buff, field) >          \
790                      offsetof(struct sk_buff, headers_end));    \
791 
792 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
793 {
794         new->tstamp             = old->tstamp;
795         /* We do not copy old->sk */
796         new->dev                = old->dev;
797         memcpy(new->cb, old->cb, sizeof(old->cb));
798         skb_dst_copy(new, old);
799 #ifdef CONFIG_XFRM
800         new->sp                 = secpath_get(old->sp);
801 #endif
802         __nf_copy(new, old, false);
803 
804         /* Note : this field could be in headers_start/headers_end section
805          * It is not yet because we do not want to have a 16 bit hole
806          */
807         new->queue_mapping = old->queue_mapping;
808 
809         memcpy(&new->headers_start, &old->headers_start,
810                offsetof(struct sk_buff, headers_end) -
811                offsetof(struct sk_buff, headers_start));
812         CHECK_SKB_FIELD(protocol);
813         CHECK_SKB_FIELD(csum);
814         CHECK_SKB_FIELD(hash);
815         CHECK_SKB_FIELD(priority);
816         CHECK_SKB_FIELD(skb_iif);
817         CHECK_SKB_FIELD(vlan_proto);
818         CHECK_SKB_FIELD(vlan_tci);
819         CHECK_SKB_FIELD(transport_header);
820         CHECK_SKB_FIELD(network_header);
821         CHECK_SKB_FIELD(mac_header);
822         CHECK_SKB_FIELD(inner_protocol);
823         CHECK_SKB_FIELD(inner_transport_header);
824         CHECK_SKB_FIELD(inner_network_header);
825         CHECK_SKB_FIELD(inner_mac_header);
826         CHECK_SKB_FIELD(mark);
827 #ifdef CONFIG_NETWORK_SECMARK
828         CHECK_SKB_FIELD(secmark);
829 #endif
830 #ifdef CONFIG_NET_RX_BUSY_POLL
831         CHECK_SKB_FIELD(napi_id);
832 #endif
833 #ifdef CONFIG_XPS
834         CHECK_SKB_FIELD(sender_cpu);
835 #endif
836 #ifdef CONFIG_NET_SCHED
837         CHECK_SKB_FIELD(tc_index);
838 #endif
839 
840 }
841 
842 /*
843  * You should not add any new code to this function.  Add it to
844  * __copy_skb_header above instead.
845  */
846 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
847 {
848 #define C(x) n->x = skb->x
849 
850         n->next = n->prev = NULL;
851         n->sk = NULL;
852         __copy_skb_header(n, skb);
853 
854         C(len);
855         C(data_len);
856         C(mac_len);
857         n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
858         n->cloned = 1;
859         n->nohdr = 0;
860         n->peeked = 0;
861         n->destructor = NULL;
862         C(tail);
863         C(end);
864         C(head);
865         C(head_frag);
866         C(data);
867         C(truesize);
868         refcount_set(&n->users, 1);
869 
870         atomic_inc(&(skb_shinfo(skb)->dataref));
871         skb->cloned = 1;
872 
873         return n;
874 #undef C
875 }
876 
877 /**
878  *      skb_morph       -       morph one skb into another
879  *      @dst: the skb to receive the contents
880  *      @src: the skb to supply the contents
881  *
882  *      This is identical to skb_clone except that the target skb is
883  *      supplied by the user.
884  *
885  *      The target skb is returned upon exit.
886  */
887 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
888 {
889         skb_release_all(dst);
890         return __skb_clone(dst, src);
891 }
892 EXPORT_SYMBOL_GPL(skb_morph);
893 
894 static int mm_account_pinned_pages(struct mmpin *mmp, size_t size)
895 {
896         unsigned long max_pg, num_pg, new_pg, old_pg;
897         struct user_struct *user;
898 
899         if (capable(CAP_IPC_LOCK) || !size)
900                 return 0;
901 
902         num_pg = (size >> PAGE_SHIFT) + 2;      /* worst case */
903         max_pg = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
904         user = mmp->user ? : current_user();
905 
906         do {
907                 old_pg = atomic_long_read(&user->locked_vm);
908                 new_pg = old_pg + num_pg;
909                 if (new_pg > max_pg)
910                         return -ENOBUFS;
911         } while (atomic_long_cmpxchg(&user->locked_vm, old_pg, new_pg) !=
912                  old_pg);
913 
914         if (!mmp->user) {
915                 mmp->user = get_uid(user);
916                 mmp->num_pg = num_pg;
917         } else {
918                 mmp->num_pg += num_pg;
919         }
920 
921         return 0;
922 }
923 
924 static void mm_unaccount_pinned_pages(struct mmpin *mmp)
925 {
926         if (mmp->user) {
927                 atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm);
928                 free_uid(mmp->user);
929         }
930 }
931 
932 struct ubuf_info *sock_zerocopy_alloc(struct sock *sk, size_t size)
933 {
934         struct ubuf_info *uarg;
935         struct sk_buff *skb;
936 
937         WARN_ON_ONCE(!in_task());
938 
939         if (!sock_flag(sk, SOCK_ZEROCOPY))
940                 return NULL;
941 
942         skb = sock_omalloc(sk, 0, GFP_KERNEL);
943         if (!skb)
944                 return NULL;
945 
946         BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb));
947         uarg = (void *)skb->cb;
948         uarg->mmp.user = NULL;
949 
950         if (mm_account_pinned_pages(&uarg->mmp, size)) {
951                 kfree_skb(skb);
952                 return NULL;
953         }
954 
955         uarg->callback = sock_zerocopy_callback;
956         uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
957         uarg->len = 1;
958         uarg->bytelen = size;
959         uarg->zerocopy = 1;
960         refcount_set(&uarg->refcnt, 1);
961         sock_hold(sk);
962 
963         return uarg;
964 }
965 EXPORT_SYMBOL_GPL(sock_zerocopy_alloc);
966 
967 static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg)
968 {
969         return container_of((void *)uarg, struct sk_buff, cb);
970 }
971 
972 struct ubuf_info *sock_zerocopy_realloc(struct sock *sk, size_t size,
973                                         struct ubuf_info *uarg)
974 {
975         if (uarg) {
976                 const u32 byte_limit = 1 << 19;         /* limit to a few TSO */
977                 u32 bytelen, next;
978 
979                 /* realloc only when socket is locked (TCP, UDP cork),
980                  * so uarg->len and sk_zckey access is serialized
981                  */
982                 if (!sock_owned_by_user(sk)) {
983                         WARN_ON_ONCE(1);
984                         return NULL;
985                 }
986 
987                 bytelen = uarg->bytelen + size;
988                 if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) {
989                         /* TCP can create new skb to attach new uarg */
990                         if (sk->sk_type == SOCK_STREAM)
991                                 goto new_alloc;
992                         return NULL;
993                 }
994 
995                 next = (u32)atomic_read(&sk->sk_zckey);
996                 if ((u32)(uarg->id + uarg->len) == next) {
997                         if (mm_account_pinned_pages(&uarg->mmp, size))
998                                 return NULL;
999                         uarg->len++;
1000                         uarg->bytelen = bytelen;
1001                         atomic_set(&sk->sk_zckey, ++next);
1002                         sock_zerocopy_get(uarg);
1003                         return uarg;
1004                 }
1005         }
1006 
1007 new_alloc:
1008         return sock_zerocopy_alloc(sk, size);
1009 }
1010 EXPORT_SYMBOL_GPL(sock_zerocopy_realloc);
1011 
1012 static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len)
1013 {
1014         struct sock_exterr_skb *serr = SKB_EXT_ERR(skb);
1015         u32 old_lo, old_hi;
1016         u64 sum_len;
1017 
1018         old_lo = serr->ee.ee_info;
1019         old_hi = serr->ee.ee_data;
1020         sum_len = old_hi - old_lo + 1ULL + len;
1021 
1022         if (sum_len >= (1ULL << 32))
1023                 return false;
1024 
1025         if (lo != old_hi + 1)
1026                 return false;
1027 
1028         serr->ee.ee_data += len;
1029         return true;
1030 }
1031 
1032 void sock_zerocopy_callback(struct ubuf_info *uarg, bool success)
1033 {
1034         struct sk_buff *tail, *skb = skb_from_uarg(uarg);
1035         struct sock_exterr_skb *serr;
1036         struct sock *sk = skb->sk;
1037         struct sk_buff_head *q;
1038         unsigned long flags;
1039         u32 lo, hi;
1040         u16 len;
1041 
1042         mm_unaccount_pinned_pages(&uarg->mmp);
1043 
1044         /* if !len, there was only 1 call, and it was aborted
1045          * so do not queue a completion notification
1046          */
1047         if (!uarg->len || sock_flag(sk, SOCK_DEAD))
1048                 goto release;
1049 
1050         len = uarg->len;
1051         lo = uarg->id;
1052         hi = uarg->id + len - 1;
1053 
1054         serr = SKB_EXT_ERR(skb);
1055         memset(serr, 0, sizeof(*serr));
1056         serr->ee.ee_errno = 0;
1057         serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY;
1058         serr->ee.ee_data = hi;
1059         serr->ee.ee_info = lo;
1060         if (!success)
1061                 serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED;
1062 
1063         q = &sk->sk_error_queue;
1064         spin_lock_irqsave(&q->lock, flags);
1065         tail = skb_peek_tail(q);
1066         if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY ||
1067             !skb_zerocopy_notify_extend(tail, lo, len)) {
1068                 __skb_queue_tail(q, skb);
1069                 skb = NULL;
1070         }
1071         spin_unlock_irqrestore(&q->lock, flags);
1072 
1073         sk->sk_error_report(sk);
1074 
1075 release:
1076         consume_skb(skb);
1077         sock_put(sk);
1078 }
1079 EXPORT_SYMBOL_GPL(sock_zerocopy_callback);
1080 
1081 void sock_zerocopy_put(struct ubuf_info *uarg)
1082 {
1083         if (uarg && refcount_dec_and_test(&uarg->refcnt)) {
1084                 if (uarg->callback)
1085                         uarg->callback(uarg, uarg->zerocopy);
1086                 else
1087                         consume_skb(skb_from_uarg(uarg));
1088         }
1089 }
1090 EXPORT_SYMBOL_GPL(sock_zerocopy_put);
1091 
1092 void sock_zerocopy_put_abort(struct ubuf_info *uarg)
1093 {
1094         if (uarg) {
1095                 struct sock *sk = skb_from_uarg(uarg)->sk;
1096 
1097                 atomic_dec(&sk->sk_zckey);
1098                 uarg->len--;
1099 
1100                 sock_zerocopy_put(uarg);
1101         }
1102 }
1103 EXPORT_SYMBOL_GPL(sock_zerocopy_put_abort);
1104 
1105 extern int __zerocopy_sg_from_iter(struct sock *sk, struct sk_buff *skb,
1106                                    struct iov_iter *from, size_t length);
1107 
1108 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
1109                              struct msghdr *msg, int len,
1110                              struct ubuf_info *uarg)
1111 {
1112         struct ubuf_info *orig_uarg = skb_zcopy(skb);
1113         struct iov_iter orig_iter = msg->msg_iter;
1114         int err, orig_len = skb->len;
1115 
1116         /* An skb can only point to one uarg. This edge case happens when
1117          * TCP appends to an skb, but zerocopy_realloc triggered a new alloc.
1118          */
1119         if (orig_uarg && uarg != orig_uarg)
1120                 return -EEXIST;
1121 
1122         err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len);
1123         if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
1124                 struct sock *save_sk = skb->sk;
1125 
1126                 /* Streams do not free skb on error. Reset to prev state. */
1127                 msg->msg_iter = orig_iter;
1128                 skb->sk = sk;
1129                 ___pskb_trim(skb, orig_len);
1130                 skb->sk = save_sk;
1131                 return err;
1132         }
1133 
1134         skb_zcopy_set(skb, uarg);
1135         return skb->len - orig_len;
1136 }
1137 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);
1138 
1139 static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig,
1140                               gfp_t gfp_mask)
1141 {
1142         if (skb_zcopy(orig)) {
1143                 if (skb_zcopy(nskb)) {
1144                         /* !gfp_mask callers are verified to !skb_zcopy(nskb) */
1145                         if (!gfp_mask) {
1146                                 WARN_ON_ONCE(1);
1147                                 return -ENOMEM;
1148                         }
1149                         if (skb_uarg(nskb) == skb_uarg(orig))
1150                                 return 0;
1151                         if (skb_copy_ubufs(nskb, GFP_ATOMIC))
1152                                 return -EIO;
1153                 }
1154                 skb_zcopy_set(nskb, skb_uarg(orig));
1155         }
1156         return 0;
1157 }
1158 
1159 /**
1160  *      skb_copy_ubufs  -       copy userspace skb frags buffers to kernel
1161  *      @skb: the skb to modify
1162  *      @gfp_mask: allocation priority
1163  *
1164  *      This must be called on SKBTX_DEV_ZEROCOPY skb.
1165  *      It will copy all frags into kernel and drop the reference
1166  *      to userspace pages.
1167  *
1168  *      If this function is called from an interrupt gfp_mask() must be
1169  *      %GFP_ATOMIC.
1170  *
1171  *      Returns 0 on success or a negative error code on failure
1172  *      to allocate kernel memory to copy to.
1173  */
1174 int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
1175 {
1176         int num_frags = skb_shinfo(skb)->nr_frags;
1177         struct page *page, *head = NULL;
1178         int i, new_frags;
1179         u32 d_off;
1180 
1181         if (skb_shared(skb) || skb_unclone(skb, gfp_mask))
1182                 return -EINVAL;
1183 
1184         if (!num_frags)
1185                 goto release;
1186 
1187         new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT;
1188         for (i = 0; i < new_frags; i++) {
1189                 page = alloc_page(gfp_mask);
1190                 if (!page) {
1191                         while (head) {
1192                                 struct page *next = (struct page *)page_private(head);
1193                                 put_page(head);
1194                                 head = next;
1195                         }
1196                         return -ENOMEM;
1197                 }
1198                 set_page_private(page, (unsigned long)head);
1199                 head = page;
1200         }
1201 
1202         page = head;
1203         d_off = 0;
1204         for (i = 0; i < num_frags; i++) {
1205                 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
1206                 u32 p_off, p_len, copied;
1207                 struct page *p;
1208                 u8 *vaddr;
1209 
1210                 skb_frag_foreach_page(f, f->page_offset, skb_frag_size(f),
1211                                       p, p_off, p_len, copied) {
1212                         u32 copy, done = 0;
1213                         vaddr = kmap_atomic(p);
1214 
1215                         while (done < p_len) {
1216                                 if (d_off == PAGE_SIZE) {
1217                                         d_off = 0;
1218                                         page = (struct page *)page_private(page);
1219                                 }
1220                                 copy = min_t(u32, PAGE_SIZE - d_off, p_len - done);
1221                                 memcpy(page_address(page) + d_off,
1222                                        vaddr + p_off + done, copy);
1223                                 done += copy;
1224                                 d_off += copy;
1225                         }
1226                         kunmap_atomic(vaddr);
1227                 }
1228         }
1229 
1230         /* skb frags release userspace buffers */
1231         for (i = 0; i < num_frags; i++)
1232                 skb_frag_unref(skb, i);
1233 
1234         /* skb frags point to kernel buffers */
1235         for (i = 0; i < new_frags - 1; i++) {
1236                 __skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE);
1237                 head = (struct page *)page_private(head);
1238         }
1239         __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off);
1240         skb_shinfo(skb)->nr_frags = new_frags;
1241 
1242 release:
1243         skb_zcopy_clear(skb, false);
1244         return 0;
1245 }
1246 EXPORT_SYMBOL_GPL(skb_copy_ubufs);
1247 
1248 /**
1249  *      skb_clone       -       duplicate an sk_buff
1250  *      @skb: buffer to clone
1251  *      @gfp_mask: allocation priority
1252  *
1253  *      Duplicate an &sk_buff. The new one is not owned by a socket. Both
1254  *      copies share the same packet data but not structure. The new
1255  *      buffer has a reference count of 1. If the allocation fails the
1256  *      function returns %NULL otherwise the new buffer is returned.
1257  *
1258  *      If this function is called from an interrupt gfp_mask() must be
1259  *      %GFP_ATOMIC.
1260  */
1261 
1262 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
1263 {
1264         struct sk_buff_fclones *fclones = container_of(skb,
1265                                                        struct sk_buff_fclones,
1266                                                        skb1);
1267         struct sk_buff *n;
1268 
1269         if (skb_orphan_frags(skb, gfp_mask))
1270                 return NULL;
1271 
1272         if (skb->fclone == SKB_FCLONE_ORIG &&
1273             refcount_read(&fclones->fclone_ref) == 1) {
1274                 n = &fclones->skb2;
1275                 refcount_set(&fclones->fclone_ref, 2);
1276         } else {
1277                 if (skb_pfmemalloc(skb))
1278                         gfp_mask |= __GFP_MEMALLOC;
1279 
1280                 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
1281                 if (!n)
1282                         return NULL;
1283 
1284                 n->fclone = SKB_FCLONE_UNAVAILABLE;
1285         }
1286 
1287         return __skb_clone(n, skb);
1288 }
1289 EXPORT_SYMBOL(skb_clone);
1290 
1291 static void skb_headers_offset_update(struct sk_buff *skb, int off)
1292 {
1293         /* Only adjust this if it actually is csum_start rather than csum */
1294         if (skb->ip_summed == CHECKSUM_PARTIAL)
1295                 skb->csum_start += off;
1296         /* {transport,network,mac}_header and tail are relative to skb->head */
1297         skb->transport_header += off;
1298         skb->network_header   += off;
1299         if (skb_mac_header_was_set(skb))
1300                 skb->mac_header += off;
1301         skb->inner_transport_header += off;
1302         skb->inner_network_header += off;
1303         skb->inner_mac_header += off;
1304 }
1305 
1306 static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
1307 {
1308         __copy_skb_header(new, old);
1309 
1310         skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
1311         skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
1312         skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
1313 }
1314 
1315 static inline int skb_alloc_rx_flag(const struct sk_buff *skb)
1316 {
1317         if (skb_pfmemalloc(skb))
1318                 return SKB_ALLOC_RX;
1319         return 0;
1320 }
1321 
1322 /**
1323  *      skb_copy        -       create private copy of an sk_buff
1324  *      @skb: buffer to copy
1325  *      @gfp_mask: allocation priority
1326  *
1327  *      Make a copy of both an &sk_buff and its data. This is used when the
1328  *      caller wishes to modify the data and needs a private copy of the
1329  *      data to alter. Returns %NULL on failure or the pointer to the buffer
1330  *      on success. The returned buffer has a reference count of 1.
1331  *
1332  *      As by-product this function converts non-linear &sk_buff to linear
1333  *      one, so that &sk_buff becomes completely private and caller is allowed
1334  *      to modify all the data of returned buffer. This means that this
1335  *      function is not recommended for use in circumstances when only
1336  *      header is going to be modified. Use pskb_copy() instead.
1337  */
1338 
1339 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
1340 {
1341         int headerlen = skb_headroom(skb);
1342         unsigned int size = skb_end_offset(skb) + skb->data_len;
1343         struct sk_buff *n = __alloc_skb(size, gfp_mask,
1344                                         skb_alloc_rx_flag(skb), NUMA_NO_NODE);
1345 
1346         if (!n)
1347                 return NULL;
1348 
1349         /* Set the data pointer */
1350         skb_reserve(n, headerlen);
1351         /* Set the tail pointer and length */
1352         skb_put(n, skb->len);
1353 
1354         BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len));
1355 
1356         copy_skb_header(n, skb);
1357         return n;
1358 }
1359 EXPORT_SYMBOL(skb_copy);
1360 
1361 /**
1362  *      __pskb_copy_fclone      -  create copy of an sk_buff with private head.
1363  *      @skb: buffer to copy
1364  *      @headroom: headroom of new skb
1365  *      @gfp_mask: allocation priority
1366  *      @fclone: if true allocate the copy of the skb from the fclone
1367  *      cache instead of the head cache; it is recommended to set this
1368  *      to true for the cases where the copy will likely be cloned
1369  *
1370  *      Make a copy of both an &sk_buff and part of its data, located
1371  *      in header. Fragmented data remain shared. This is used when
1372  *      the caller wishes to modify only header of &sk_buff and needs
1373  *      private copy of the header to alter. Returns %NULL on failure
1374  *      or the pointer to the buffer on success.
1375  *      The returned buffer has a reference count of 1.
1376  */
1377 
1378 struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
1379                                    gfp_t gfp_mask, bool fclone)
1380 {
1381         unsigned int size = skb_headlen(skb) + headroom;
1382         int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
1383         struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE);
1384 
1385         if (!n)
1386                 goto out;
1387 
1388         /* Set the data pointer */
1389         skb_reserve(n, headroom);
1390         /* Set the tail pointer and length */
1391         skb_put(n, skb_headlen(skb));
1392         /* Copy the bytes */
1393         skb_copy_from_linear_data(skb, n->data, n->len);
1394 
1395         n->truesize += skb->data_len;
1396         n->data_len  = skb->data_len;
1397         n->len       = skb->len;
1398 
1399         if (skb_shinfo(skb)->nr_frags) {
1400                 int i;
1401 
1402                 if (skb_orphan_frags(skb, gfp_mask) ||
1403                     skb_zerocopy_clone(n, skb, gfp_mask)) {
1404                         kfree_skb(n);
1405                         n = NULL;
1406                         goto out;
1407                 }
1408                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1409                         skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
1410                         skb_frag_ref(skb, i);
1411                 }
1412                 skb_shinfo(n)->nr_frags = i;
1413         }
1414 
1415         if (skb_has_frag_list(skb)) {
1416                 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
1417                 skb_clone_fraglist(n);
1418         }
1419 
1420         copy_skb_header(n, skb);
1421 out:
1422         return n;
1423 }
1424 EXPORT_SYMBOL(__pskb_copy_fclone);
1425 
1426 /**
1427  *      pskb_expand_head - reallocate header of &sk_buff
1428  *      @skb: buffer to reallocate
1429  *      @nhead: room to add at head
1430  *      @ntail: room to add at tail
1431  *      @gfp_mask: allocation priority
1432  *
1433  *      Expands (or creates identical copy, if @nhead and @ntail are zero)
1434  *      header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
1435  *      reference count of 1. Returns zero in the case of success or error,
1436  *      if expansion failed. In the last case, &sk_buff is not changed.
1437  *
1438  *      All the pointers pointing into skb header may change and must be
1439  *      reloaded after call to this function.
1440  */
1441 
1442 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
1443                      gfp_t gfp_mask)
1444 {
1445         int i, osize = skb_end_offset(skb);
1446         int size = osize + nhead + ntail;
1447         long off;
1448         u8 *data;
1449 
1450         BUG_ON(nhead < 0);
1451 
1452         BUG_ON(skb_shared(skb));
1453 
1454         size = SKB_DATA_ALIGN(size);
1455 
1456         if (skb_pfmemalloc(skb))
1457                 gfp_mask |= __GFP_MEMALLOC;
1458         data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
1459                                gfp_mask, NUMA_NO_NODE, NULL);
1460         if (!data)
1461                 goto nodata;
1462         size = SKB_WITH_OVERHEAD(ksize(data));
1463 
1464         /* Copy only real data... and, alas, header. This should be
1465          * optimized for the cases when header is void.
1466          */
1467         memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
1468 
1469         memcpy((struct skb_shared_info *)(data + size),
1470                skb_shinfo(skb),
1471                offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
1472 
1473         /*
1474          * if shinfo is shared we must drop the old head gracefully, but if it
1475          * is not we can just drop the old head and let the existing refcount
1476          * be since all we did is relocate the values
1477          */
1478         if (skb_cloned(skb)) {
1479                 if (skb_orphan_frags(skb, gfp_mask))
1480                         goto nofrags;
1481                 if (skb_zcopy(skb))
1482                         refcount_inc(&skb_uarg(skb)->refcnt);
1483                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1484                         skb_frag_ref(skb, i);
1485 
1486                 if (skb_has_frag_list(skb))
1487                         skb_clone_fraglist(skb);
1488 
1489                 skb_release_data(skb);
1490         } else {
1491                 skb_free_head(skb);
1492         }
1493         off = (data + nhead) - skb->head;
1494 
1495         skb->head     = data;
1496         skb->head_frag = 0;
1497         skb->data    += off;
1498 #ifdef NET_SKBUFF_DATA_USES_OFFSET
1499         skb->end      = size;
1500         off           = nhead;
1501 #else
1502         skb->end      = skb->head + size;
1503 #endif
1504         skb->tail             += off;
1505         skb_headers_offset_update(skb, nhead);
1506         skb->cloned   = 0;
1507         skb->hdr_len  = 0;
1508         skb->nohdr    = 0;
1509         atomic_set(&skb_shinfo(skb)->dataref, 1);
1510 
1511         skb_metadata_clear(skb);
1512 
1513         /* It is not generally safe to change skb->truesize.
1514          * For the moment, we really care of rx path, or
1515          * when skb is orphaned (not attached to a socket).
1516          */
1517         if (!skb->sk || skb->destructor == sock_edemux)
1518                 skb->truesize += size - osize;
1519 
1520         return 0;
1521 
1522 nofrags:
1523         kfree(data);
1524 nodata:
1525         return -ENOMEM;
1526 }
1527 EXPORT_SYMBOL(pskb_expand_head);
1528 
1529 /* Make private copy of skb with writable head and some headroom */
1530 
1531 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
1532 {
1533         struct sk_buff *skb2;
1534         int delta = headroom - skb_headroom(skb);
1535 
1536         if (delta <= 0)
1537                 skb2 = pskb_copy(skb, GFP_ATOMIC);
1538         else {
1539                 skb2 = skb_clone(skb, GFP_ATOMIC);
1540                 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
1541                                              GFP_ATOMIC)) {
1542                         kfree_skb(skb2);
1543                         skb2 = NULL;
1544                 }
1545         }
1546         return skb2;
1547 }
1548 EXPORT_SYMBOL(skb_realloc_headroom);
1549 
1550 /**
1551  *      skb_copy_expand -       copy and expand sk_buff
1552  *      @skb: buffer to copy
1553  *      @newheadroom: new free bytes at head
1554  *      @newtailroom: new free bytes at tail
1555  *      @gfp_mask: allocation priority
1556  *
1557  *      Make a copy of both an &sk_buff and its data and while doing so
1558  *      allocate additional space.
1559  *
1560  *      This is used when the caller wishes to modify the data and needs a
1561  *      private copy of the data to alter as well as more space for new fields.
1562  *      Returns %NULL on failure or the pointer to the buffer
1563  *      on success. The returned buffer has a reference count of 1.
1564  *
1565  *      You must pass %GFP_ATOMIC as the allocation priority if this function
1566  *      is called from an interrupt.
1567  */
1568 struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
1569                                 int newheadroom, int newtailroom,
1570                                 gfp_t gfp_mask)
1571 {
1572         /*
1573          *      Allocate the copy buffer
1574          */
1575         struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
1576                                         gfp_mask, skb_alloc_rx_flag(skb),
1577                                         NUMA_NO_NODE);
1578         int oldheadroom = skb_headroom(skb);
1579         int head_copy_len, head_copy_off;
1580 
1581         if (!n)
1582                 return NULL;
1583 
1584         skb_reserve(n, newheadroom);
1585 
1586         /* Set the tail pointer and length */
1587         skb_put(n, skb->len);
1588 
1589         head_copy_len = oldheadroom;
1590         head_copy_off = 0;
1591         if (newheadroom <= head_copy_len)
1592                 head_copy_len = newheadroom;
1593         else
1594                 head_copy_off = newheadroom - head_copy_len;
1595 
1596         /* Copy the linear header and data. */
1597         BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
1598                              skb->len + head_copy_len));
1599 
1600         copy_skb_header(n, skb);
1601 
1602         skb_headers_offset_update(n, newheadroom - oldheadroom);
1603 
1604         return n;
1605 }
1606 EXPORT_SYMBOL(skb_copy_expand);
1607 
1608 /**
1609  *      __skb_pad               -       zero pad the tail of an skb
1610  *      @skb: buffer to pad
1611  *      @pad: space to pad
1612  *      @free_on_error: free buffer on error
1613  *
1614  *      Ensure that a buffer is followed by a padding area that is zero
1615  *      filled. Used by network drivers which may DMA or transfer data
1616  *      beyond the buffer end onto the wire.
1617  *
1618  *      May return error in out of memory cases. The skb is freed on error
1619  *      if @free_on_error is true.
1620  */
1621 
1622 int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error)
1623 {
1624         int err;
1625         int ntail;
1626 
1627         /* If the skbuff is non linear tailroom is always zero.. */
1628         if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
1629                 memset(skb->data+skb->len, 0, pad);
1630                 return 0;
1631         }
1632 
1633         ntail = skb->data_len + pad - (skb->end - skb->tail);
1634         if (likely(skb_cloned(skb) || ntail > 0)) {
1635                 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
1636                 if (unlikely(err))
1637                         goto free_skb;
1638         }
1639 
1640         /* FIXME: The use of this function with non-linear skb's really needs
1641          * to be audited.
1642          */
1643         err = skb_linearize(skb);
1644         if (unlikely(err))
1645                 goto free_skb;
1646 
1647         memset(skb->data + skb->len, 0, pad);
1648         return 0;
1649 
1650 free_skb:
1651         if (free_on_error)
1652                 kfree_skb(skb);
1653         return err;
1654 }
1655 EXPORT_SYMBOL(__skb_pad);
1656 
1657 /**
1658  *      pskb_put - add data to the tail of a potentially fragmented buffer
1659  *      @skb: start of the buffer to use
1660  *      @tail: tail fragment of the buffer to use
1661  *      @len: amount of data to add
1662  *
1663  *      This function extends the used data area of the potentially
1664  *      fragmented buffer. @tail must be the last fragment of @skb -- or
1665  *      @skb itself. If this would exceed the total buffer size the kernel
1666  *      will panic. A pointer to the first byte of the extra data is
1667  *      returned.
1668  */
1669 
1670 void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
1671 {
1672         if (tail != skb) {
1673                 skb->data_len += len;
1674                 skb->len += len;
1675         }
1676         return skb_put(tail, len);
1677 }
1678 EXPORT_SYMBOL_GPL(pskb_put);
1679 
1680 /**
1681  *      skb_put - add data to a buffer
1682  *      @skb: buffer to use
1683  *      @len: amount of data to add
1684  *
1685  *      This function extends the used data area of the buffer. If this would
1686  *      exceed the total buffer size the kernel will panic. A pointer to the
1687  *      first byte of the extra data is returned.
1688  */
1689 void *skb_put(struct sk_buff *skb, unsigned int len)
1690 {
1691         void *tmp = skb_tail_pointer(skb);
1692         SKB_LINEAR_ASSERT(skb);
1693         skb->tail += len;
1694         skb->len  += len;
1695         if (unlikely(skb->tail > skb->end))
1696                 skb_over_panic(skb, len, __builtin_return_address(0));
1697         return tmp;
1698 }
1699 EXPORT_SYMBOL(skb_put);
1700 
1701 /**
1702  *      skb_push - add data to the start of a buffer
1703  *      @skb: buffer to use
1704  *      @len: amount of data to add
1705  *
1706  *      This function extends the used data area of the buffer at the buffer
1707  *      start. If this would exceed the total buffer headroom the kernel will
1708  *      panic. A pointer to the first byte of the extra data is returned.
1709  */
1710 void *skb_push(struct sk_buff *skb, unsigned int len)
1711 {
1712         skb->data -= len;
1713         skb->len  += len;
1714         if (unlikely(skb->data<skb->head))
1715                 skb_under_panic(skb, len, __builtin_return_address(0));
1716         return skb->data;
1717 }
1718 EXPORT_SYMBOL(skb_push);
1719 
1720 /**
1721  *      skb_pull - remove data from the start of a buffer
1722  *      @skb: buffer to use
1723  *      @len: amount of data to remove
1724  *
1725  *      This function removes data from the start of a buffer, returning
1726  *      the memory to the headroom. A pointer to the next data in the buffer
1727  *      is returned. Once the data has been pulled future pushes will overwrite
1728  *      the old data.
1729  */
1730 void *skb_pull(struct sk_buff *skb, unsigned int len)
1731 {
1732         return skb_pull_inline(skb, len);
1733 }
1734 EXPORT_SYMBOL(skb_pull);
1735 
1736 /**
1737  *      skb_trim - remove end from a buffer
1738  *      @skb: buffer to alter
1739  *      @len: new length
1740  *
1741  *      Cut the length of a buffer down by removing data from the tail. If
1742  *      the buffer is already under the length specified it is not modified.
1743  *      The skb must be linear.
1744  */
1745 void skb_trim(struct sk_buff *skb, unsigned int len)
1746 {
1747         if (skb->len > len)
1748                 __skb_trim(skb, len);
1749 }
1750 EXPORT_SYMBOL(skb_trim);
1751 
1752 /* Trims skb to length len. It can change skb pointers.
1753  */
1754 
1755 int ___pskb_trim(struct sk_buff *skb, unsigned int len)
1756 {
1757         struct sk_buff **fragp;
1758         struct sk_buff *frag;
1759         int offset = skb_headlen(skb);
1760         int nfrags = skb_shinfo(skb)->nr_frags;
1761         int i;
1762         int err;
1763 
1764         if (skb_cloned(skb) &&
1765             unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
1766                 return err;
1767 
1768         i = 0;
1769         if (offset >= len)
1770                 goto drop_pages;
1771 
1772         for (; i < nfrags; i++) {
1773                 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]);
1774 
1775                 if (end < len) {
1776                         offset = end;
1777                         continue;
1778                 }
1779 
1780                 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset);
1781 
1782 drop_pages:
1783                 skb_shinfo(skb)->nr_frags = i;
1784 
1785                 for (; i < nfrags; i++)
1786                         skb_frag_unref(skb, i);
1787 
1788                 if (skb_has_frag_list(skb))
1789                         skb_drop_fraglist(skb);
1790                 goto done;
1791         }
1792 
1793         for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
1794              fragp = &frag->next) {
1795                 int end = offset + frag->len;
1796 
1797                 if (skb_shared(frag)) {
1798                         struct sk_buff *nfrag;
1799 
1800                         nfrag = skb_clone(frag, GFP_ATOMIC);
1801                         if (unlikely(!nfrag))
1802                                 return -ENOMEM;
1803 
1804                         nfrag->next = frag->next;
1805                         consume_skb(frag);
1806                         frag = nfrag;
1807                         *fragp = frag;
1808                 }
1809 
1810                 if (end < len) {
1811                         offset = end;
1812                         continue;
1813                 }
1814 
1815                 if (end > len &&
1816                     unlikely((err = pskb_trim(frag, len - offset))))
1817                         return err;
1818 
1819                 if (frag->next)
1820                         skb_drop_list(&frag->next);
1821                 break;
1822         }
1823 
1824 done:
1825         if (len > skb_headlen(skb)) {
1826                 skb->data_len -= skb->len - len;
1827                 skb->len       = len;
1828         } else {
1829                 skb->len       = len;
1830                 skb->data_len  = 0;
1831                 skb_set_tail_pointer(skb, len);
1832         }
1833 
1834         if (!skb->sk || skb->destructor == sock_edemux)
1835                 skb_condense(skb);
1836         return 0;
1837 }
1838 EXPORT_SYMBOL(___pskb_trim);
1839 
1840 /**
1841  *      __pskb_pull_tail - advance tail of skb header
1842  *      @skb: buffer to reallocate
1843  *      @delta: number of bytes to advance tail
1844  *
1845  *      The function makes a sense only on a fragmented &sk_buff,
1846  *      it expands header moving its tail forward and copying necessary
1847  *      data from fragmented part.
1848  *
1849  *      &sk_buff MUST have reference count of 1.
1850  *
1851  *      Returns %NULL (and &sk_buff does not change) if pull failed
1852  *      or value of new tail of skb in the case of success.
1853  *
1854  *      All the pointers pointing into skb header may change and must be
1855  *      reloaded after call to this function.
1856  */
1857 
1858 /* Moves tail of skb head forward, copying data from fragmented part,
1859  * when it is necessary.
1860  * 1. It may fail due to malloc failure.
1861  * 2. It may change skb pointers.
1862  *
1863  * It is pretty complicated. Luckily, it is called only in exceptional cases.
1864  */
1865 void *__pskb_pull_tail(struct sk_buff *skb, int delta)
1866 {
1867         /* If skb has not enough free space at tail, get new one
1868          * plus 128 bytes for future expansions. If we have enough
1869          * room at tail, reallocate without expansion only if skb is cloned.
1870          */
1871         int i, k, eat = (skb->tail + delta) - skb->end;
1872 
1873         if (eat > 0 || skb_cloned(skb)) {
1874                 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
1875                                      GFP_ATOMIC))
1876                         return NULL;
1877         }
1878 
1879         BUG_ON(skb_copy_bits(skb, skb_headlen(skb),
1880                              skb_tail_pointer(skb), delta));
1881 
1882         /* Optimization: no fragments, no reasons to preestimate
1883          * size of pulled pages. Superb.
1884          */
1885         if (!skb_has_frag_list(skb))
1886                 goto pull_pages;
1887 
1888         /* Estimate size of pulled pages. */
1889         eat = delta;
1890         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1891                 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1892 
1893                 if (size >= eat)
1894                         goto pull_pages;
1895                 eat -= size;
1896         }
1897 
1898         /* If we need update frag list, we are in troubles.
1899          * Certainly, it is possible to add an offset to skb data,
1900          * but taking into account that pulling is expected to
1901          * be very rare operation, it is worth to fight against
1902          * further bloating skb head and crucify ourselves here instead.
1903          * Pure masohism, indeed. 8)8)
1904          */
1905         if (eat) {
1906                 struct sk_buff *list = skb_shinfo(skb)->frag_list;
1907                 struct sk_buff *clone = NULL;
1908                 struct sk_buff *insp = NULL;
1909 
1910                 do {
1911                         BUG_ON(!list);
1912 
1913                         if (list->len <= eat) {
1914                                 /* Eaten as whole. */
1915                                 eat -= list->len;
1916                                 list = list->next;
1917                                 insp = list;
1918                         } else {
1919                                 /* Eaten partially. */
1920 
1921                                 if (skb_shared(list)) {
1922                                         /* Sucks! We need to fork list. :-( */
1923                                         clone = skb_clone(list, GFP_ATOMIC);
1924                                         if (!clone)
1925                                                 return NULL;
1926                                         insp = list->next;
1927                                         list = clone;
1928                                 } else {
1929                                         /* This may be pulled without
1930                                          * problems. */
1931                                         insp = list;
1932                                 }
1933                                 if (!pskb_pull(list, eat)) {
1934                                         kfree_skb(clone);
1935                                         return NULL;
1936                                 }
1937                                 break;
1938                         }
1939                 } while (eat);
1940 
1941                 /* Free pulled out fragments. */
1942                 while ((list = skb_shinfo(skb)->frag_list) != insp) {
1943                         skb_shinfo(skb)->frag_list = list->next;
1944                         kfree_skb(list);
1945                 }
1946                 /* And insert new clone at head. */
1947                 if (clone) {
1948                         clone->next = list;
1949                         skb_shinfo(skb)->frag_list = clone;
1950                 }
1951         }
1952         /* Success! Now we may commit changes to skb data. */
1953 
1954 pull_pages:
1955         eat = delta;
1956         k = 0;
1957         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1958                 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
1959 
1960                 if (size <= eat) {
1961                         skb_frag_unref(skb, i);
1962                         eat -= size;
1963                 } else {
1964                         skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
1965                         if (eat) {
1966                                 skb_shinfo(skb)->frags[k].page_offset += eat;
1967                                 skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat);
1968                                 if (!i)
1969                                         goto end;
1970                                 eat = 0;
1971                         }
1972                         k++;
1973                 }
1974         }
1975         skb_shinfo(skb)->nr_frags = k;
1976 
1977 end:
1978         skb->tail     += delta;
1979         skb->data_len -= delta;
1980 
1981         if (!skb->data_len)
1982                 skb_zcopy_clear(skb, false);
1983 
1984         return skb_tail_pointer(skb);
1985 }
1986 EXPORT_SYMBOL(__pskb_pull_tail);
1987 
1988 /**
1989  *      skb_copy_bits - copy bits from skb to kernel buffer
1990  *      @skb: source skb
1991  *      @offset: offset in source
1992  *      @to: destination buffer
1993  *      @len: number of bytes to copy
1994  *
1995  *      Copy the specified number of bytes from the source skb to the
1996  *      destination buffer.
1997  *
1998  *      CAUTION ! :
1999  *              If its prototype is ever changed,
2000  *              check arch/{*}/net/{*}.S files,
2001  *              since it is called from BPF assembly code.
2002  */
2003 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
2004 {
2005         int start = skb_headlen(skb);
2006         struct sk_buff *frag_iter;
2007         int i, copy;
2008 
2009         if (offset > (int)skb->len - len)
2010                 goto fault;
2011 
2012         /* Copy header. */
2013         if ((copy = start - offset) > 0) {
2014                 if (copy > len)
2015                         copy = len;
2016                 skb_copy_from_linear_data_offset(skb, offset, to, copy);
2017                 if ((len -= copy) == 0)
2018                         return 0;
2019                 offset += copy;
2020                 to     += copy;
2021         }
2022 
2023         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2024                 int end;
2025                 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
2026 
2027                 WARN_ON(start > offset + len);
2028 
2029                 end = start + skb_frag_size(f);
2030                 if ((copy = end - offset) > 0) {
2031                         u32 p_off, p_len, copied;
2032                         struct page *p;
2033                         u8 *vaddr;
2034 
2035                         if (copy > len)
2036                                 copy = len;
2037 
2038                         skb_frag_foreach_page(f,
2039                                               f->page_offset + offset - start,
2040                                               copy, p, p_off, p_len, copied) {
2041                                 vaddr = kmap_atomic(p);
2042                                 memcpy(to + copied, vaddr + p_off, p_len);
2043                                 kunmap_atomic(vaddr);
2044                         }
2045 
2046                         if ((len -= copy) == 0)
2047                                 return 0;
2048                         offset += copy;
2049                         to     += copy;
2050                 }
2051                 start = end;
2052         }
2053 
2054         skb_walk_frags(skb, frag_iter) {
2055                 int end;
2056 
2057                 WARN_ON(start > offset + len);
2058 
2059                 end = start + frag_iter->len;
2060                 if ((copy = end - offset) > 0) {
2061                         if (copy > len)
2062                                 copy = len;
2063                         if (skb_copy_bits(frag_iter, offset - start, to, copy))
2064                                 goto fault;
2065                         if ((len -= copy) == 0)
2066                                 return 0;
2067                         offset += copy;
2068                         to     += copy;
2069                 }
2070                 start = end;
2071         }
2072 
2073         if (!len)
2074                 return 0;
2075 
2076 fault:
2077         return -EFAULT;
2078 }
2079 EXPORT_SYMBOL(skb_copy_bits);
2080 
2081 /*
2082  * Callback from splice_to_pipe(), if we need to release some pages
2083  * at the end of the spd in case we error'ed out in filling the pipe.
2084  */
2085 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
2086 {
2087         put_page(spd->pages[i]);
2088 }
2089 
2090 static struct page *linear_to_page(struct page *page, unsigned int *len,
2091                                    unsigned int *offset,
2092                                    struct sock *sk)
2093 {
2094         struct page_frag *pfrag = sk_page_frag(sk);
2095 
2096         if (!sk_page_frag_refill(sk, pfrag))
2097                 return NULL;
2098 
2099         *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset);
2100 
2101         memcpy(page_address(pfrag->page) + pfrag->offset,
2102                page_address(page) + *offset, *len);
2103         *offset = pfrag->offset;
2104         pfrag->offset += *len;
2105 
2106         return pfrag->page;
2107 }
2108 
2109 static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
2110                              struct page *page,
2111                              unsigned int offset)
2112 {
2113         return  spd->nr_pages &&
2114                 spd->pages[spd->nr_pages - 1] == page &&
2115                 (spd->partial[spd->nr_pages - 1].offset +
2116                  spd->partial[spd->nr_pages - 1].len == offset);
2117 }
2118 
2119 /*
2120  * Fill page/offset/length into spd, if it can hold more pages.
2121  */
2122 static bool spd_fill_page(struct splice_pipe_desc *spd,
2123                           struct pipe_inode_info *pipe, struct page *page,
2124                           unsigned int *len, unsigned int offset,
2125                           bool linear,
2126                           struct sock *sk)
2127 {
2128         if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
2129                 return true;
2130 
2131         if (linear) {
2132                 page = linear_to_page(page, len, &offset, sk);
2133                 if (!page)
2134                         return true;
2135         }
2136         if (spd_can_coalesce(spd, page, offset)) {
2137                 spd->partial[spd->nr_pages - 1].len += *len;
2138                 return false;
2139         }
2140         get_page(page);
2141         spd->pages[spd->nr_pages] = page;
2142         spd->partial[spd->nr_pages].len = *len;
2143         spd->partial[spd->nr_pages].offset = offset;
2144         spd->nr_pages++;
2145 
2146         return false;
2147 }
2148 
2149 static bool __splice_segment(struct page *page, unsigned int poff,
2150                              unsigned int plen, unsigned int *off,
2151                              unsigned int *len,
2152                              struct splice_pipe_desc *spd, bool linear,
2153                              struct sock *sk,
2154                              struct pipe_inode_info *pipe)
2155 {
2156         if (!*len)
2157                 return true;
2158 
2159         /* skip this segment if already processed */
2160         if (*off >= plen) {
2161                 *off -= plen;
2162                 return false;
2163         }
2164 
2165         /* ignore any bits we already processed */
2166         poff += *off;
2167         plen -= *off;
2168         *off = 0;
2169 
2170         do {
2171                 unsigned int flen = min(*len, plen);
2172 
2173                 if (spd_fill_page(spd, pipe, page, &flen, poff,
2174                                   linear, sk))
2175                         return true;
2176                 poff += flen;
2177                 plen -= flen;
2178                 *len -= flen;
2179         } while (*len && plen);
2180 
2181         return false;
2182 }
2183 
2184 /*
2185  * Map linear and fragment data from the skb to spd. It reports true if the
2186  * pipe is full or if we already spliced the requested length.
2187  */
2188 static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
2189                               unsigned int *offset, unsigned int *len,
2190                               struct splice_pipe_desc *spd, struct sock *sk)
2191 {
2192         int seg;
2193         struct sk_buff *iter;
2194 
2195         /* map the linear part :
2196          * If skb->head_frag is set, this 'linear' part is backed by a
2197          * fragment, and if the head is not shared with any clones then
2198          * we can avoid a copy since we own the head portion of this page.
2199          */
2200         if (__splice_segment(virt_to_page(skb->data),
2201                              (unsigned long) skb->data & (PAGE_SIZE - 1),
2202                              skb_headlen(skb),
2203                              offset, len, spd,
2204                              skb_head_is_locked(skb),
2205                              sk, pipe))
2206                 return true;
2207 
2208         /*
2209          * then map the fragments
2210          */
2211         for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
2212                 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
2213 
2214                 if (__splice_segment(skb_frag_page(f),
2215                                      f->page_offset, skb_frag_size(f),
2216                                      offset, len, spd, false, sk, pipe))
2217                         return true;
2218         }
2219 
2220         skb_walk_frags(skb, iter) {
2221                 if (*offset >= iter->len) {
2222                         *offset -= iter->len;
2223                         continue;
2224                 }
2225                 /* __skb_splice_bits() only fails if the output has no room
2226                  * left, so no point in going over the frag_list for the error
2227                  * case.
2228                  */
2229                 if (__skb_splice_bits(iter, pipe, offset, len, spd, sk))
2230                         return true;
2231         }
2232 
2233         return false;
2234 }
2235 
2236 /*
2237  * Map data from the skb to a pipe. Should handle both the linear part,
2238  * the fragments, and the frag list.
2239  */
2240 int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
2241                     struct pipe_inode_info *pipe, unsigned int tlen,
2242                     unsigned int flags)
2243 {
2244         struct partial_page partial[MAX_SKB_FRAGS];
2245         struct page *pages[MAX_SKB_FRAGS];
2246         struct splice_pipe_desc spd = {
2247                 .pages = pages,
2248                 .partial = partial,
2249                 .nr_pages_max = MAX_SKB_FRAGS,
2250                 .ops = &nosteal_pipe_buf_ops,
2251                 .spd_release = sock_spd_release,
2252         };
2253         int ret = 0;
2254 
2255         __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk);
2256 
2257         if (spd.nr_pages)
2258                 ret = splice_to_pipe(pipe, &spd);
2259 
2260         return ret;
2261 }
2262 EXPORT_SYMBOL_GPL(skb_splice_bits);
2263 
2264 /* Send skb data on a socket. Socket must be locked. */
2265 int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset,
2266                          int len)
2267 {
2268         unsigned int orig_len = len;
2269         struct sk_buff *head = skb;
2270         unsigned short fragidx;
2271         int slen, ret;
2272 
2273 do_frag_list:
2274 
2275         /* Deal with head data */
2276         while (offset < skb_headlen(skb) && len) {
2277                 struct kvec kv;
2278                 struct msghdr msg;
2279 
2280                 slen = min_t(int, len, skb_headlen(skb) - offset);
2281                 kv.iov_base = skb->data + offset;
2282                 kv.iov_len = slen;
2283                 memset(&msg, 0, sizeof(msg));
2284 
2285                 ret = kernel_sendmsg_locked(sk, &msg, &kv, 1, slen);
2286                 if (ret <= 0)
2287                         goto error;
2288 
2289                 offset += ret;
2290                 len -= ret;
2291         }
2292 
2293         /* All the data was skb head? */
2294         if (!len)
2295                 goto out;
2296 
2297         /* Make offset relative to start of frags */
2298         offset -= skb_headlen(skb);
2299 
2300         /* Find where we are in frag list */
2301         for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2302                 skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];
2303 
2304                 if (offset < frag->size)
2305                         break;
2306 
2307                 offset -= frag->size;
2308         }
2309 
2310         for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2311                 skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];
2312 
2313                 slen = min_t(size_t, len, frag->size - offset);
2314 
2315                 while (slen) {
2316                         ret = kernel_sendpage_locked(sk, frag->page.p,
2317                                                      frag->page_offset + offset,
2318                                                      slen, MSG_DONTWAIT);
2319                         if (ret <= 0)
2320                                 goto error;
2321 
2322                         len -= ret;
2323                         offset += ret;
2324                         slen -= ret;
2325                 }
2326 
2327                 offset = 0;
2328         }
2329 
2330         if (len) {
2331                 /* Process any frag lists */
2332 
2333                 if (skb == head) {
2334                         if (skb_has_frag_list(skb)) {
2335                                 skb = skb_shinfo(skb)->frag_list;
2336                                 goto do_frag_list;
2337                         }
2338                 } else if (skb->next) {
2339                         skb = skb->next;
2340                         goto do_frag_list;
2341                 }
2342         }
2343 
2344 out:
2345         return orig_len - len;
2346 
2347 error:
2348         return orig_len == len ? ret : orig_len - len;
2349 }
2350 EXPORT_SYMBOL_GPL(skb_send_sock_locked);
2351 
2352 /* Send skb data on a socket. */
2353 int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len)
2354 {
2355         int ret = 0;
2356 
2357         lock_sock(sk);
2358         ret = skb_send_sock_locked(sk, skb, offset, len);
2359         release_sock(sk);
2360 
2361         return ret;
2362 }
2363 EXPORT_SYMBOL_GPL(skb_send_sock);
2364 
2365 /**
2366  *      skb_store_bits - store bits from kernel buffer to skb
2367  *      @skb: destination buffer
2368  *      @offset: offset in destination
2369  *      @from: source buffer
2370  *      @len: number of bytes to copy
2371  *
2372  *      Copy the specified number of bytes from the source buffer to the
2373  *      destination skb.  This function handles all the messy bits of
2374  *      traversing fragment lists and such.
2375  */
2376 
2377 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
2378 {
2379         int start = skb_headlen(skb);
2380         struct sk_buff *frag_iter;
2381         int i, copy;
2382 
2383         if (offset > (int)skb->len - len)
2384                 goto fault;
2385 
2386         if ((copy = start - offset) > 0) {
2387                 if (copy > len)
2388                         copy = len;
2389                 skb_copy_to_linear_data_offset(skb, offset, from, copy);
2390                 if ((len -= copy) == 0)
2391                         return 0;
2392                 offset += copy;
2393                 from += copy;
2394         }
2395 
2396         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2397                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2398                 int end;
2399 
2400                 WARN_ON(start > offset + len);
2401 
2402                 end = start + skb_frag_size(frag);
2403                 if ((copy = end - offset) > 0) {
2404                         u32 p_off, p_len, copied;
2405                         struct page *p;
2406                         u8 *vaddr;
2407 
2408                         if (copy > len)
2409                                 copy = len;
2410 
2411                         skb_frag_foreach_page(frag,
2412                                               frag->page_offset + offset - start,
2413                                               copy, p, p_off, p_len, copied) {
2414                                 vaddr = kmap_atomic(p);
2415                                 memcpy(vaddr + p_off, from + copied, p_len);
2416                                 kunmap_atomic(vaddr);
2417                         }
2418 
2419                         if ((len -= copy) == 0)
2420                                 return 0;
2421                         offset += copy;
2422                         from += copy;
2423                 }
2424                 start = end;
2425         }
2426 
2427         skb_walk_frags(skb, frag_iter) {
2428                 int end;
2429 
2430                 WARN_ON(start > offset + len);
2431 
2432                 end = start + frag_iter->len;
2433                 if ((copy = end - offset) > 0) {
2434                         if (copy > len)
2435                                 copy = len;
2436                         if (skb_store_bits(frag_iter, offset - start,
2437                                            from, copy))
2438                                 goto fault;
2439                         if ((len -= copy) == 0)
2440                                 return 0;
2441                         offset += copy;
2442                         from += copy;
2443                 }
2444                 start = end;
2445         }
2446         if (!len)
2447                 return 0;
2448 
2449 fault:
2450         return -EFAULT;
2451 }
2452 EXPORT_SYMBOL(skb_store_bits);
2453 
2454 /* Checksum skb data. */
2455 __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
2456                       __wsum csum, const struct skb_checksum_ops *ops)
2457 {
2458         int start = skb_headlen(skb);
2459         int i, copy = start - offset;
2460         struct sk_buff *frag_iter;
2461         int pos = 0;
2462 
2463         /* Checksum header. */
2464         if (copy > 0) {
2465                 if (copy > len)
2466                         copy = len;
2467                 csum = ops->update(skb->data + offset, copy, csum);
2468                 if ((len -= copy) == 0)
2469                         return csum;
2470                 offset += copy;
2471                 pos     = copy;
2472         }
2473 
2474         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2475                 int end;
2476                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2477 
2478                 WARN_ON(start > offset + len);
2479 
2480                 end = start + skb_frag_size(frag);
2481                 if ((copy = end - offset) > 0) {
2482                         u32 p_off, p_len, copied;
2483                         struct page *p;
2484                         __wsum csum2;
2485                         u8 *vaddr;
2486 
2487                         if (copy > len)
2488                                 copy = len;
2489 
2490                         skb_frag_foreach_page(frag,
2491                                               frag->page_offset + offset - start,
2492                                               copy, p, p_off, p_len, copied) {
2493                                 vaddr = kmap_atomic(p);
2494                                 csum2 = ops->update(vaddr + p_off, p_len, 0);
2495                                 kunmap_atomic(vaddr);
2496                                 csum = ops->combine(csum, csum2, pos, p_len);
2497                                 pos += p_len;
2498                         }
2499 
2500                         if (!(len -= copy))
2501                                 return csum;
2502                         offset += copy;
2503                 }
2504                 start = end;
2505         }
2506 
2507         skb_walk_frags(skb, frag_iter) {
2508                 int end;
2509 
2510                 WARN_ON(start > offset + len);
2511 
2512                 end = start + frag_iter->len;
2513                 if ((copy = end - offset) > 0) {
2514                         __wsum csum2;
2515                         if (copy > len)
2516                                 copy = len;
2517                         csum2 = __skb_checksum(frag_iter, offset - start,
2518                                                copy, 0, ops);
2519                         csum = ops->combine(csum, csum2, pos, copy);
2520                         if ((len -= copy) == 0)
2521                                 return csum;
2522                         offset += copy;
2523                         pos    += copy;
2524                 }
2525                 start = end;
2526         }
2527         BUG_ON(len);
2528 
2529         return csum;
2530 }
2531 EXPORT_SYMBOL(__skb_checksum);
2532 
2533 __wsum skb_checksum(const struct sk_buff *skb, int offset,
2534                     int len, __wsum csum)
2535 {
2536         const struct skb_checksum_ops ops = {
2537                 .update  = csum_partial_ext,
2538                 .combine = csum_block_add_ext,
2539         };
2540 
2541         return __skb_checksum(skb, offset, len, csum, &ops);
2542 }
2543 EXPORT_SYMBOL(skb_checksum);
2544 
2545 /* Both of above in one bottle. */
2546 
2547 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
2548                                     u8 *to, int len, __wsum csum)
2549 {
2550         int start = skb_headlen(skb);
2551         int i, copy = start - offset;
2552         struct sk_buff *frag_iter;
2553         int pos = 0;
2554 
2555         /* Copy header. */
2556         if (copy > 0) {
2557                 if (copy > len)
2558                         copy = len;
2559                 csum = csum_partial_copy_nocheck(skb->data + offset, to,
2560                                                  copy, csum);
2561                 if ((len -= copy) == 0)
2562                         return csum;
2563                 offset += copy;
2564                 to     += copy;
2565                 pos     = copy;
2566         }
2567 
2568         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2569                 int end;
2570 
2571                 WARN_ON(start > offset + len);
2572 
2573                 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
2574                 if ((copy = end - offset) > 0) {
2575                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2576                         u32 p_off, p_len, copied;
2577                         struct page *p;
2578                         __wsum csum2;
2579                         u8 *vaddr;
2580 
2581                         if (copy > len)
2582                                 copy = len;
2583 
2584                         skb_frag_foreach_page(frag,
2585                                               frag->page_offset + offset - start,
2586                                               copy, p, p_off, p_len, copied) {
2587                                 vaddr = kmap_atomic(p);
2588                                 csum2 = csum_partial_copy_nocheck(vaddr + p_off,
2589                                                                   to + copied,
2590                                                                   p_len, 0);
2591                                 kunmap_atomic(vaddr);
2592                                 csum = csum_block_add(csum, csum2, pos);
2593                                 pos += p_len;
2594                         }
2595 
2596                         if (!(len -= copy))
2597                                 return csum;
2598                         offset += copy;
2599                         to     += copy;
2600                 }
2601                 start = end;
2602         }
2603 
2604         skb_walk_frags(skb, frag_iter) {
2605                 __wsum csum2;
2606                 int end;
2607 
2608                 WARN_ON(start > offset + len);
2609 
2610                 end = start + frag_iter->len;
2611                 if ((copy = end - offset) > 0) {
2612                         if (copy > len)
2613                                 copy = len;
2614                         csum2 = skb_copy_and_csum_bits(frag_iter,
2615                                                        offset - start,
2616                                                        to, copy, 0);
2617                         csum = csum_block_add(csum, csum2, pos);
2618                         if ((len -= copy) == 0)
2619                                 return csum;
2620                         offset += copy;
2621                         to     += copy;
2622                         pos    += copy;
2623                 }
2624                 start = end;
2625         }
2626         BUG_ON(len);
2627         return csum;
2628 }
2629 EXPORT_SYMBOL(skb_copy_and_csum_bits);
2630 
2631 static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum)
2632 {
2633         net_warn_ratelimited(
2634                 "%s: attempt to compute crc32c without libcrc32c.ko\n",
2635                 __func__);
2636         return 0;
2637 }
2638 
2639 static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2,
2640                                        int offset, int len)
2641 {
2642         net_warn_ratelimited(
2643                 "%s: attempt to compute crc32c without libcrc32c.ko\n",
2644                 __func__);
2645         return 0;
2646 }
2647 
2648 static const struct skb_checksum_ops default_crc32c_ops = {
2649         .update  = warn_crc32c_csum_update,
2650         .combine = warn_crc32c_csum_combine,
2651 };
2652 
2653 const struct skb_checksum_ops *crc32c_csum_stub __read_mostly =
2654         &default_crc32c_ops;
2655 EXPORT_SYMBOL(crc32c_csum_stub);
2656 
2657  /**
2658  *      skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy()
2659  *      @from: source buffer
2660  *
2661  *      Calculates the amount of linear headroom needed in the 'to' skb passed
2662  *      into skb_zerocopy().
2663  */
2664 unsigned int
2665 skb_zerocopy_headlen(const struct sk_buff *from)
2666 {
2667         unsigned int hlen = 0;
2668 
2669         if (!from->head_frag ||
2670             skb_headlen(from) < L1_CACHE_BYTES ||
2671             skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS)
2672                 hlen = skb_headlen(from);
2673 
2674         if (skb_has_frag_list(from))
2675                 hlen = from->len;
2676 
2677         return hlen;
2678 }
2679 EXPORT_SYMBOL_GPL(skb_zerocopy_headlen);
2680 
2681 /**
2682  *      skb_zerocopy - Zero copy skb to skb
2683  *      @to: destination buffer
2684  *      @from: source buffer
2685  *      @len: number of bytes to copy from source buffer
2686  *      @hlen: size of linear headroom in destination buffer
2687  *
2688  *      Copies up to `len` bytes from `from` to `to` by creating references
2689  *      to the frags in the source buffer.
2690  *
2691  *      The `hlen` as calculated by skb_zerocopy_headlen() specifies the
2692  *      headroom in the `to` buffer.
2693  *
2694  *      Return value:
2695  *      0: everything is OK
2696  *      -ENOMEM: couldn't orphan frags of @from due to lack of memory
2697  *      -EFAULT: skb_copy_bits() found some problem with skb geometry
2698  */
2699 int
2700 skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
2701 {
2702         int i, j = 0;
2703         int plen = 0; /* length of skb->head fragment */
2704         int ret;
2705         struct page *page;
2706         unsigned int offset;
2707 
2708         BUG_ON(!from->head_frag && !hlen);
2709 
2710         /* dont bother with small payloads */
2711         if (len <= skb_tailroom(to))
2712                 return skb_copy_bits(from, 0, skb_put(to, len), len);
2713 
2714         if (hlen) {
2715                 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen);
2716                 if (unlikely(ret))
2717                         return ret;
2718                 len -= hlen;
2719         } else {
2720                 plen = min_t(int, skb_headlen(from), len);
2721                 if (plen) {
2722                         page = virt_to_head_page(from->head);
2723                         offset = from->data - (unsigned char *)page_address(page);
2724                         __skb_fill_page_desc(to, 0, page, offset, plen);
2725                         get_page(page);
2726                         j = 1;
2727                         len -= plen;
2728                 }
2729         }
2730 
2731         to->truesize += len + plen;
2732         to->len += len + plen;
2733         to->data_len += len + plen;
2734 
2735         if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) {
2736                 skb_tx_error(from);
2737                 return -ENOMEM;
2738         }
2739         skb_zerocopy_clone(to, from, GFP_ATOMIC);
2740 
2741         for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
2742                 if (!len)
2743                         break;
2744                 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
2745                 skb_shinfo(to)->frags[j].size = min_t(int, skb_shinfo(to)->frags[j].size, len);
2746                 len -= skb_shinfo(to)->frags[j].size;
2747                 skb_frag_ref(to, j);
2748                 j++;
2749         }
2750         skb_shinfo(to)->nr_frags = j;
2751 
2752         return 0;
2753 }
2754 EXPORT_SYMBOL_GPL(skb_zerocopy);
2755 
2756 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
2757 {
2758         __wsum csum;
2759         long csstart;
2760 
2761         if (skb->ip_summed == CHECKSUM_PARTIAL)
2762                 csstart = skb_checksum_start_offset(skb);
2763         else
2764                 csstart = skb_headlen(skb);
2765 
2766         BUG_ON(csstart > skb_headlen(skb));
2767 
2768         skb_copy_from_linear_data(skb, to, csstart);
2769 
2770         csum = 0;
2771         if (csstart != skb->len)
2772                 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
2773                                               skb->len - csstart, 0);
2774 
2775         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2776                 long csstuff = csstart + skb->csum_offset;
2777 
2778                 *((__sum16 *)(to + csstuff)) = csum_fold(csum);
2779         }
2780 }
2781 EXPORT_SYMBOL(skb_copy_and_csum_dev);
2782 
2783 /**
2784  *      skb_dequeue - remove from the head of the queue
2785  *      @list: list to dequeue from
2786  *
2787  *      Remove the head of the list. The list lock is taken so the function
2788  *      may be used safely with other locking list functions. The head item is
2789  *      returned or %NULL if the list is empty.
2790  */
2791 
2792 struct sk_buff *skb_dequeue(struct sk_buff_head *list)
2793 {
2794         unsigned long flags;
2795         struct sk_buff *result;
2796 
2797         spin_lock_irqsave(&list->lock, flags);
2798         result = __skb_dequeue(list);
2799         spin_unlock_irqrestore(&list->lock, flags);
2800         return result;
2801 }
2802 EXPORT_SYMBOL(skb_dequeue);
2803 
2804 /**
2805  *      skb_dequeue_tail - remove from the tail of the queue
2806  *      @list: list to dequeue from
2807  *
2808  *      Remove the tail of the list. The list lock is taken so the function
2809  *      may be used safely with other locking list functions. The tail item is
2810  *      returned or %NULL if the list is empty.
2811  */
2812 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
2813 {
2814         unsigned long flags;
2815         struct sk_buff *result;
2816 
2817         spin_lock_irqsave(&list->lock, flags);
2818         result = __skb_dequeue_tail(list);
2819         spin_unlock_irqrestore(&list->lock, flags);
2820         return result;
2821 }
2822 EXPORT_SYMBOL(skb_dequeue_tail);
2823 
2824 /**
2825  *      skb_queue_purge - empty a list
2826  *      @list: list to empty
2827  *
2828  *      Delete all buffers on an &sk_buff list. Each buffer is removed from
2829  *      the list and one reference dropped. This function takes the list
2830  *      lock and is atomic with respect to other list locking functions.
2831  */
2832 void skb_queue_purge(struct sk_buff_head *list)
2833 {
2834         struct sk_buff *skb;
2835         while ((skb = skb_dequeue(list)) != NULL)
2836                 kfree_skb(skb);
2837 }
2838 EXPORT_SYMBOL(skb_queue_purge);
2839 
2840 /**
2841  *      skb_rbtree_purge - empty a skb rbtree
2842  *      @root: root of the rbtree to empty
2843  *
2844  *      Delete all buffers on an &sk_buff rbtree. Each buffer is removed from
2845  *      the list and one reference dropped. This function does not take
2846  *      any lock. Synchronization should be handled by the caller (e.g., TCP
2847  *      out-of-order queue is protected by the socket lock).
2848  */
2849 void skb_rbtree_purge(struct rb_root *root)
2850 {
2851         struct rb_node *p = rb_first(root);
2852 
2853         while (p) {
2854                 struct sk_buff *skb = rb_entry(p, struct sk_buff, rbnode);
2855 
2856                 p = rb_next(p);
2857                 rb_erase(&skb->rbnode, root);
2858                 kfree_skb(skb);
2859         }
2860 }
2861 
2862 /**
2863  *      skb_queue_head - queue a buffer at the list head
2864  *      @list: list to use
2865  *      @newsk: buffer to queue
2866  *
2867  *      Queue a buffer at the start of the list. This function takes the
2868  *      list lock and can be used safely with other locking &sk_buff functions
2869  *      safely.
2870  *
2871  *      A buffer cannot be placed on two lists at the same time.
2872  */
2873 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
2874 {
2875         unsigned long flags;
2876 
2877         spin_lock_irqsave(&list->lock, flags);
2878         __skb_queue_head(list, newsk);
2879         spin_unlock_irqrestore(&list->lock, flags);
2880 }
2881 EXPORT_SYMBOL(skb_queue_head);
2882 
2883 /**
2884  *      skb_queue_tail - queue a buffer at the list tail
2885  *      @list: list to use
2886  *      @newsk: buffer to queue
2887  *
2888  *      Queue a buffer at the tail of the list. This function takes the
2889  *      list lock and can be used safely with other locking &sk_buff functions
2890  *      safely.
2891  *
2892  *      A buffer cannot be placed on two lists at the same time.
2893  */
2894 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
2895 {
2896         unsigned long flags;
2897 
2898         spin_lock_irqsave(&list->lock, flags);
2899         __skb_queue_tail(list, newsk);
2900         spin_unlock_irqrestore(&list->lock, flags);
2901 }
2902 EXPORT_SYMBOL(skb_queue_tail);
2903 
2904 /**
2905  *      skb_unlink      -       remove a buffer from a list
2906  *      @skb: buffer to remove
2907  *      @list: list to use
2908  *
2909  *      Remove a packet from a list. The list locks are taken and this
2910  *      function is atomic with respect to other list locked calls
2911  *
2912  *      You must know what list the SKB is on.
2913  */
2914 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
2915 {
2916         unsigned long flags;
2917 
2918         spin_lock_irqsave(&list->lock, flags);
2919         __skb_unlink(skb, list);
2920         spin_unlock_irqrestore(&list->lock, flags);
2921 }
2922 EXPORT_SYMBOL(skb_unlink);
2923 
2924 /**
2925  *      skb_append      -       append a buffer
2926  *      @old: buffer to insert after
2927  *      @newsk: buffer to insert
2928  *      @list: list to use
2929  *
2930  *      Place a packet after a given packet in a list. The list locks are taken
2931  *      and this function is atomic with respect to other list locked calls.
2932  *      A buffer cannot be placed on two lists at the same time.
2933  */
2934 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
2935 {
2936         unsigned long flags;
2937 
2938         spin_lock_irqsave(&list->lock, flags);
2939         __skb_queue_after(list, old, newsk);
2940         spin_unlock_irqrestore(&list->lock, flags);
2941 }
2942 EXPORT_SYMBOL(skb_append);
2943 
2944 /**
2945  *      skb_insert      -       insert a buffer
2946  *      @old: buffer to insert before
2947  *      @newsk: buffer to insert
2948  *      @list: list to use
2949  *
2950  *      Place a packet before a given packet in a list. The list locks are
2951  *      taken and this function is atomic with respect to other list locked
2952  *      calls.
2953  *
2954  *      A buffer cannot be placed on two lists at the same time.
2955  */
2956 void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
2957 {
2958         unsigned long flags;
2959 
2960         spin_lock_irqsave(&list->lock, flags);
2961         __skb_insert(newsk, old->prev, old, list);
2962         spin_unlock_irqrestore(&list->lock, flags);
2963 }
2964 EXPORT_SYMBOL(skb_insert);
2965 
2966 static inline void skb_split_inside_header(struct sk_buff *skb,
2967                                            struct sk_buff* skb1,
2968                                            const u32 len, const int pos)
2969 {
2970         int i;
2971 
2972         skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
2973                                          pos - len);
2974         /* And move data appendix as is. */
2975         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
2976                 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
2977 
2978         skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
2979         skb_shinfo(skb)->nr_frags  = 0;
2980         skb1->data_len             = skb->data_len;
2981         skb1->len                  += skb1->data_len;
2982         skb->data_len              = 0;
2983         skb->len                   = len;
2984         skb_set_tail_pointer(skb, len);
2985 }
2986 
2987 static inline void skb_split_no_header(struct sk_buff *skb,
2988                                        struct sk_buff* skb1,
2989                                        const u32 len, int pos)
2990 {
2991         int i, k = 0;
2992         const int nfrags = skb_shinfo(skb)->nr_frags;
2993 
2994         skb_shinfo(skb)->nr_frags = 0;
2995         skb1->len                 = skb1->data_len = skb->len - len;
2996         skb->len                  = len;
2997         skb->data_len             = len - pos;
2998 
2999         for (i = 0; i < nfrags; i++) {
3000                 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]);
3001 
3002                 if (pos + size > len) {
3003                         skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
3004 
3005                         if (pos < len) {
3006                                 /* Split frag.
3007                                  * We have two variants in this case:
3008                                  * 1. Move all the frag to the second
3009                                  *    part, if it is possible. F.e.
3010                                  *    this approach is mandatory for TUX,
3011                                  *    where splitting is expensive.
3012                                  * 2. Split is accurately. We make this.
3013                                  */
3014                                 skb_frag_ref(skb, i);
3015                                 skb_shinfo(skb1)->frags[0].page_offset += len - pos;
3016                                 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
3017                                 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
3018                                 skb_shinfo(skb)->nr_frags++;
3019                         }
3020                         k++;
3021                 } else
3022                         skb_shinfo(skb)->nr_frags++;
3023                 pos += size;
3024         }
3025         skb_shinfo(skb1)->nr_frags = k;
3026 }
3027 
3028 /**
3029  * skb_split - Split fragmented skb to two parts at length len.
3030  * @skb: the buffer to split
3031  * @skb1: the buffer to receive the second part
3032  * @len: new length for skb
3033  */
3034 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
3035 {
3036         int pos = skb_headlen(skb);
3037 
3038         skb_shinfo(skb1)->tx_flags |= skb_shinfo(skb)->tx_flags &
3039                                       SKBTX_SHARED_FRAG;
3040         skb_zerocopy_clone(skb1, skb, 0);
3041         if (len < pos)  /* Split line is inside header. */
3042                 skb_split_inside_header(skb, skb1, len, pos);
3043         else            /* Second chunk has no header, nothing to copy. */
3044                 skb_split_no_header(skb, skb1, len, pos);
3045 }
3046 EXPORT_SYMBOL(skb_split);
3047 
3048 /* Shifting from/to a cloned skb is a no-go.
3049  *
3050  * Caller cannot keep skb_shinfo related pointers past calling here!
3051  */
3052 static int skb_prepare_for_shift(struct sk_buff *skb)
3053 {
3054         return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3055 }
3056 
3057 /**
3058  * skb_shift - Shifts paged data partially from skb to another
3059  * @tgt: buffer into which tail data gets added
3060  * @skb: buffer from which the paged data comes from
3061  * @shiftlen: shift up to this many bytes
3062  *
3063  * Attempts to shift up to shiftlen worth of bytes, which may be less than
3064  * the length of the skb, from skb to tgt. Returns number bytes shifted.
3065  * It's up to caller to free skb if everything was shifted.
3066  *
3067  * If @tgt runs out of frags, the whole operation is aborted.
3068  *
3069  * Skb cannot include anything else but paged data while tgt is allowed
3070  * to have non-paged data as well.
3071  *
3072  * TODO: full sized shift could be optimized but that would need
3073  * specialized skb free'er to handle frags without up-to-date nr_frags.
3074  */
3075 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
3076 {
3077         int from, to, merge, todo;
3078         struct skb_frag_struct *fragfrom, *fragto;
3079 
3080         BUG_ON(shiftlen > skb->len);
3081 
3082         if (skb_headlen(skb))
3083                 return 0;
3084         if (skb_zcopy(tgt) || skb_zcopy(skb))
3085                 return 0;
3086 
3087         todo = shiftlen;
3088         from = 0;
3089         to = skb_shinfo(tgt)->nr_frags;
3090         fragfrom = &skb_shinfo(skb)->frags[from];
3091 
3092         /* Actual merge is delayed until the point when we know we can
3093          * commit all, so that we don't have to undo partial changes
3094          */
3095         if (!to ||
3096             !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
3097                               fragfrom->page_offset)) {
3098                 merge = -1;
3099         } else {
3100                 merge = to - 1;
3101 
3102                 todo -= skb_frag_size(fragfrom);
3103                 if (todo < 0) {
3104                         if (skb_prepare_for_shift(skb) ||
3105                             skb_prepare_for_shift(tgt))
3106                                 return 0;
3107 
3108                         /* All previous frag pointers might be stale! */
3109                         fragfrom = &skb_shinfo(skb)->frags[from];
3110                         fragto = &skb_shinfo(tgt)->frags[merge];
3111 
3112                         skb_frag_size_add(fragto, shiftlen);
3113                         skb_frag_size_sub(fragfrom, shiftlen);
3114                         fragfrom->page_offset += shiftlen;
3115 
3116                         goto onlymerged;
3117                 }
3118 
3119                 from++;
3120         }
3121 
3122         /* Skip full, not-fitting skb to avoid expensive operations */
3123         if ((shiftlen == skb->len) &&
3124             (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
3125                 return 0;
3126 
3127         if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
3128                 return 0;
3129 
3130         while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
3131                 if (to == MAX_SKB_FRAGS)
3132                         return 0;
3133 
3134                 fragfrom = &skb_shinfo(skb)->frags[from];
3135                 fragto = &skb_shinfo(tgt)->frags[to];
3136 
3137                 if (todo >= skb_frag_size(fragfrom)) {
3138                         *fragto = *fragfrom;
3139                         todo -= skb_frag_size(fragfrom);
3140                         from++;
3141                         to++;
3142 
3143                 } else {
3144                         __skb_frag_ref(fragfrom);
3145                         fragto->page = fragfrom->page;
3146                         fragto->page_offset = fragfrom->page_offset;
3147                         skb_frag_size_set(fragto, todo);
3148 
3149                         fragfrom->page_offset += todo;
3150                         skb_frag_size_sub(fragfrom, todo);
3151                         todo = 0;
3152 
3153                         to++;
3154                         break;
3155                 }
3156         }
3157 
3158         /* Ready to "commit" this state change to tgt */
3159         skb_shinfo(tgt)->nr_frags = to;
3160 
3161         if (merge >= 0) {
3162                 fragfrom = &skb_shinfo(skb)->frags[0];
3163                 fragto = &skb_shinfo(tgt)->frags[merge];
3164 
3165                 skb_frag_size_add(fragto, skb_frag_size(fragfrom));
3166                 __skb_frag_unref(fragfrom);
3167         }
3168 
3169         /* Reposition in the original skb */
3170         to = 0;
3171         while (from < skb_shinfo(skb)->nr_frags)
3172                 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
3173         skb_shinfo(skb)->nr_frags = to;
3174 
3175         BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
3176 
3177 onlymerged:
3178         /* Most likely the tgt won't ever need its checksum anymore, skb on
3179          * the other hand might need it if it needs to be resent
3180          */
3181         tgt->ip_summed = CHECKSUM_PARTIAL;
3182         skb->ip_summed = CHECKSUM_PARTIAL;
3183 
3184         /* Yak, is it really working this way? Some helper please? */
3185         skb->len -= shiftlen;
3186         skb->data_len -= shiftlen;
3187         skb->truesize -= shiftlen;
3188         tgt->len += shiftlen;
3189         tgt->data_len += shiftlen;
3190         tgt->truesize += shiftlen;
3191 
3192         return shiftlen;
3193 }
3194 
3195 /**
3196  * skb_prepare_seq_read - Prepare a sequential read of skb data
3197  * @skb: the buffer to read
3198  * @from: lower offset of data to be read
3199  * @to: upper offset of data to be read
3200  * @st: state variable
3201  *
3202  * Initializes the specified state variable. Must be called before
3203  * invoking skb_seq_read() for the first time.
3204  */
3205 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
3206                           unsigned int to, struct skb_seq_state *st)
3207 {
3208         st->lower_offset = from;
3209         st->upper_offset = to;
3210         st->root_skb = st->cur_skb = skb;
3211         st->frag_idx = st->stepped_offset = 0;
3212         st->frag_data = NULL;
3213 }
3214 EXPORT_SYMBOL(skb_prepare_seq_read);
3215 
3216 /**
3217  * skb_seq_read - Sequentially read skb data
3218  * @consumed: number of bytes consumed by the caller so far
3219  * @data: destination pointer for data to be returned
3220  * @st: state variable
3221  *
3222  * Reads a block of skb data at @consumed relative to the
3223  * lower offset specified to skb_prepare_seq_read(). Assigns
3224  * the head of the data block to @data and returns the length
3225  * of the block or 0 if the end of the skb data or the upper
3226  * offset has been reached.
3227  *
3228  * The caller is not required to consume all of the data
3229  * returned, i.e. @consumed is typically set to the number
3230  * of bytes already consumed and the next call to
3231  * skb_seq_read() will return the remaining part of the block.
3232  *
3233  * Note 1: The size of each block of data returned can be arbitrary,
3234  *       this limitation is the cost for zerocopy sequential
3235  *       reads of potentially non linear data.
3236  *
3237  * Note 2: Fragment lists within fragments are not implemented
3238  *       at the moment, state->root_skb could be replaced with
3239  *       a stack for this purpose.
3240  */
3241 unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
3242                           struct skb_seq_state *st)
3243 {
3244         unsigned int block_limit, abs_offset = consumed + st->lower_offset;
3245         skb_frag_t *frag;
3246 
3247         if (unlikely(abs_offset >= st->upper_offset)) {
3248                 if (st->frag_data) {
3249                         kunmap_atomic(st->frag_data);
3250                         st->frag_data = NULL;
3251                 }
3252                 return 0;
3253         }
3254 
3255 next_skb:
3256         block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
3257 
3258         if (abs_offset < block_limit && !st->frag_data) {
3259                 *data = st->cur_skb->data + (abs_offset - st->stepped_offset);
3260                 return block_limit - abs_offset;
3261         }
3262 
3263         if (st->frag_idx == 0 && !st->frag_data)
3264                 st->stepped_offset += skb_headlen(st->cur_skb);
3265 
3266         while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
3267                 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
3268                 block_limit = skb_frag_size(frag) + st->stepped_offset;
3269 
3270                 if (abs_offset < block_limit) {
3271                         if (!st->frag_data)
3272                                 st->frag_data = kmap_atomic(skb_frag_page(frag));
3273 
3274                         *data = (u8 *) st->frag_data + frag->page_offset +
3275                                 (abs_offset - st->stepped_offset);
3276 
3277                         return block_limit - abs_offset;
3278                 }
3279 
3280                 if (st->frag_data) {
3281                         kunmap_atomic(st->frag_data);
3282                         st->frag_data = NULL;
3283                 }
3284 
3285                 st->frag_idx++;
3286                 st->stepped_offset += skb_frag_size(frag);
3287         }
3288 
3289         if (st->frag_data) {
3290                 kunmap_atomic(st->frag_data);
3291                 st->frag_data = NULL;
3292         }
3293 
3294         if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
3295                 st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
3296                 st->frag_idx = 0;
3297                 goto next_skb;
3298         } else if (st->cur_skb->next) {
3299                 st->cur_skb = st->cur_skb->next;
3300                 st->frag_idx = 0;
3301                 goto next_skb;
3302         }
3303 
3304         return 0;
3305 }
3306 EXPORT_SYMBOL(skb_seq_read);
3307 
3308 /**
3309  * skb_abort_seq_read - Abort a sequential read of skb data
3310  * @st: state variable
3311  *
3312  * Must be called if skb_seq_read() was not called until it
3313  * returned 0.
3314  */
3315 void skb_abort_seq_read(struct skb_seq_state *st)
3316 {
3317         if (st->frag_data)
3318                 kunmap_atomic(st->frag_data);
3319 }
3320 EXPORT_SYMBOL(skb_abort_seq_read);
3321 
3322 #define TS_SKB_CB(state)        ((struct skb_seq_state *) &((state)->cb))
3323 
3324 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
3325                                           struct ts_config *conf,
3326                                           struct ts_state *state)
3327 {
3328         return skb_seq_read(offset, text, TS_SKB_CB(state));
3329 }
3330 
3331 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
3332 {
3333         skb_abort_seq_read(TS_SKB_CB(state));
3334 }
3335 
3336 /**
3337  * skb_find_text - Find a text pattern in skb data
3338  * @skb: the buffer to look in
3339  * @from: search offset
3340  * @to: search limit
3341  * @config: textsearch configuration
3342  *
3343  * Finds a pattern in the skb data according to the specified
3344  * textsearch configuration. Use textsearch_next() to retrieve
3345  * subsequent occurrences of the pattern. Returns the offset
3346  * to the first occurrence or UINT_MAX if no match was found.
3347  */
3348 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
3349                            unsigned int to, struct ts_config *config)
3350 {
3351         struct ts_state state;
3352         unsigned int ret;
3353 
3354         config->get_next_block = skb_ts_get_next_block;
3355         config->finish = skb_ts_finish;
3356 
3357         skb_prepare_seq_read(skb, from, to, TS_SKB_CB(&state));
3358 
3359         ret = textsearch_find(config, &state);
3360         return (ret <= to - from ? ret : UINT_MAX);
3361 }
3362 EXPORT_SYMBOL(skb_find_text);
3363 
3364 /**
3365  * skb_append_datato_frags - append the user data to a skb
3366  * @sk: sock  structure
3367  * @skb: skb structure to be appended with user data.
3368  * @getfrag: call back function to be used for getting the user data
3369  * @from: pointer to user message iov
3370  * @length: length of the iov message
3371  *
3372  * Description: This procedure append the user data in the fragment part
3373  * of the skb if any page alloc fails user this procedure returns  -ENOMEM
3374  */
3375 int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
3376                         int (*getfrag)(void *from, char *to, int offset,
3377                                         int len, int odd, struct sk_buff *skb),
3378                         void *from, int length)
3379 {
3380         int frg_cnt = skb_shinfo(skb)->nr_frags;
3381         int copy;
3382         int offset = 0;
3383         int ret;
3384         struct page_frag *pfrag = &current->task_frag;
3385 
3386         do {
3387                 /* Return error if we don't have space for new frag */
3388                 if (frg_cnt >= MAX_SKB_FRAGS)
3389                         return -EMSGSIZE;
3390 
3391                 if (!sk_page_frag_refill(sk, pfrag))
3392                         return -ENOMEM;
3393 
3394                 /* copy the user data to page */
3395                 copy = min_t(int, length, pfrag->size - pfrag->offset);
3396 
3397                 ret = getfrag(from, page_address(pfrag->page) + pfrag->offset,
3398                               offset, copy, 0, skb);
3399                 if (ret < 0)
3400                         return -EFAULT;
3401 
3402                 /* copy was successful so update the size parameters */
3403                 skb_fill_page_desc(skb, frg_cnt, pfrag->page, pfrag->offset,
3404                                    copy);
3405                 frg_cnt++;
3406                 pfrag->offset += copy;
3407                 get_page(pfrag->page);
3408 
3409                 skb->truesize += copy;
3410                 refcount_add(copy, &sk->sk_wmem_alloc);
3411                 skb->len += copy;
3412                 skb->data_len += copy;
3413                 offset += copy;
3414                 length -= copy;
3415 
3416         } while (length > 0);
3417 
3418         return 0;
3419 }
3420 EXPORT_SYMBOL(skb_append_datato_frags);
3421 
3422 int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
3423                          int offset, size_t size)
3424 {
3425         int i = skb_shinfo(skb)->nr_frags;
3426 
3427         if (skb_can_coalesce(skb, i, page, offset)) {
3428                 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
3429         } else if (i < MAX_SKB_FRAGS) {
3430                 get_page(page);
3431                 skb_fill_page_desc(skb, i, page, offset, size);
3432         } else {
3433                 return -EMSGSIZE;
3434         }
3435 
3436         return 0;
3437 }
3438 EXPORT_SYMBOL_GPL(skb_append_pagefrags);
3439 
3440 /**
3441  *      skb_pull_rcsum - pull skb and update receive checksum
3442  *      @skb: buffer to update
3443  *      @len: length of data pulled
3444  *
3445  *      This function performs an skb_pull on the packet and updates
3446  *      the CHECKSUM_COMPLETE checksum.  It should be used on
3447  *      receive path processing instead of skb_pull unless you know
3448  *      that the checksum difference is zero (e.g., a valid IP header)
3449  *      or you are setting ip_summed to CHECKSUM_NONE.
3450  */
3451 void *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
3452 {
3453         unsigned char *data = skb->data;
3454 
3455         BUG_ON(len > skb->len);
3456         __skb_pull(skb, len);
3457         skb_postpull_rcsum(skb, data, len);
3458         return skb->data;
3459 }
3460 EXPORT_SYMBOL_GPL(skb_pull_rcsum);
3461 
3462 /**
3463  *      skb_segment - Perform protocol segmentation on skb.
3464  *      @head_skb: buffer to segment
3465  *      @features: features for the output path (see dev->features)
3466  *
3467  *      This function performs segmentation on the given skb.  It returns
3468  *      a pointer to the first in a list of new skbs for the segments.
3469  *      In case of error it returns ERR_PTR(err).
3470  */
3471 struct sk_buff *skb_segment(struct sk_buff *head_skb,
3472                             netdev_features_t features)
3473 {
3474         struct sk_buff *segs = NULL;
3475         struct sk_buff *tail = NULL;
3476         struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list;
3477         skb_frag_t *frag = skb_shinfo(head_skb)->frags;
3478         unsigned int mss = skb_shinfo(head_skb)->gso_size;
3479         unsigned int doffset = head_skb->data - skb_mac_header(head_skb);
3480         struct sk_buff *frag_skb = head_skb;
3481         unsigned int offset = doffset;
3482         unsigned int tnl_hlen = skb_tnl_header_len(head_skb);
3483         unsigned int partial_segs = 0;
3484         unsigned int headroom;
3485         unsigned int len = head_skb->len;
3486         __be16 proto;
3487         bool csum, sg;
3488         int nfrags = skb_shinfo(head_skb)->nr_frags;
3489         int err = -ENOMEM;
3490         int i = 0;
3491         int pos;
3492         int dummy;
3493 
3494         __skb_push(head_skb, doffset);
3495         proto = skb_network_protocol(head_skb, &dummy);
3496         if (unlikely(!proto))
3497                 return ERR_PTR(-EINVAL);
3498 
3499         sg = !!(features & NETIF_F_SG);
3500         csum = !!can_checksum_protocol(features, proto);
3501 
3502         if (sg && csum && (mss != GSO_BY_FRAGS))  {
3503                 if (!(features & NETIF_F_GSO_PARTIAL)) {
3504                         struct sk_buff *iter;
3505                         unsigned int frag_len;
3506 
3507                         if (!list_skb ||
3508                             !net_gso_ok(features, skb_shinfo(head_skb)->gso_type))
3509                                 goto normal;
3510 
3511                         /* If we get here then all the required
3512                          * GSO features except frag_list are supported.
3513                          * Try to split the SKB to multiple GSO SKBs
3514                          * with no frag_list.
3515                          * Currently we can do that only when the buffers don't
3516                          * have a linear part and all the buffers except
3517                          * the last are of the same length.
3518                          */
3519                         frag_len = list_skb->len;
3520                         skb_walk_frags(head_skb, iter) {
3521                                 if (frag_len != iter->len && iter->next)
3522                                         goto normal;
3523                                 if (skb_headlen(iter) && !iter->head_frag)
3524                                         goto normal;
3525 
3526                                 len -= iter->len;
3527                         }
3528 
3529                         if (len != frag_len)
3530                                 goto normal;
3531                 }
3532 
3533                 /* GSO partial only requires that we trim off any excess that
3534                  * doesn't fit into an MSS sized block, so take care of that
3535                  * now.
3536                  */
3537                 partial_segs = len / mss;
3538                 if (partial_segs > 1)
3539                         mss *= partial_segs;
3540                 else
3541                         partial_segs = 0;
3542         }
3543 
3544 normal:
3545         headroom = skb_headroom(head_skb);
3546         pos = skb_headlen(head_skb);
3547 
3548         do {
3549                 struct sk_buff *nskb;
3550                 skb_frag_t *nskb_frag;
3551                 int hsize;
3552                 int size;
3553 
3554                 if (unlikely(mss == GSO_BY_FRAGS)) {
3555                         len = list_skb->len;
3556                 } else {
3557                         len = head_skb->len - offset;
3558                         if (len > mss)
3559                                 len = mss;
3560                 }
3561 
3562                 hsize = skb_headlen(head_skb) - offset;
3563                 if (hsize < 0)
3564                         hsize = 0;
3565                 if (hsize > len || !sg)
3566                         hsize = len;
3567 
3568                 if (!hsize && i >= nfrags && skb_headlen(list_skb) &&
3569                     (skb_headlen(list_skb) == len || sg)) {
3570                         BUG_ON(skb_headlen(list_skb) > len);
3571 
3572                         i = 0;
3573                         nfrags = skb_shinfo(list_skb)->nr_frags;
3574                         frag = skb_shinfo(list_skb)->frags;
3575                         frag_skb = list_skb;
3576                         pos += skb_headlen(list_skb);
3577 
3578                         while (pos < offset + len) {
3579                                 BUG_ON(i >= nfrags);
3580 
3581                                 size = skb_frag_size(frag);
3582                                 if (pos + size > offset + len)
3583                                         break;
3584 
3585                                 i++;
3586                                 pos += size;
3587                                 frag++;
3588                         }
3589 
3590                         nskb = skb_clone(list_skb, GFP_ATOMIC);
3591                         list_skb = list_skb->next;
3592 
3593                         if (unlikely(!nskb))
3594                                 goto err;
3595 
3596                         if (unlikely(pskb_trim(nskb, len))) {
3597                                 kfree_skb(nskb);
3598                                 goto err;
3599                         }
3600 
3601                         hsize = skb_end_offset(nskb);
3602                         if (skb_cow_head(nskb, doffset + headroom)) {
3603                                 kfree_skb(nskb);
3604                                 goto err;
3605                         }
3606 
3607                         nskb->truesize += skb_end_offset(nskb) - hsize;
3608                         skb_release_head_state(nskb);
3609                         __skb_push(nskb, doffset);
3610                 } else {
3611                         nskb = __alloc_skb(hsize + doffset + headroom,
3612                                            GFP_ATOMIC, skb_alloc_rx_flag(head_skb),
3613                                            NUMA_NO_NODE);
3614 
3615                         if (unlikely(!nskb))
3616                                 goto err;
3617 
3618                         skb_reserve(nskb, headroom);
3619                         __skb_put(nskb, doffset);
3620                 }
3621 
3622                 if (segs)
3623                         tail->next = nskb;
3624                 else
3625                         segs = nskb;
3626                 tail = nskb;
3627 
3628                 __copy_skb_header(nskb, head_skb);
3629 
3630                 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom);
3631                 skb_reset_mac_len(nskb);
3632 
3633                 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen,
3634                                                  nskb->data - tnl_hlen,
3635                                                  doffset + tnl_hlen);
3636 
3637                 if (nskb->len == len + doffset)
3638                         goto perform_csum_check;
3639 
3640                 if (!sg) {
3641                         if (!nskb->remcsum_offload)
3642                                 nskb->ip_summed = CHECKSUM_NONE;
3643                         SKB_GSO_CB(nskb)->csum =
3644                                 skb_copy_and_csum_bits(head_skb, offset,
3645                                                        skb_put(nskb, len),
3646                                                        len, 0);
3647                         SKB_GSO_CB(nskb)->csum_start =
3648                                 skb_headroom(nskb) + doffset;
3649                         continue;
3650                 }
3651 
3652                 nskb_frag = skb_shinfo(nskb)->frags;
3653 
3654                 skb_copy_from_linear_data_offset(head_skb, offset,
3655                                                  skb_put(nskb, hsize), hsize);
3656 
3657                 skb_shinfo(nskb)->tx_flags |= skb_shinfo(head_skb)->tx_flags &
3658                                               SKBTX_SHARED_FRAG;
3659 
3660                 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) ||
3661                     skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC))
3662                         goto err;
3663 
3664                 while (pos < offset + len) {
3665                         if (i >= nfrags) {
3666                                 BUG_ON(skb_headlen(list_skb));
3667 
3668                                 i = 0;
3669                                 nfrags = skb_shinfo(list_skb)->nr_frags;
3670                                 frag = skb_shinfo(list_skb)->frags;
3671                                 frag_skb = list_skb;
3672 
3673                                 BUG_ON(!nfrags);
3674 
3675                                 if (skb_orphan_frags(frag_skb, GFP_ATOMIC) ||
3676                                     skb_zerocopy_clone(nskb, frag_skb,
3677                                                        GFP_ATOMIC))
3678                                         goto err;
3679 
3680                                 list_skb = list_skb->next;
3681                         }
3682 
3683                         if (unlikely(skb_shinfo(nskb)->nr_frags >=
3684                                      MAX_SKB_FRAGS)) {
3685                                 net_warn_ratelimited(
3686                                         "skb_segment: too many frags: %u %u\n",
3687                                         pos, mss);
3688                                 goto err;
3689                         }
3690 
3691                         *nskb_frag = *frag;
3692                         __skb_frag_ref(nskb_frag);
3693                         size = skb_frag_size(nskb_frag);
3694 
3695                         if (pos < offset) {
3696                                 nskb_frag->page_offset += offset - pos;
3697                                 skb_frag_size_sub(nskb_frag, offset - pos);
3698                         }
3699 
3700                         skb_shinfo(nskb)->nr_frags++;
3701 
3702                         if (pos + size <= offset + len) {
3703                                 i++;
3704                                 frag++;
3705                                 pos += size;
3706                         } else {
3707                                 skb_frag_size_sub(nskb_frag, pos + size - (offset + len));
3708                                 goto skip_fraglist;
3709                         }
3710 
3711                         nskb_frag++;
3712                 }
3713 
3714 skip_fraglist:
3715                 nskb->data_len = len - hsize;
3716                 nskb->len += nskb->data_len;
3717                 nskb->truesize += nskb->data_len;
3718 
3719 perform_csum_check:
3720                 if (!csum) {
3721                         if (skb_has_shared_frag(nskb)) {
3722                                 err = __skb_linearize(nskb);
3723                                 if (err)
3724                                         goto err;
3725                         }
3726                         if (!nskb->remcsum_offload)
3727                                 nskb->ip_summed = CHECKSUM_NONE;
3728                         SKB_GSO_CB(nskb)->csum =
3729                                 skb_checksum(nskb, doffset,
3730                                              nskb->len - doffset, 0);
3731                         SKB_GSO_CB(nskb)->csum_start =
3732                                 skb_headroom(nskb) + doffset;
3733                 }
3734         } while ((offset += len) < head_skb->len);
3735 
3736         /* Some callers want to get the end of the list.
3737          * Put it in segs->prev to avoid walking the list.
3738          * (see validate_xmit_skb_list() for example)
3739          */
3740         segs->prev = tail;
3741 
3742         if (partial_segs) {
3743                 struct sk_buff *iter;
3744                 int type = skb_shinfo(head_skb)->gso_type;
3745                 unsigned short gso_size = skb_shinfo(head_skb)->gso_size;
3746 
3747                 /* Update type to add partial and then remove dodgy if set */
3748                 type |= (features & NETIF_F_GSO_PARTIAL) / NETIF_F_GSO_PARTIAL * SKB_GSO_PARTIAL;
3749                 type &= ~SKB_GSO_DODGY;
3750 
3751                 /* Update GSO info and prepare to start updating headers on
3752                  * our way back down the stack of protocols.
3753                  */
3754                 for (iter = segs; iter; iter = iter->next) {
3755                         skb_shinfo(iter)->gso_size = gso_size;
3756                         skb_shinfo(iter)->gso_segs = partial_segs;
3757                         skb_shinfo(iter)->gso_type = type;
3758                         SKB_GSO_CB(iter)->data_offset = skb_headroom(iter) + doffset;
3759                 }
3760 
3761                 if (tail->len - doffset <= gso_size)
3762                         skb_shinfo(tail)->gso_size = 0;
3763                 else if (tail != segs)
3764                         skb_shinfo(tail)->gso_segs = DIV_ROUND_UP(tail->len - doffset, gso_size);
3765         }
3766 
3767         /* Following permits correct backpressure, for protocols
3768          * using skb_set_owner_w().
3769          * Idea is to tranfert ownership from head_skb to last segment.
3770          */
3771         if (head_skb->destructor == sock_wfree) {
3772                 swap(tail->truesize, head_skb->truesize);
3773                 swap(tail->destructor, head_skb->destructor);
3774                 swap(tail->sk, head_skb->sk);
3775         }
3776         return segs;
3777 
3778 err:
3779         kfree_skb_list(segs);
3780         return ERR_PTR(err);
3781 }
3782 EXPORT_SYMBOL_GPL(skb_segment);
3783 
3784 int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
3785 {
3786         struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb);
3787         unsigned int offset = skb_gro_offset(skb);
3788         unsigned int headlen = skb_headlen(skb);
3789         unsigned int len = skb_gro_len(skb);
3790         struct sk_buff *lp, *p = *head;
3791         unsigned int delta_truesize;
3792 
3793         if (unlikely(p->len + len >= 65536))
3794                 return -E2BIG;
3795 
3796         lp = NAPI_GRO_CB(p)->last;
3797         pinfo = skb_shinfo(lp);
3798 
3799         if (headlen <= offset) {
3800                 skb_frag_t *frag;
3801                 skb_frag_t *frag2;
3802                 int i = skbinfo->nr_frags;
3803                 int nr_frags = pinfo->nr_frags + i;
3804 
3805                 if (nr_frags > MAX_SKB_FRAGS)
3806                         goto merge;
3807 
3808                 offset -= headlen;
3809                 pinfo->nr_frags = nr_frags;
3810                 skbinfo->nr_frags = 0;
3811 
3812                 frag = pinfo->frags + nr_frags;
3813                 frag2 = skbinfo->frags + i;
3814                 do {
3815                         *--frag = *--frag2;
3816                 } while (--i);
3817 
3818                 frag->page_offset += offset;
3819                 skb_frag_size_sub(frag, offset);
3820 
3821                 /* all fragments truesize : remove (head size + sk_buff) */
3822                 delta_truesize = skb->truesize -
3823                                  SKB_TRUESIZE(skb_end_offset(skb));
3824 
3825                 skb->truesize -= skb->data_len;
3826                 skb->len -= skb->data_len;
3827                 skb->data_len = 0;
3828 
3829                 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE;
3830                 goto done;
3831         } else if (skb->head_frag) {
3832                 int nr_frags = pinfo->nr_frags;
3833                 skb_frag_t *frag = pinfo->frags + nr_frags;
3834                 struct page *page = virt_to_head_page(skb->head);
3835                 unsigned int first_size = headlen - offset;
3836                 unsigned int first_offset;
3837 
3838                 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
3839                         goto merge;
3840 
3841                 first_offset = skb->data -
3842                                (unsigned char *)page_address(page) +
3843                                offset;
3844 
3845                 pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
3846 
3847                 frag->page.p      = page;
3848                 frag->page_offset = first_offset;
3849                 skb_frag_size_set(frag, first_size);
3850 
3851                 memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
3852                 /* We dont need to clear skbinfo->nr_frags here */
3853 
3854                 delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
3855                 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
3856                 goto done;
3857         }
3858 
3859 merge:
3860         delta_truesize = skb->truesize;
3861         if (offset > headlen) {
3862                 unsigned int eat = offset - headlen;
3863 
3864                 skbinfo->frags[0].page_offset += eat;
3865                 skb_frag_size_sub(&skbinfo->frags[0], eat);
3866                 skb->data_len -= eat;
3867                 skb->len -= eat;
3868                 offset = headlen;
3869         }
3870 
3871         __skb_pull(skb, offset);
3872 
3873         if (NAPI_GRO_CB(p)->last == p)
3874                 skb_shinfo(p)->frag_list = skb;
3875         else
3876                 NAPI_GRO_CB(p)->last->next = skb;
3877         NAPI_GRO_CB(p)->last = skb;
3878         __skb_header_release(skb);
3879         lp = p;
3880 
3881 done:
3882         NAPI_GRO_CB(p)->count++;
3883         p->data_len += len;
3884         p->truesize += delta_truesize;
3885         p->len += len;
3886         if (lp != p) {
3887                 lp->data_len += len;
3888                 lp->truesize += delta_truesize;
3889                 lp->len += len;
3890         }
3891         NAPI_GRO_CB(skb)->same_flow = 1;
3892         return 0;
3893 }
3894 EXPORT_SYMBOL_GPL(skb_gro_receive);
3895 
3896 void __init skb_init(void)
3897 {
3898         skbuff_head_cache = kmem_cache_create_usercopy("skbuff_head_cache",
3899                                               sizeof(struct sk_buff),
3900                                               0,
3901                                               SLAB_HWCACHE_ALIGN|SLAB_PANIC,
3902                                               offsetof(struct sk_buff, cb),
3903                                               sizeof_field(struct sk_buff, cb),
3904                                               NULL);
3905         skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
3906                                                 sizeof(struct sk_buff_fclones),
3907                                                 0,
3908                                                 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
3909                                                 NULL);
3910 }
3911 
3912 static int
3913 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len,
3914                unsigned int recursion_level)
3915 {
3916         int start = skb_headlen(skb);
3917         int i, copy = start - offset;
3918         struct sk_buff *frag_iter;
3919         int elt = 0;
3920 
3921         if (unlikely(recursion_level >= 24))
3922                 return -EMSGSIZE;
3923 
3924         if (copy > 0) {
3925                 if (copy > len)
3926                         copy = len;
3927                 sg_set_buf(sg, skb->data + offset, copy);
3928                 elt++;
3929                 if ((len -= copy) == 0)
3930                         return elt;
3931                 offset += copy;
3932         }
3933 
3934         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3935                 int end;
3936 
3937                 WARN_ON(start > offset + len);
3938 
3939                 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
3940                 if ((copy = end - offset) > 0) {
3941                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3942                         if (unlikely(elt && sg_is_last(&sg[elt - 1])))
3943                                 return -EMSGSIZE;
3944 
3945                         if (copy > len)
3946                                 copy = len;
3947                         sg_set_page(&sg[elt], skb_frag_page(frag), copy,
3948                                         frag->page_offset+offset-start);
3949                         elt++;
3950                         if (!(len -= copy))
3951                                 return elt;
3952                         offset += copy;
3953                 }
3954                 start = end;
3955         }
3956 
3957         skb_walk_frags(skb, frag_iter) {
3958                 int end, ret;
3959 
3960                 WARN_ON(start > offset + len);
3961 
3962                 end = start + frag_iter->len;
3963                 if ((copy = end - offset) > 0) {
3964                         if (unlikely(elt && sg_is_last(&sg[elt - 1])))
3965                                 return -EMSGSIZE;
3966 
3967                         if (copy > len)
3968                                 copy = len;
3969                         ret = __skb_to_sgvec(frag_iter, sg+elt, offset - start,
3970                                               copy, recursion_level + 1);
3971                         if (unlikely(ret < 0))
3972                                 return ret;
3973                         elt += ret;
3974                         if ((len -= copy) == 0)
3975                                 return elt;
3976                         offset += copy;
3977                 }
3978                 start = end;
3979         }
3980         BUG_ON(len);
3981         return elt;
3982 }
3983 
3984 /**
3985  *      skb_to_sgvec - Fill a scatter-gather list from a socket buffer
3986  *      @skb: Socket buffer containing the buffers to be mapped
3987  *      @sg: The scatter-gather list to map into
3988  *      @offset: The offset into the buffer's contents to start mapping
3989  *      @len: Length of buffer space to be mapped
3990  *
3991  *      Fill the specified scatter-gather list with mappings/pointers into a
3992  *      region of the buffer space attached to a socket buffer. Returns either
3993  *      the number of scatterlist items used, or -EMSGSIZE if the contents
3994  *      could not fit.
3995  */
3996 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
3997 {
3998         int nsg = __skb_to_sgvec(skb, sg, offset, len, 0);
3999 
4000         if (nsg <= 0)
4001                 return nsg;
4002 
4003         sg_mark_end(&sg[nsg - 1]);
4004 
4005         return nsg;
4006 }
4007 EXPORT_SYMBOL_GPL(skb_to_sgvec);
4008 
4009 /* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given
4010  * sglist without mark the sg which contain last skb data as the end.
4011  * So the caller can mannipulate sg list as will when padding new data after
4012  * the first call without calling sg_unmark_end to expend sg list.
4013  *
4014  * Scenario to use skb_to_sgvec_nomark:
4015  * 1. sg_init_table
4016  * 2. skb_to_sgvec_nomark(payload1)
4017  * 3. skb_to_sgvec_nomark(payload2)
4018  *
4019  * This is equivalent to:
4020  * 1. sg_init_table
4021  * 2. skb_to_sgvec(payload1)
4022  * 3. sg_unmark_end
4023  * 4. skb_to_sgvec(payload2)
4024  *
4025  * When mapping mutilple payload conditionally, skb_to_sgvec_nomark
4026  * is more preferable.
4027  */
4028 int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg,
4029                         int offset, int len)
4030 {
4031         return __skb_to_sgvec(skb, sg, offset, len, 0);
4032 }
4033 EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark);
4034 
4035 
4036 
4037 /**
4038  *      skb_cow_data - Check that a socket buffer's data buffers are writable
4039  *      @skb: The socket buffer to check.
4040  *      @tailbits: Amount of trailing space to be added
4041  *      @trailer: Returned pointer to the skb where the @tailbits space begins
4042  *
4043  *      Make sure that the data buffers attached to a socket buffer are
4044  *      writable. If they are not, private copies are made of the data buffers
4045  *      and the socket buffer is set to use these instead.
4046  *
4047  *      If @tailbits is given, make sure that there is space to write @tailbits
4048  *      bytes of data beyond current end of socket buffer.  @trailer will be
4049  *      set to point to the skb in which this space begins.
4050  *
4051  *      The number of scatterlist elements required to completely map the
4052  *      COW'd and extended socket buffer will be returned.
4053  */
4054 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
4055 {
4056         int copyflag;
4057         int elt;
4058         struct sk_buff *skb1, **skb_p;
4059 
4060         /* If skb is cloned or its head is paged, reallocate
4061          * head pulling out all the pages (pages are considered not writable
4062          * at the moment even if they are anonymous).
4063          */
4064         if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
4065             __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
4066                 return -ENOMEM;
4067 
4068         /* Easy case. Most of packets will go this way. */
4069         if (!skb_has_frag_list(skb)) {
4070                 /* A little of trouble, not enough of space for trailer.
4071                  * This should not happen, when stack is tuned to generate
4072                  * good frames. OK, on miss we reallocate and reserve even more
4073                  * space, 128 bytes is fair. */
4074 
4075                 if (skb_tailroom(skb) < tailbits &&
4076                     pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
4077                         return -ENOMEM;
4078 
4079                 /* Voila! */
4080                 *trailer = skb;
4081                 return 1;
4082         }
4083 
4084         /* Misery. We are in troubles, going to mincer fragments... */
4085 
4086         elt = 1;
4087         skb_p = &skb_shinfo(skb)->frag_list;
4088         copyflag = 0;
4089 
4090         while ((skb1 = *skb_p) != NULL) {
4091                 int ntail = 0;
4092 
4093                 /* The fragment is partially pulled by someone,
4094                  * this can happen on input. Copy it and everything
4095                  * after it. */
4096 
4097                 if (skb_shared(skb1))
4098                         copyflag = 1;
4099 
4100                 /* If the skb is the last, worry about trailer. */
4101 
4102                 if (skb1->next == NULL && tailbits) {
4103                         if (skb_shinfo(skb1)->nr_frags ||
4104                             skb_has_frag_list(skb1) ||
4105                             skb_tailroom(skb1) < tailbits)
4106                                 ntail = tailbits + 128;
4107                 }
4108 
4109                 if (copyflag ||
4110                     skb_cloned(skb1) ||
4111                     ntail ||
4112                     skb_shinfo(skb1)->nr_frags ||
4113                     skb_has_frag_list(skb1)) {
4114                         struct sk_buff *skb2;
4115 
4116                         /* Fuck, we are miserable poor guys... */
4117                         if (ntail == 0)
4118                                 skb2 = skb_copy(skb1, GFP_ATOMIC);
4119                         else
4120                                 skb2 = skb_copy_expand(skb1,
4121                                                        skb_headroom(skb1),
4122                                                        ntail,
4123                                                        GFP_ATOMIC);
4124                         if (unlikely(skb2 == NULL))
4125                                 return -ENOMEM;
4126 
4127                         if (skb1->sk)
4128                                 skb_set_owner_w(skb2, skb1->sk);
4129 
4130                         /* Looking around. Are we still alive?
4131                          * OK, link new skb, drop old one */
4132 
4133                         skb2->next = skb1->next;
4134                         *skb_p = skb2;
4135                         kfree_skb(skb1);
4136                         skb1 = skb2;
4137                 }
4138                 elt++;
4139                 *trailer = skb1;
4140                 skb_p = &skb1->next;
4141         }
4142 
4143         return elt;
4144 }
4145 EXPORT_SYMBOL_GPL(skb_cow_data);
4146 
4147 static void sock_rmem_free(struct sk_buff *skb)
4148 {
4149         struct sock *sk = skb->sk;
4150 
4151         atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
4152 }
4153 
4154 static void skb_set_err_queue(struct sk_buff *skb)
4155 {
4156         /* pkt_type of skbs received on local sockets is never PACKET_OUTGOING.
4157          * So, it is safe to (mis)use it to mark skbs on the error queue.
4158          */
4159         skb->pkt_type = PACKET_OUTGOING;
4160         BUILD_BUG_ON(PACKET_OUTGOING == 0);
4161 }
4162 
4163 /*
4164  * Note: We dont mem charge error packets (no sk_forward_alloc changes)
4165  */
4166 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
4167 {
4168         if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
4169             (unsigned int)sk->sk_rcvbuf)
4170                 return -ENOMEM;
4171 
4172         skb_orphan(skb);
4173         skb->sk = sk;
4174         skb->destructor = sock_rmem_free;
4175         atomic_add(skb->truesize, &sk->sk_rmem_alloc);
4176         skb_set_err_queue(skb);
4177 
4178         /* before exiting rcu section, make sure dst is refcounted */
4179         skb_dst_force(skb);
4180 
4181         skb_queue_tail(&sk->sk_error_queue, skb);
4182         if (!sock_flag(sk, SOCK_DEAD))
4183                 sk->sk_error_report(sk);
4184         return 0;
4185 }
4186 EXPORT_SYMBOL(sock_queue_err_skb);
4187 
4188 static bool is_icmp_err_skb(const struct sk_buff *skb)
4189 {
4190         return skb && (SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP ||
4191                        SKB_EXT_ERR(skb)->ee.ee_origin == SO_EE_ORIGIN_ICMP6);
4192 }
4193 
4194 struct sk_buff *sock_dequeue_err_skb(struct sock *sk)
4195 {
4196         struct sk_buff_head *q = &sk->sk_error_queue;
4197         struct sk_buff *skb, *skb_next = NULL;
4198         bool icmp_next = false;
4199         unsigned long flags;
4200 
4201         spin_lock_irqsave(&q->lock, flags);
4202         skb = __skb_dequeue(q);
4203         if (skb && (skb_next = skb_peek(q))) {
4204                 icmp_next = is_icmp_err_skb(skb_next);
4205                 if (icmp_next)
4206                         sk->sk_err = SKB_EXT_ERR(skb_next)->ee.ee_origin;
4207         }
4208         spin_unlock_irqrestore(&q->lock, flags);
4209 
4210         if (is_icmp_err_skb(skb) && !icmp_next)
4211                 sk->sk_err = 0;
4212 
4213         if (skb_next)
4214                 sk->sk_error_report(sk);
4215 
4216         return skb;
4217 }
4218 EXPORT_SYMBOL(sock_dequeue_err_skb);
4219 
4220 /**
4221  * skb_clone_sk - create clone of skb, and take reference to socket
4222  * @skb: the skb to clone
4223  *
4224  * This function creates a clone of a buffer that holds a reference on
4225  * sk_refcnt.  Buffers created via this function are meant to be
4226  * returned using sock_queue_err_skb, or free via kfree_skb.
4227  *
4228  * When passing buffers allocated with this function to sock_queue_err_skb
4229  * it is necessary to wrap the call with sock_hold/sock_put in order to
4230  * prevent the socket from being released prior to being enqueued on
4231  * the sk_error_queue.
4232  */
4233 struct sk_buff *skb_clone_sk(struct sk_buff *skb)
4234 {
4235         struct sock *sk = skb->sk;
4236         struct sk_buff *clone;
4237 
4238         if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
4239                 return NULL;
4240 
4241         clone = skb_clone(skb, GFP_ATOMIC);
4242         if (!clone) {
4243                 sock_put(sk);
4244                 return NULL;
4245         }
4246 
4247         clone->sk = sk;
4248         clone->destructor = sock_efree;
4249 
4250         return clone;
4251 }
4252 EXPORT_SYMBOL(skb_clone_sk);
4253 
4254 static void __skb_complete_tx_timestamp(struct sk_buff *skb,
4255                                         struct sock *sk,
4256                                         int tstype,
4257                                         bool opt_stats)
4258 {
4259         struct sock_exterr_skb *serr;
4260         int err;
4261 
4262         BUILD_BUG_ON(sizeof(struct sock_exterr_skb) > sizeof(skb->cb));
4263 
4264         serr = SKB_EXT_ERR(skb);
4265         memset(serr, 0, sizeof(*serr));
4266         serr->ee.ee_errno = ENOMSG;
4267         serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
4268         serr->ee.ee_info = tstype;
4269         serr->opt_stats = opt_stats;
4270         serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0;
4271         if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) {
4272                 serr->ee.ee_data = skb_shinfo(skb)->tskey;
4273                 if (sk->sk_protocol == IPPROTO_TCP &&
4274                     sk->sk_type == SOCK_STREAM)
4275                         serr->ee.ee_data -= sk->sk_tskey;
4276         }
4277 
4278         err = sock_queue_err_skb(sk, skb);
4279 
4280         if (err)
4281                 kfree_skb(skb);
4282 }
4283 
4284 static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly)
4285 {
4286         bool ret;
4287 
4288         if (likely(sysctl_tstamp_allow_data || tsonly))
4289                 return true;
4290 
4291         read_lock_bh(&sk->sk_callback_lock);
4292         ret = sk->sk_socket && sk->sk_socket->file &&
4293               file_ns_capable(sk->sk_socket->file, &init_user_ns, CAP_NET_RAW);
4294         read_unlock_bh(&sk->sk_callback_lock);
4295         return ret;
4296 }
4297 
4298 void skb_complete_tx_timestamp(struct sk_buff *skb,
4299                                struct skb_shared_hwtstamps *hwtstamps)
4300 {
4301         struct sock *sk = skb->sk;
4302 
4303         if (!skb_may_tx_timestamp(sk, false))
4304                 goto err;
4305 
4306         /* Take a reference to prevent skb_orphan() from freeing the socket,
4307          * but only if the socket refcount is not zero.
4308          */
4309         if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
4310                 *skb_hwtstamps(skb) = *hwtstamps;
4311                 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND, false);
4312                 sock_put(sk);
4313                 return;
4314         }
4315 
4316 err:
4317         kfree_skb(skb);
4318 }
4319 EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp);
4320 
4321 void __skb_tstamp_tx(struct sk_buff *orig_skb,
4322                      struct skb_shared_hwtstamps *hwtstamps,
4323                      struct sock *sk, int tstype)
4324 {
4325         struct sk_buff *skb;
4326         bool tsonly, opt_stats = false;
4327 
4328         if (!sk)
4329                 return;
4330 
4331         if (!hwtstamps && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TX_SWHW) &&
4332             skb_shinfo(orig_skb)->tx_flags & SKBTX_IN_PROGRESS)
4333                 return;
4334 
4335         tsonly = sk->sk_tsflags & SOF_TIMESTAMPING_OPT_TSONLY;
4336         if (!skb_may_tx_timestamp(sk, tsonly))
4337                 return;
4338 
4339         if (tsonly) {
4340 #ifdef CONFIG_INET
4341                 if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) &&
4342                     sk->sk_protocol == IPPROTO_TCP &&
4343                     sk->sk_type == SOCK_STREAM) {
4344                         skb = tcp_get_timestamping_opt_stats(sk);
4345                         opt_stats = true;
4346                 } else
4347 #endif
4348                         skb = alloc_skb(0, GFP_ATOMIC);
4349         } else {
4350                 skb = skb_clone(orig_skb, GFP_ATOMIC);
4351         }
4352         if (!skb)
4353                 return;
4354 
4355         if (tsonly) {
4356                 skb_shinfo(skb)->tx_flags |= skb_shinfo(orig_skb)->tx_flags &
4357                                              SKBTX_ANY_TSTAMP;
4358                 skb_shinfo(skb)->tskey = skb_shinfo(orig_skb)->tskey;
4359         }
4360 
4361         if (hwtstamps)
4362                 *skb_hwtstamps(skb) = *hwtstamps;
4363         else
4364                 skb->tstamp = ktime_get_real();
4365 
4366         __skb_complete_tx_timestamp(skb, sk, tstype, opt_stats);
4367 }
4368 EXPORT_SYMBOL_GPL(__skb_tstamp_tx);
4369 
4370 void skb_tstamp_tx(struct sk_buff *orig_skb,
4371                    struct skb_shared_hwtstamps *hwtstamps)
4372 {
4373         return __skb_tstamp_tx(orig_skb, hwtstamps, orig_skb->sk,
4374                                SCM_TSTAMP_SND);
4375 }
4376 EXPORT_SYMBOL_GPL(skb_tstamp_tx);
4377 
4378 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)
4379 {
4380         struct sock *sk = skb->sk;
4381         struct sock_exterr_skb *serr;
4382         int err = 1;
4383 
4384         skb->wifi_acked_valid = 1;
4385         skb->wifi_acked = acked;
4386 
4387         serr = SKB_EXT_ERR(skb);
4388         memset(serr, 0, sizeof(*serr));
4389         serr->ee.ee_errno = ENOMSG;
4390         serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS;
4391 
4392         /* Take a reference to prevent skb_orphan() from freeing the socket,
4393          * but only if the socket refcount is not zero.
4394          */
4395         if (likely(refcount_inc_not_zero(&sk->sk_refcnt))) {
4396                 err = sock_queue_err_skb(sk, skb);
4397                 sock_put(sk);
4398         }
4399         if (err)
4400                 kfree_skb(skb);
4401 }
4402 EXPORT_SYMBOL_GPL(skb_complete_wifi_ack);
4403 
4404 /**
4405  * skb_partial_csum_set - set up and verify partial csum values for packet
4406  * @skb: the skb to set
4407  * @start: the number of bytes after skb->data to start checksumming.
4408  * @off: the offset from start to place the checksum.
4409  *
4410  * For untrusted partially-checksummed packets, we need to make sure the values
4411  * for skb->csum_start and skb->csum_offset are valid so we don't oops.
4412  *
4413  * This function checks and sets those values and skb->ip_summed: if this
4414  * returns false you should drop the packet.
4415  */
4416 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
4417 {
4418         if (unlikely(start > skb_headlen(skb)) ||
4419             unlikely((int)start + off > skb_headlen(skb) - 2)) {
4420                 net_warn_ratelimited("bad partial csum: csum=%u/%u len=%u\n",
4421                                      start, off, skb_headlen(skb));
4422                 return false;
4423         }
4424         skb->ip_summed = CHECKSUM_PARTIAL;
4425         skb->csum_start = skb_headroom(skb) + start;
4426         skb->csum_offset = off;
4427         skb_set_transport_header(skb, start);
4428         return true;
4429 }
4430 EXPORT_SYMBOL_GPL(skb_partial_csum_set);
4431 
4432 static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len,
4433                                unsigned int max)
4434 {
4435         if (skb_headlen(skb) >= len)
4436                 return 0;
4437 
4438         /* If we need to pullup then pullup to the max, so we
4439          * won't need to do it again.
4440          */
4441         if (max > skb->len)
4442                 max = skb->len;
4443 
4444         if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL)
4445                 return -ENOMEM;
4446 
4447         if (skb_headlen(skb) < len)
4448                 return -EPROTO;
4449 
4450         return 0;
4451 }
4452 
4453 #define MAX_TCP_HDR_LEN (15 * 4)
4454 
4455 static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb,
4456                                       typeof(IPPROTO_IP) proto,
4457                                       unsigned int off)
4458 {
4459         switch (proto) {
4460                 int err;
4461 
4462         case IPPROTO_TCP:
4463                 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
4464                                           off + MAX_TCP_HDR_LEN);
4465                 if (!err && !skb_partial_csum_set(skb, off,
4466                                                   offsetof(struct tcphdr,
4467                                                            check)))
4468                         err = -EPROTO;
4469                 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check;
4470 
4471         case IPPROTO_UDP:
4472                 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr),
4473                                           off + sizeof(struct udphdr));
4474                 if (!err && !skb_partial_csum_set(skb, off,
4475                                                   offsetof(struct udphdr,
4476                                                            check)))
4477                         err = -EPROTO;
4478                 return err ? ERR_PTR(err) : &udp_hdr(skb)->check;
4479         }
4480 
4481         return ERR_PTR(-EPROTO);
4482 }
4483 
4484 /* This value should be large enough to cover a tagged ethernet header plus
4485  * maximally sized IP and TCP or UDP headers.
4486  */
4487 #define MAX_IP_HDR_LEN 128
4488 
4489 static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate)
4490 {
4491         unsigned int off;
4492         bool fragment;
4493         __sum16 *csum;
4494         int err;
4495 
4496         fragment = false;
4497 
4498         err = skb_maybe_pull_tail(skb,
4499                                   sizeof(struct iphdr),
4500                                   MAX_IP_HDR_LEN);
4501         if (err < 0)
4502                 goto out;
4503 
4504         if (ip_hdr(skb)->frag_off & htons(IP_OFFSET | IP_MF))
4505                 fragment = true;
4506 
4507         off = ip_hdrlen(skb);
4508 
4509         err = -EPROTO;
4510 
4511         if (fragment)
4512                 goto out;
4513 
4514         csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off);
4515         if (IS_ERR(csum))
4516                 return PTR_ERR(csum);
4517 
4518         if (recalculate)
4519                 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
4520                                            ip_hdr(skb)->daddr,
4521                                            skb->len - off,
4522                                            ip_hdr(skb)->protocol, 0);
4523         err = 0;
4524 
4525 out:
4526         return err;
4527 }
4528 
4529 /* This value should be large enough to cover a tagged ethernet header plus
4530  * an IPv6 header, all options, and a maximal TCP or UDP header.
4531  */
4532 #define MAX_IPV6_HDR_LEN 256
4533 
4534 #define OPT_HDR(type, skb, off) \
4535         (type *)(skb_network_header(skb) + (off))
4536 
4537 static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate)
4538 {
4539         int err;
4540         u8 nexthdr;
4541         unsigned int off;
4542         unsigned int len;
4543         bool fragment;
4544         bool done;
4545         __sum16 *csum;
4546 
4547         fragment = false;
4548         done = false;
4549 
4550         off = sizeof(struct ipv6hdr);
4551 
4552         err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN);
4553         if (err < 0)
4554                 goto out;
4555 
4556         nexthdr = ipv6_hdr(skb)->nexthdr;
4557 
4558         len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len);
4559         while (off <= len && !done) {
4560                 switch (nexthdr) {
4561                 case IPPROTO_DSTOPTS:
4562                 case IPPROTO_HOPOPTS:
4563                 case IPPROTO_ROUTING: {
4564                         struct ipv6_opt_hdr *hp;
4565 
4566                         err = skb_maybe_pull_tail(skb,
4567                                                   off +
4568                                                   sizeof(struct ipv6_opt_hdr),
4569                                                   MAX_IPV6_HDR_LEN);
4570                         if (err < 0)
4571                                 goto out;
4572 
4573                         hp = OPT_HDR(struct ipv6_opt_hdr, skb, off);
4574                         nexthdr = hp->nexthdr;
4575                         off += ipv6_optlen(hp);
4576                         break;
4577                 }
4578                 case IPPROTO_AH: {
4579                         struct ip_auth_hdr *hp;
4580 
4581                         err = skb_maybe_pull_tail(skb,
4582                                                   off +
4583                                                   sizeof(struct ip_auth_hdr),
4584                                                   MAX_IPV6_HDR_LEN);
4585                         if (err < 0)
4586                                 goto out;
4587 
4588                         hp = OPT_HDR(struct ip_auth_hdr, skb, off);
4589                         nexthdr = hp->nexthdr;
4590                         off += ipv6_authlen(hp);
4591                         break;
4592                 }
4593                 case IPPROTO_FRAGMENT: {
4594                         struct frag_hdr *hp;
4595 
4596                         err = skb_maybe_pull_tail(skb,
4597                                                   off +
4598                                                   sizeof(struct frag_hdr),
4599                                                   MAX_IPV6_HDR_LEN);
4600                         if (err < 0)
4601                                 goto out;
4602 
4603                         hp = OPT_HDR(struct frag_hdr, skb, off);
4604 
4605                         if (hp->frag_off & htons(IP6_OFFSET | IP6_MF))
4606                                 fragment = true;
4607 
4608                         nexthdr = hp->nexthdr;
4609                         off += sizeof(struct frag_hdr);
4610                         break;
4611                 }
4612                 default:
4613                         done = true;
4614                         break;
4615                 }
4616         }
4617 
4618         err = -EPROTO;
4619 
4620         if (!done || fragment)
4621                 goto out;
4622 
4623         csum = skb_checksum_setup_ip(skb, nexthdr, off);
4624         if (IS_ERR(csum))
4625                 return PTR_ERR(csum);
4626 
4627         if (recalculate)
4628                 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4629                                          &ipv6_hdr(skb)->daddr,
4630                                          skb->len - off, nexthdr, 0);
4631         err = 0;
4632 
4633 out:
4634         return err;
4635 }
4636 
4637 /**
4638  * skb_checksum_setup - set up partial checksum offset
4639  * @skb: the skb to set up
4640  * @recalculate: if true the pseudo-header checksum will be recalculated
4641  */
4642 int skb_checksum_setup(struct sk_buff *skb, bool recalculate)
4643 {
4644         int err;
4645 
4646         switch (skb->protocol) {
4647         case htons(ETH_P_IP):
4648                 err = skb_checksum_setup_ipv4(skb, recalculate);
4649                 break;
4650 
4651         case htons(ETH_P_IPV6):
4652                 err = skb_checksum_setup_ipv6(skb, recalculate);
4653                 break;
4654 
4655         default:
4656                 err = -EPROTO;
4657                 break;
4658         }
4659 
4660         return err;
4661 }
4662 EXPORT_SYMBOL(skb_checksum_setup);
4663 
4664 /**
4665  * skb_checksum_maybe_trim - maybe trims the given skb
4666  * @skb: the skb to check
4667  * @transport_len: the data length beyond the network header
4668  *
4669  * Checks whether the given skb has data beyond the given transport length.
4670  * If so, returns a cloned skb trimmed to this transport length.
4671  * Otherwise returns the provided skb. Returns NULL in error cases
4672  * (e.g. transport_len exceeds skb length or out-of-memory).
4673  *
4674  * Caller needs to set the skb transport header and free any returned skb if it
4675  * differs from the provided skb.
4676  */
4677 static struct sk_buff *skb_checksum_maybe_trim(struct sk_buff *skb,
4678                                                unsigned int transport_len)
4679 {
4680         struct sk_buff *skb_chk;
4681         unsigned int len = skb_transport_offset(skb) + transport_len;
4682         int ret;
4683 
4684         if (skb->len < len)
4685                 return NULL;
4686         else if (skb->len == len)
4687                 return skb;
4688 
4689         skb_chk = skb_clone(skb, GFP_ATOMIC);
4690         if (!skb_chk)
4691                 return NULL;
4692 
4693         ret = pskb_trim_rcsum(skb_chk, len);
4694         if (ret) {
4695                 kfree_skb(skb_chk);
4696                 return NULL;
4697         }
4698 
4699         return skb_chk;
4700 }
4701 
4702 /**
4703  * skb_checksum_trimmed - validate checksum of an skb
4704  * @skb: the skb to check
4705  * @transport_len: the data length beyond the network header
4706  * @skb_chkf: checksum function to use
4707  *
4708  * Applies the given checksum function skb_chkf to the provided skb.
4709  * Returns a checked and maybe trimmed skb. Returns NULL on error.
4710  *
4711  * If the skb has data beyond the given transport length, then a
4712  * trimmed & cloned skb is checked and returned.
4713  *
4714  * Caller needs to set the skb transport header and free any returned skb if it
4715  * differs from the provided skb.
4716  */
4717 struct sk_buff *skb_checksum_trimmed(struct sk_buff *skb,
4718                                      unsigned int transport_len,
4719                                      __sum16(*skb_chkf)(struct sk_buff *skb))
4720 {
4721         struct sk_buff *skb_chk;
4722         unsigned int offset = skb_transport_offset(skb);
4723         __sum16 ret;
4724 
4725         skb_chk = skb_checksum_maybe_trim(skb, transport_len);
4726         if (!skb_chk)
4727                 goto err;
4728 
4729         if (!pskb_may_pull(skb_chk, offset))
4730                 goto err;
4731 
4732         skb_pull_rcsum(skb_chk, offset);
4733         ret = skb_chkf(skb_chk);
4734         skb_push_rcsum(skb_chk, offset);
4735 
4736         if (ret)
4737                 goto err;
4738 
4739         return skb_chk;
4740 
4741 err:
4742         if (skb_chk && skb_chk != skb)
4743                 kfree_skb(skb_chk);
4744 
4745         return NULL;
4746 
4747 }
4748 EXPORT_SYMBOL(skb_checksum_trimmed);
4749 
4750 void __skb_warn_lro_forwarding(const struct sk_buff *skb)
4751 {
4752         net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n",
4753                              skb->dev->name);
4754 }
4755 EXPORT_SYMBOL(__skb_warn_lro_forwarding);
4756 
4757 void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
4758 {
4759         if (head_stolen) {
4760                 skb_release_head_state(skb);
4761                 kmem_cache_free(skbuff_head_cache, skb);
4762         } else {
4763                 __kfree_skb(skb);
4764         }
4765 }
4766 EXPORT_SYMBOL(kfree_skb_partial);
4767 
4768 /**
4769  * skb_try_coalesce - try to merge skb to prior one
4770  * @to: prior buffer
4771  * @from: buffer to add
4772  * @fragstolen: pointer to boolean
4773  * @delta_truesize: how much more was allocated than was requested
4774  */
4775 bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
4776                       bool *fragstolen, int *delta_truesize)
4777 {
4778         struct skb_shared_info *to_shinfo, *from_shinfo;
4779         int i, delta, len = from->len;
4780 
4781         *fragstolen = false;
4782 
4783         if (skb_cloned(to))
4784                 return false;
4785 
4786         if (len <= skb_tailroom(to)) {
4787                 if (len)
4788                         BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len));
4789                 *delta_truesize = 0;
4790                 return true;
4791         }
4792 
4793         to_shinfo = skb_shinfo(to);
4794         from_shinfo = skb_shinfo(from);
4795         if (to_shinfo->frag_list || from_shinfo->frag_list)
4796                 return false;
4797         if (skb_zcopy(to) || skb_zcopy(from))
4798                 return false;
4799 
4800         if (skb_headlen(from) != 0) {
4801                 struct page *page;
4802                 unsigned int offset;
4803 
4804                 if (to_shinfo->nr_frags +
4805                     from_shinfo->nr_frags >= MAX_SKB_FRAGS)
4806                         return false;
4807 
4808                 if (skb_head_is_locked(from))
4809                         return false;
4810 
4811                 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
4812 
4813                 page = virt_to_head_page(from->head);
4814                 offset = from->data - (unsigned char *)page_address(page);
4815 
4816                 skb_fill_page_desc(to, to_shinfo->nr_frags,
4817                                    page, offset, skb_headlen(from));
4818                 *fragstolen = true;
4819         } else {
4820                 if (to_shinfo->nr_frags +
4821                     from_shinfo->nr_frags > MAX_SKB_FRAGS)
4822                         return false;
4823 
4824                 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from));
4825         }
4826 
4827         WARN_ON_ONCE(delta < len);
4828 
4829         memcpy(to_shinfo->frags + to_shinfo->nr_frags,
4830                from_shinfo->frags,
4831                from_shinfo->nr_frags * sizeof(skb_frag_t));
4832         to_shinfo->nr_frags += from_shinfo->nr_frags;
4833 
4834         if (!skb_cloned(from))
4835                 from_shinfo->nr_frags = 0;
4836 
4837         /* if the skb is not cloned this does nothing
4838          * since we set nr_frags to 0.
4839          */
4840         for (i = 0; i < from_shinfo->nr_frags; i++)
4841                 __skb_frag_ref(&from_shinfo->frags[i]);
4842 
4843         to->truesize += delta;
4844         to->len += len;
4845         to->data_len += len;
4846 
4847         *delta_truesize = delta;
4848         return true;
4849 }
4850 EXPORT_SYMBOL(skb_try_coalesce);
4851 
4852 /**
4853  * skb_scrub_packet - scrub an skb
4854  *
4855  * @skb: buffer to clean
4856  * @xnet: packet is crossing netns
4857  *
4858  * skb_scrub_packet can be used after encapsulating or decapsulting a packet
4859  * into/from a tunnel. Some information have to be cleared during these
4860  * operations.
4861  * skb_scrub_packet can also be used to clean a skb before injecting it in
4862  * another namespace (@xnet == true). We have to clear all information in the
4863  * skb that could impact namespace isolation.
4864  */
4865 void skb_scrub_packet(struct sk_buff *skb, bool xnet)
4866 {
4867         skb->tstamp = 0;
4868         skb->pkt_type = PACKET_HOST;
4869         skb->skb_iif = 0;
4870         skb->ignore_df = 0;
4871         skb_dst_drop(skb);
4872         secpath_reset(skb);
4873         nf_reset(skb);
4874         nf_reset_trace(skb);
4875 
4876         if (!xnet)
4877                 return;
4878 
4879         ipvs_reset(skb);
4880         skb_orphan(skb);
4881         skb->mark = 0;
4882 }
4883 EXPORT_SYMBOL_GPL(skb_scrub_packet);
4884 
4885 /**
4886  * skb_gso_transport_seglen - Return length of individual segments of a gso packet
4887  *
4888  * @skb: GSO skb
4889  *
4890  * skb_gso_transport_seglen is used to determine the real size of the
4891  * individual segments, including Layer4 headers (TCP/UDP).
4892  *
4893  * The MAC/L2 or network (IP, IPv6) headers are not accounted for.
4894  */
4895 static unsigned int skb_gso_transport_seglen(const struct sk_buff *skb)
4896 {
4897         const struct skb_shared_info *shinfo = skb_shinfo(skb);
4898         unsigned int thlen = 0;
4899 
4900         if (skb->encapsulation) {
4901                 thlen = skb_inner_transport_header(skb) -
4902                         skb_transport_header(skb);
4903 
4904                 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
4905                         thlen += inner_tcp_hdrlen(skb);
4906         } else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
4907                 thlen = tcp_hdrlen(skb);
4908         } else if (unlikely(skb_is_gso_sctp(skb))) {
4909                 thlen = sizeof(struct sctphdr);
4910         }
4911         /* UFO sets gso_size to the size of the fragmentation
4912          * payload, i.e. the size of the L4 (UDP) header is already
4913          * accounted for.
4914          */
4915         return thlen + shinfo->gso_size;
4916 }
4917 
4918 /**
4919  * skb_gso_network_seglen - Return length of individual segments of a gso packet
4920  *
4921  * @skb: GSO skb
4922  *
4923  * skb_gso_network_seglen is used to determine the real size of the
4924  * individual segments, including Layer3 (IP, IPv6) and L4 headers (TCP/UDP).
4925  *
4926  * The MAC/L2 header is not accounted for.
4927  */
4928 static unsigned int skb_gso_network_seglen(const struct sk_buff *skb)
4929 {
4930         unsigned int hdr_len = skb_transport_header(skb) -
4931                                skb_network_header(skb);
4932 
4933         return hdr_len + skb_gso_transport_seglen(skb);
4934 }
4935 
4936 /**
4937  * skb_gso_mac_seglen - Return length of individual segments of a gso packet
4938  *
4939  * @skb: GSO skb
4940  *
4941  * skb_gso_mac_seglen is used to determine the real size of the
4942  * individual segments, including MAC/L2, Layer3 (IP, IPv6) and L4
4943  * headers (TCP/UDP).
4944  */
4945 static unsigned int skb_gso_mac_seglen(const struct sk_buff *skb)
4946 {
4947         unsigned int hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
4948 
4949         return hdr_len + skb_gso_transport_seglen(skb);
4950 }
4951 
4952 /**
4953  * skb_gso_size_check - check the skb size, considering GSO_BY_FRAGS
4954  *
4955  * There are a couple of instances where we have a GSO skb, and we
4956  * want to determine what size it would be after it is segmented.
4957  *
4958  * We might want to check:
4959  * -    L3+L4+payload size (e.g. IP forwarding)
4960  * - L2+L3+L4+payload size (e.g. sanity check before passing to driver)
4961  *
4962  * This is a helper to do that correctly considering GSO_BY_FRAGS.
4963  *
4964  * @seg_len: The segmented length (from skb_gso_*_seglen). In the
4965  *           GSO_BY_FRAGS case this will be [header sizes + GSO_BY_FRAGS].
4966  *
4967  * @max_len: The maximum permissible length.
4968  *
4969  * Returns true if the segmented length <= max length.
4970  */
4971 static inline bool skb_gso_size_check(const struct sk_buff *skb,
4972                                       unsigned int seg_len,
4973                                       unsigned int max_len) {
4974         const struct skb_shared_info *shinfo = skb_shinfo(skb);
4975         const struct sk_buff *iter;
4976 
4977         if (shinfo->gso_size != GSO_BY_FRAGS)
4978                 return seg_len <= max_len;
4979 
4980         /* Undo this so we can re-use header sizes */
4981         seg_len -= GSO_BY_FRAGS;
4982 
4983         skb_walk_frags(skb, iter) {
4984                 if (seg_len + skb_headlen(iter) > max_len)
4985                         return false;
4986         }
4987 
4988         return true;
4989 }
4990 
4991 /**
4992  * skb_gso_validate_network_len - Will a split GSO skb fit into a given MTU?
4993  *
4994  * @skb: GSO skb
4995  * @mtu: MTU to validate against
4996  *
4997  * skb_gso_validate_network_len validates if a given skb will fit a
4998  * wanted MTU once split. It considers L3 headers, L4 headers, and the
4999  * payload.
5000  */
5001 bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu)
5002 {
5003         return skb_gso_size_check(skb, skb_gso_network_seglen(skb), mtu);
5004 }
5005 EXPORT_SYMBOL_GPL(skb_gso_validate_network_len);
5006 
5007 /**
5008  * skb_gso_validate_mac_len - Will a split GSO skb fit in a given length?
5009  *
5010  * @skb: GSO skb
5011  * @len: length to validate against
5012  *
5013  * skb_gso_validate_mac_len validates if a given skb will fit a wanted
5014  * length once split, including L2, L3 and L4 headers and the payload.
5015  */
5016 bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len)
5017 {
5018         return skb_gso_size_check(skb, skb_gso_mac_seglen(skb), len);
5019 }
5020 EXPORT_SYMBOL_GPL(skb_gso_validate_mac_len);
5021 
5022 static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb)
5023 {
5024         int mac_len;
5025 
5026         if (skb_cow(skb, skb_headroom(skb)) < 0) {
5027                 kfree_skb(skb);
5028                 return NULL;
5029         }
5030 
5031         mac_len = skb->data - skb_mac_header(skb);
5032         if (likely(mac_len > VLAN_HLEN + ETH_TLEN)) {
5033                 memmove(skb_mac_header(skb) + VLAN_HLEN, skb_mac_header(skb),
5034                         mac_len - VLAN_HLEN - ETH_TLEN);
5035         }
5036         skb->mac_header += VLAN_HLEN;
5037         return skb;
5038 }
5039 
5040 struct sk_buff *skb_vlan_untag(struct sk_buff *skb)
5041 {
5042         struct vlan_hdr *vhdr;
5043         u16 vlan_tci;
5044 
5045         if (unlikely(skb_vlan_tag_present(skb))) {
5046                 /* vlan_tci is already set-up so leave this for another time */
5047                 return skb;
5048         }
5049 
5050         skb = skb_share_check(skb, GFP_ATOMIC);
5051         if (unlikely(!skb))
5052                 goto err_free;
5053 
5054         if (unlikely(!pskb_may_pull(skb, VLAN_HLEN)))
5055                 goto err_free;
5056 
5057         vhdr = (struct vlan_hdr *)skb->data;
5058         vlan_tci = ntohs(vhdr->h_vlan_TCI);
5059         __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci);
5060 
5061         skb_pull_rcsum(skb, VLAN_HLEN);
5062         vlan_set_encap_proto(skb, vhdr);
5063 
5064         skb = skb_reorder_vlan_header(skb);
5065         if (unlikely(!skb))
5066                 goto err_free;
5067 
5068         skb_reset_network_header(skb);
5069         skb_reset_transport_header(skb);
5070         skb_reset_mac_len(skb);
5071 
5072         return skb;
5073 
5074 err_free:
5075         kfree_skb(skb);
5076         return NULL;
5077 }
5078 EXPORT_SYMBOL(skb_vlan_untag);
5079 
5080 int skb_ensure_writable(struct sk_buff *skb, int write_len)
5081 {
5082         if (!pskb_may_pull(skb, write_len))
5083                 return -ENOMEM;
5084 
5085         if (!skb_cloned(skb) || skb_clone_writable(skb, write_len))
5086                 return 0;
5087 
5088         return pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
5089 }
5090 EXPORT_SYMBOL(skb_ensure_writable);
5091 
5092 /* remove VLAN header from packet and update csum accordingly.
5093  * expects a non skb_vlan_tag_present skb with a vlan tag payload
5094  */
5095 int __skb_vlan_pop(struct sk_buff *skb, u16 *vlan_tci)
5096 {
5097         struct vlan_hdr *vhdr;
5098         int offset = skb->data - skb_mac_header(skb);
5099         int err;
5100 
5101         if (WARN_ONCE(offset,
5102                       "__skb_vlan_pop got skb with skb->data not at mac header (offset %d)\n",
5103                       offset)) {
5104                 return -EINVAL;
5105         }
5106 
5107         err = skb_ensure_writable(skb, VLAN_ETH_HLEN);
5108         if (unlikely(err))
5109                 return err;
5110 
5111         skb_postpull_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
5112 
5113         vhdr = (struct vlan_hdr *)(skb->data + ETH_HLEN);
5114         *vlan_tci = ntohs(vhdr->h_vlan_TCI);
5115 
5116         memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN);
5117         __skb_pull(skb, VLAN_HLEN);
5118 
5119         vlan_set_encap_proto(skb, vhdr);
5120         skb->mac_header += VLAN_HLEN;
5121 
5122         if (skb_network_offset(skb) < ETH_HLEN)
5123                 skb_set_network_header(skb, ETH_HLEN);
5124 
5125         skb_reset_mac_len(skb);
5126 
5127         return err;
5128 }
5129 EXPORT_SYMBOL(__skb_vlan_pop);
5130 
5131 /* Pop a vlan tag either from hwaccel or from payload.
5132  * Expects skb->data at mac header.
5133  */
5134 int skb_vlan_pop(struct sk_buff *skb)
5135 {
5136         u16 vlan_tci;
5137         __be16 vlan_proto;
5138         int err;
5139 
5140         if (likely(skb_vlan_tag_present(skb))) {
5141                 skb->vlan_tci = 0;
5142         } else {
5143                 if (unlikely(!eth_type_vlan(skb->protocol)))
5144                         return 0;
5145 
5146                 err = __skb_vlan_pop(skb, &vlan_tci);
5147                 if (err)
5148                         return err;
5149         }
5150         /* move next vlan tag to hw accel tag */
5151         if (likely(!eth_type_vlan(skb->protocol)))
5152                 return 0;
5153 
5154         vlan_proto = skb->protocol;
5155         err = __skb_vlan_pop(skb, &vlan_tci);
5156         if (unlikely(err))
5157                 return err;
5158 
5159         __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
5160         return 0;
5161 }
5162 EXPORT_SYMBOL(skb_vlan_pop);
5163 
5164 /* Push a vlan tag either into hwaccel or into payload (if hwaccel tag present).
5165  * Expects skb->data at mac header.
5166  */
5167 int skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci)
5168 {
5169         if (skb_vlan_tag_present(skb)) {
5170                 int offset = skb->data - skb_mac_header(skb);
5171                 int err;
5172 
5173                 if (WARN_ONCE(offset,
5174                               "skb_vlan_push got skb with skb->data not at mac header (offset %d)\n",
5175                               offset)) {
5176                         return -EINVAL;
5177                 }
5178 
5179                 err = __vlan_insert_tag(skb, skb->vlan_proto,
5180                                         skb_vlan_tag_get(skb));
5181                 if (err)
5182                         return err;
5183 
5184                 skb->protocol = skb->vlan_proto;
5185                 skb->mac_len += VLAN_HLEN;
5186 
5187                 skb_postpush_rcsum(skb, skb->data + (2 * ETH_ALEN), VLAN_HLEN);
5188         }
5189         __vlan_hwaccel_put_tag(skb, vlan_proto, vlan_tci);
5190         return 0;
5191 }
5192 EXPORT_SYMBOL(skb_vlan_push);
5193 
5194 /**
5195  * alloc_skb_with_frags - allocate skb with page frags
5196  *
5197  * @header_len: size of linear part
5198  * @data_len: needed length in frags
5199  * @max_page_order: max page order desired.
5200  * @errcode: pointer to error code if any
5201  * @gfp_mask: allocation mask
5202  *
5203  * This can be used to allocate a paged skb, given a maximal order for frags.
5204  */
5205 struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
5206                                      unsigned long data_len,
5207                                      int max_page_order,
5208                                      int *errcode,
5209                                      gfp_t gfp_mask)
5210 {
5211         int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
5212         unsigned long chunk;
5213         struct sk_buff *skb;
5214         struct page *page;
5215         gfp_t gfp_head;
5216         int i;
5217 
5218         *errcode = -EMSGSIZE;
5219         /* Note this test could be relaxed, if we succeed to allocate
5220          * high order pages...
5221          */
5222         if (npages > MAX_SKB_FRAGS)
5223                 return NULL;
5224 
5225         gfp_head = gfp_mask;
5226         if (gfp_head & __GFP_DIRECT_RECLAIM)
5227                 gfp_head |= __GFP_RETRY_MAYFAIL;
5228 
5229         *errcode = -ENOBUFS;
5230         skb = alloc_skb(header_len, gfp_head);
5231         if (!skb)
5232                 return NULL;
5233 
5234         skb->truesize += npages << PAGE_SHIFT;
5235 
5236         for (i = 0; npages > 0; i++) {
5237                 int order = max_page_order;
5238 
5239                 while (order) {
5240                         if (npages >= 1 << order) {
5241                                 page = alloc_pages((gfp_mask & ~__GFP_DIRECT_RECLAIM) |
5242                                                    __GFP_COMP |
5243                                                    __GFP_NOWARN |
5244                                                    __GFP_NORETRY,
5245                                                    order);
5246                                 if (page)
5247                                         goto fill_page;
5248                                 /* Do not retry other high order allocations */
5249                                 order = 1;
5250                                 max_page_order = 0;
5251                         }
5252                         order--;
5253                 }
5254                 page = alloc_page(gfp_mask);
5255                 if (!page)
5256                         goto failure;
5257 fill_page:
5258                 chunk = min_t(unsigned long, data_len,
5259                               PAGE_SIZE << order);
5260                 skb_fill_page_desc(skb, i, page, 0, chunk);
5261                 data_len -= chunk;
5262                 npages -= 1 << order;
5263         }
5264         return skb;
5265 
5266 failure:
5267         kfree_skb(skb);
5268         return NULL;
5269 }
5270 EXPORT_SYMBOL(alloc_skb_with_frags);
5271 
5272 /* carve out the first off bytes from skb when off < headlen */
5273 static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off,
5274                                     const int headlen, gfp_t gfp_mask)
5275 {
5276         int i;
5277         int size = skb_end_offset(skb);
5278         int new_hlen = headlen - off;
5279         u8 *data;
5280 
5281         size = SKB_DATA_ALIGN(size);
5282 
5283         if (skb_pfmemalloc(skb))
5284                 gfp_mask |= __GFP_MEMALLOC;
5285         data = kmalloc_reserve(size +
5286                                SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
5287                                gfp_mask, NUMA_NO_NODE, NULL);
5288         if (!data)
5289                 return -ENOMEM;
5290 
5291         size = SKB_WITH_OVERHEAD(ksize(data));
5292 
5293         /* Copy real data, and all frags */
5294         skb_copy_from_linear_data_offset(skb, off, data, new_hlen);
5295         skb->len -= off;
5296 
5297         memcpy((struct skb_shared_info *)(data + size),
5298                skb_shinfo(skb),
5299                offsetof(struct skb_shared_info,
5300                         frags[skb_shinfo(skb)->nr_frags]));
5301         if (skb_cloned(skb)) {
5302                 /* drop the old head gracefully */
5303                 if (skb_orphan_frags(skb, gfp_mask)) {
5304                         kfree(data);
5305                         return -ENOMEM;
5306                 }
5307                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
5308                         skb_frag_ref(skb, i);
5309                 if (skb_has_frag_list(skb))
5310                         skb_clone_fraglist(skb);
5311                 skb_release_data(skb);
5312         } else {
5313                 /* we can reuse existing recount- all we did was
5314                  * relocate values
5315                  */
5316                 skb_free_head(skb);
5317         }
5318 
5319         skb->head = data;
5320         skb->data = data;
5321         skb->head_frag = 0;
5322 #ifdef NET_SKBUFF_DATA_USES_OFFSET
5323         skb->end = size;
5324 #else
5325         skb->end = skb->head + size;
5326 #endif
5327         skb_set_tail_pointer(skb, skb_headlen(skb));
5328         skb_headers_offset_update(skb, 0);
5329         skb->cloned = 0;
5330         skb->hdr_len = 0;
5331         skb->nohdr = 0;
5332         atomic_set(&skb_shinfo(skb)->dataref, 1);
5333 
5334         return 0;
5335 }
5336 
5337 static int pskb_carve(struct sk_buff *skb, const u32 off, gfp_t gfp);
5338 
5339 /* carve out the first eat bytes from skb's frag_list. May recurse into
5340  * pskb_carve()
5341  */
5342 static int pskb_carve_frag_list(struct sk_buff *skb,
5343                                 struct skb_shared_info *shinfo, int eat,
5344                                 gfp_t gfp_mask)
5345 {
5346         struct sk_buff *list = shinfo->frag_list;
5347         struct sk_buff *clone = NULL;
5348         struct sk_buff *insp = NULL;
5349 
5350         do {
5351                 if (!list) {
5352                         pr_err("Not enough bytes to eat. Want %d\n", eat);
5353                         return -EFAULT;
5354                 }
5355                 if (list->len <= eat) {
5356                         /* Eaten as whole. */
5357                         eat -= list->len;
5358                         list = list->next;
5359                         insp = list;
5360                 } else {
5361                         /* Eaten partially. */
5362                         if (skb_shared(list)) {
5363                                 clone = skb_clone(list, gfp_mask);
5364                                 if (!clone)
5365                                         return -ENOMEM;
5366                                 insp = list->next;
5367                                 list = clone;
5368                         } else {
5369                                 /* This may be pulled without problems. */
5370                                 insp = list;
5371                         }
5372                         if (pskb_carve(list, eat, gfp_mask) < 0) {
5373                                 kfree_skb(clone);
5374                                 return -ENOMEM;
5375                         }
5376                         break;
5377                 }
5378         } while (eat);
5379 
5380         /* Free pulled out fragments. */
5381         while ((list = shinfo->frag_list) != insp) {
5382                 shinfo->frag_list = list->next;
5383                 kfree_skb(list);
5384         }
5385         /* And insert new clone at head. */
5386         if (clone) {
5387                 clone->next = list;
5388                 shinfo->frag_list = clone;
5389         }
5390         return 0;
5391 }
5392 
5393 /* carve off first len bytes from skb. Split line (off) is in the
5394  * non-linear part of skb
5395  */
5396 static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off,
5397                                        int pos, gfp_t gfp_mask)
5398 {
5399         int i, k = 0;
5400         int size = skb_end_offset(skb);
5401         u8 *data;
5402         const int nfrags = skb_shinfo(skb)->nr_frags;
5403         struct skb_shared_info *shinfo;
5404 
5405         size = SKB_DATA_ALIGN(size);
5406 
5407         if (skb_pfmemalloc(skb))
5408                 gfp_mask |= __GFP_MEMALLOC;
5409         data = kmalloc_reserve(size +
5410                                SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
5411                                gfp_mask, NUMA_NO_NODE, NULL);
5412         if (!data)
5413                 return -ENOMEM;
5414 
5415         size = SKB_WITH_OVERHEAD(ksize(data));
5416 
5417         memcpy((struct skb_shared_info *)(data + size),
5418                skb_shinfo(skb), offsetof(struct skb_shared_info,
5419                                          frags[skb_shinfo(skb)->nr_frags]));
5420         if (skb_orphan_frags(skb, gfp_mask)) {
5421                 kfree(data);
5422                 return -ENOMEM;
5423         }
5424         shinfo = (struct skb_shared_info *)(data + size);
5425         for (i = 0; i < nfrags; i++) {
5426                 int fsize = skb_frag_size(&skb_shinfo(skb)->frags[i]);
5427 
5428                 if (pos + fsize > off) {
5429                         shinfo->frags[k] = skb_shinfo(skb)->frags[i];
5430 
5431                         if (pos < off) {
5432                                 /* Split frag.
5433                                  * We have two variants in this case:
5434                                  * 1. Move all the frag to the second
5435                                  *    part, if it is possible. F.e.
5436                                  *    this approach is mandatory for TUX,
5437                                  *    where splitting is expensive.
5438                                  * 2. Split is accurately. We make this.
5439                                  */
5440                                 shinfo->frags[0].page_offset += off - pos;
5441                                 skb_frag_size_sub(&shinfo->frags[0], off - pos);
5442                         }
5443                         skb_frag_ref(skb, i);
5444                         k++;
5445                 }
5446                 pos += fsize;
5447         }
5448         shinfo->nr_frags = k;
5449         if (skb_has_frag_list(skb))
5450                 skb_clone_fraglist(skb);
5451 
5452         if (k == 0) {
5453                 /* split line is in frag list */
5454                 pskb_carve_frag_list(skb, shinfo, off - pos, gfp_mask);
5455         }
5456         skb_release_data(skb);
5457 
5458         skb->head = data;
5459         skb->head_frag = 0;
5460         skb->data = data;
5461 #ifdef NET_SKBUFF_DATA_USES_OFFSET
5462         skb->end = size;
5463 #else
5464         skb->end = skb->head + size;
5465 #endif
5466         skb_reset_tail_pointer(skb);
5467         skb_headers_offset_update(skb, 0);
5468         skb->cloned   = 0;
5469         skb->hdr_len  = 0;
5470         skb->nohdr    = 0;
5471         skb->len -= off;
5472         skb->data_len = skb->len;
5473         atomic_set(&skb_shinfo(skb)->dataref, 1);
5474         return 0;
5475 }
5476 
5477 /* remove len bytes from the beginning of the skb */
5478 static int pskb_carve(struct sk_buff *skb, const u32 len, gfp_t gfp)
5479 {
5480         int headlen = skb_headlen(skb);
5481 
5482         if (len < headlen)
5483                 return pskb_carve_inside_header(skb, len, headlen, gfp);
5484         else
5485                 return pskb_carve_inside_nonlinear(skb, len, headlen, gfp);
5486 }
5487 
5488 /* Extract to_copy bytes starting at off from skb, and return this in
5489  * a new skb
5490  */
5491 struct sk_buff *pskb_extract(struct sk_buff *skb, int off,
5492                              int to_copy, gfp_t gfp)
5493 {
5494         struct sk_buff  *clone = skb_clone(skb, gfp);
5495 
5496         if (!clone)
5497                 return NULL;
5498 
5499         if (pskb_carve(clone, off, gfp) < 0 ||
5500             pskb_trim(clone, to_copy)) {
5501                 kfree_skb(clone);
5502                 return NULL;
5503         }
5504         return clone;
5505 }
5506 EXPORT_SYMBOL(pskb_extract);
5507 
5508 /**
5509  * skb_condense - try to get rid of fragments/frag_list if possible
5510  * @skb: buffer
5511  *
5512  * Can be used to save memory before skb is added to a busy queue.
5513  * If packet has bytes in frags and enough tail room in skb->head,
5514  * pull all of them, so that we can free the frags right now and adjust
5515  * truesize.
5516  * Notes:
5517  *      We do not reallocate skb->head thus can not fail.
5518  *      Caller must re-evaluate skb->truesize if needed.
5519  */
5520 void skb_condense(struct sk_buff *skb)
5521 {
5522         if (skb->data_len) {
5523                 if (skb->data_len > skb->end - skb->tail ||
5524                     skb_cloned(skb))
5525                         return;
5526 
5527                 /* Nice, we can free page frag(s) right now */
5528                 __pskb_pull_tail(skb, skb->data_len);
5529         }
5530         /* At this point, skb->truesize might be over estimated,
5531          * because skb had a fragment, and fragments do not tell
5532          * their truesize.
5533          * When we pulled its content into skb->head, fragment
5534          * was freed, but __pskb_pull_tail() could not possibly
5535          * adjust skb->truesize, not knowing the frag truesize.
5536          */
5537         skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
5538 }
5539 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp