~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/ipv4/udp_offload.c

Version: ~ [ linux-5.11-rc3 ] ~ [ linux-5.10.7 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.89 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.167 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.215 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.251 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.251 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *      IPV4 GSO/GRO offload support
  3  *      Linux INET implementation
  4  *
  5  *      This program is free software; you can redistribute it and/or
  6  *      modify it under the terms of the GNU General Public License
  7  *      as published by the Free Software Foundation; either version
  8  *      2 of the License, or (at your option) any later version.
  9  *
 10  *      UDPv4 GSO support
 11  */
 12 
 13 #include <linux/skbuff.h>
 14 #include <net/udp.h>
 15 #include <net/protocol.h>
 16 
 17 static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,
 18         netdev_features_t features,
 19         struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,
 20                                              netdev_features_t features),
 21         __be16 new_protocol, bool is_ipv6)
 22 {
 23         int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb);
 24         bool remcsum, need_csum, offload_csum, gso_partial;
 25         struct sk_buff *segs = ERR_PTR(-EINVAL);
 26         struct udphdr *uh = udp_hdr(skb);
 27         u16 mac_offset = skb->mac_header;
 28         __be16 protocol = skb->protocol;
 29         u16 mac_len = skb->mac_len;
 30         int udp_offset, outer_hlen;
 31         __wsum partial;
 32         bool need_ipsec;
 33 
 34         if (unlikely(!pskb_may_pull(skb, tnl_hlen)))
 35                 goto out;
 36 
 37         /* Adjust partial header checksum to negate old length.
 38          * We cannot rely on the value contained in uh->len as it is
 39          * possible that the actual value exceeds the boundaries of the
 40          * 16 bit length field due to the header being added outside of an
 41          * IP or IPv6 frame that was already limited to 64K - 1.
 42          */
 43         if (skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)
 44                 partial = (__force __wsum)uh->len;
 45         else
 46                 partial = (__force __wsum)htonl(skb->len);
 47         partial = csum_sub(csum_unfold(uh->check), partial);
 48 
 49         /* setup inner skb. */
 50         skb->encapsulation = 0;
 51         SKB_GSO_CB(skb)->encap_level = 0;
 52         __skb_pull(skb, tnl_hlen);
 53         skb_reset_mac_header(skb);
 54         skb_set_network_header(skb, skb_inner_network_offset(skb));
 55         skb->mac_len = skb_inner_network_offset(skb);
 56         skb->protocol = new_protocol;
 57 
 58         need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM);
 59         skb->encap_hdr_csum = need_csum;
 60 
 61         remcsum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_TUNNEL_REMCSUM);
 62         skb->remcsum_offload = remcsum;
 63 
 64         need_ipsec = skb_dst(skb) && dst_xfrm(skb_dst(skb));
 65         /* Try to offload checksum if possible */
 66         offload_csum = !!(need_csum &&
 67                           !need_ipsec &&
 68                           (skb->dev->features &
 69                            (is_ipv6 ? (NETIF_F_HW_CSUM | NETIF_F_IPV6_CSUM) :
 70                                       (NETIF_F_HW_CSUM | NETIF_F_IP_CSUM))));
 71 
 72         features &= skb->dev->hw_enc_features;
 73 
 74         /* The only checksum offload we care about from here on out is the
 75          * outer one so strip the existing checksum feature flags and
 76          * instead set the flag based on our outer checksum offload value.
 77          */
 78         if (remcsum) {
 79                 features &= ~NETIF_F_CSUM_MASK;
 80                 if (!need_csum || offload_csum)
 81                         features |= NETIF_F_HW_CSUM;
 82         }
 83 
 84         /* segment inner packet. */
 85         segs = gso_inner_segment(skb, features);
 86         if (IS_ERR_OR_NULL(segs)) {
 87                 skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset,
 88                                      mac_len);
 89                 goto out;
 90         }
 91 
 92         gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL);
 93 
 94         outer_hlen = skb_tnl_header_len(skb);
 95         udp_offset = outer_hlen - tnl_hlen;
 96         skb = segs;
 97         do {
 98                 unsigned int len;
 99 
100                 if (remcsum)
101                         skb->ip_summed = CHECKSUM_NONE;
102 
103                 /* Set up inner headers if we are offloading inner checksum */
104                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
105                         skb_reset_inner_headers(skb);
106                         skb->encapsulation = 1;
107                 }
108 
109                 skb->mac_len = mac_len;
110                 skb->protocol = protocol;
111 
112                 __skb_push(skb, outer_hlen);
113                 skb_reset_mac_header(skb);
114                 skb_set_network_header(skb, mac_len);
115                 skb_set_transport_header(skb, udp_offset);
116                 len = skb->len - udp_offset;
117                 uh = udp_hdr(skb);
118 
119                 /* If we are only performing partial GSO the inner header
120                  * will be using a length value equal to only one MSS sized
121                  * segment instead of the entire frame.
122                  */
123                 if (gso_partial && skb_is_gso(skb)) {
124                         uh->len = htons(skb_shinfo(skb)->gso_size +
125                                         SKB_GSO_CB(skb)->data_offset +
126                                         skb->head - (unsigned char *)uh);
127                 } else {
128                         uh->len = htons(len);
129                 }
130 
131                 if (!need_csum)
132                         continue;
133 
134                 uh->check = ~csum_fold(csum_add(partial,
135                                        (__force __wsum)htonl(len)));
136 
137                 if (skb->encapsulation || !offload_csum) {
138                         uh->check = gso_make_checksum(skb, ~uh->check);
139                         if (uh->check == 0)
140                                 uh->check = CSUM_MANGLED_0;
141                 } else {
142                         skb->ip_summed = CHECKSUM_PARTIAL;
143                         skb->csum_start = skb_transport_header(skb) - skb->head;
144                         skb->csum_offset = offsetof(struct udphdr, check);
145                 }
146         } while ((skb = skb->next));
147 out:
148         return segs;
149 }
150 
151 struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb,
152                                        netdev_features_t features,
153                                        bool is_ipv6)
154 {
155         __be16 protocol = skb->protocol;
156         const struct net_offload **offloads;
157         const struct net_offload *ops;
158         struct sk_buff *segs = ERR_PTR(-EINVAL);
159         struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,
160                                              netdev_features_t features);
161 
162         rcu_read_lock();
163 
164         switch (skb->inner_protocol_type) {
165         case ENCAP_TYPE_ETHER:
166                 protocol = skb->inner_protocol;
167                 gso_inner_segment = skb_mac_gso_segment;
168                 break;
169         case ENCAP_TYPE_IPPROTO:
170                 offloads = is_ipv6 ? inet6_offloads : inet_offloads;
171                 ops = rcu_dereference(offloads[skb->inner_ipproto]);
172                 if (!ops || !ops->callbacks.gso_segment)
173                         goto out_unlock;
174                 gso_inner_segment = ops->callbacks.gso_segment;
175                 break;
176         default:
177                 goto out_unlock;
178         }
179 
180         segs = __skb_udp_tunnel_segment(skb, features, gso_inner_segment,
181                                         protocol, is_ipv6);
182 
183 out_unlock:
184         rcu_read_unlock();
185 
186         return segs;
187 }
188 EXPORT_SYMBOL(skb_udp_tunnel_segment);
189 
190 struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
191                                   netdev_features_t features)
192 {
193         struct sock *sk = gso_skb->sk;
194         unsigned int sum_truesize = 0;
195         struct sk_buff *segs, *seg;
196         struct udphdr *uh;
197         unsigned int mss;
198         bool copy_dtor;
199         __sum16 check;
200         __be16 newlen;
201 
202         mss = skb_shinfo(gso_skb)->gso_size;
203         if (gso_skb->len <= sizeof(*uh) + mss)
204                 return ERR_PTR(-EINVAL);
205 
206         skb_pull(gso_skb, sizeof(*uh));
207 
208         /* clear destructor to avoid skb_segment assigning it to tail */
209         copy_dtor = gso_skb->destructor == sock_wfree;
210         if (copy_dtor)
211                 gso_skb->destructor = NULL;
212 
213         segs = skb_segment(gso_skb, features);
214         if (unlikely(IS_ERR_OR_NULL(segs))) {
215                 if (copy_dtor)
216                         gso_skb->destructor = sock_wfree;
217                 return segs;
218         }
219 
220         /* GSO partial and frag_list segmentation only requires splitting
221          * the frame into an MSS multiple and possibly a remainder, both
222          * cases return a GSO skb. So update the mss now.
223          */
224         if (skb_is_gso(segs))
225                 mss *= skb_shinfo(segs)->gso_segs;
226 
227         seg = segs;
228         uh = udp_hdr(seg);
229 
230         /* compute checksum adjustment based on old length versus new */
231         newlen = htons(sizeof(*uh) + mss);
232         check = csum16_add(csum16_sub(uh->check, uh->len), newlen);
233 
234         for (;;) {
235                 if (copy_dtor) {
236                         seg->destructor = sock_wfree;
237                         seg->sk = sk;
238                         sum_truesize += seg->truesize;
239                 }
240 
241                 if (!seg->next)
242                         break;
243 
244                 uh->len = newlen;
245                 uh->check = check;
246 
247                 if (seg->ip_summed == CHECKSUM_PARTIAL)
248                         gso_reset_checksum(seg, ~check);
249                 else
250                         uh->check = gso_make_checksum(seg, ~check) ? :
251                                     CSUM_MANGLED_0;
252 
253                 seg = seg->next;
254                 uh = udp_hdr(seg);
255         }
256 
257         /* last packet can be partial gso_size, account for that in checksum */
258         newlen = htons(skb_tail_pointer(seg) - skb_transport_header(seg) +
259                        seg->data_len);
260         check = csum16_add(csum16_sub(uh->check, uh->len), newlen);
261 
262         uh->len = newlen;
263         uh->check = check;
264 
265         if (seg->ip_summed == CHECKSUM_PARTIAL)
266                 gso_reset_checksum(seg, ~check);
267         else
268                 uh->check = gso_make_checksum(seg, ~check) ? : CSUM_MANGLED_0;
269 
270         /* update refcount for the packet */
271         if (copy_dtor) {
272                 int delta = sum_truesize - gso_skb->truesize;
273 
274                 /* In some pathological cases, delta can be negative.
275                  * We need to either use refcount_add() or refcount_sub_and_test()
276                  */
277                 if (likely(delta >= 0))
278                         refcount_add(delta, &sk->sk_wmem_alloc);
279                 else
280                         WARN_ON_ONCE(refcount_sub_and_test(-delta, &sk->sk_wmem_alloc));
281         }
282         return segs;
283 }
284 EXPORT_SYMBOL_GPL(__udp_gso_segment);
285 
286 static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb,
287                                          netdev_features_t features)
288 {
289         struct sk_buff *segs = ERR_PTR(-EINVAL);
290         unsigned int mss;
291         __wsum csum;
292         struct udphdr *uh;
293         struct iphdr *iph;
294 
295         if (skb->encapsulation &&
296             (skb_shinfo(skb)->gso_type &
297              (SKB_GSO_UDP_TUNNEL|SKB_GSO_UDP_TUNNEL_CSUM))) {
298                 segs = skb_udp_tunnel_segment(skb, features, false);
299                 goto out;
300         }
301 
302         if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_UDP | SKB_GSO_UDP_L4)))
303                 goto out;
304 
305         if (!pskb_may_pull(skb, sizeof(struct udphdr)))
306                 goto out;
307 
308         if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)
309                 return __udp_gso_segment(skb, features);
310 
311         mss = skb_shinfo(skb)->gso_size;
312         if (unlikely(skb->len <= mss))
313                 goto out;
314 
315         /* Do software UFO. Complete and fill in the UDP checksum as
316          * HW cannot do checksum of UDP packets sent as multiple
317          * IP fragments.
318          */
319 
320         uh = udp_hdr(skb);
321         iph = ip_hdr(skb);
322 
323         uh->check = 0;
324         csum = skb_checksum(skb, 0, skb->len, 0);
325         uh->check = udp_v4_check(skb->len, iph->saddr, iph->daddr, csum);
326         if (uh->check == 0)
327                 uh->check = CSUM_MANGLED_0;
328 
329         skb->ip_summed = CHECKSUM_UNNECESSARY;
330 
331         /* If there is no outer header we can fake a checksum offload
332          * due to the fact that we have already done the checksum in
333          * software prior to segmenting the frame.
334          */
335         if (!skb->encap_hdr_csum)
336                 features |= NETIF_F_HW_CSUM;
337 
338         /* Fragment the skb. IP headers of the fragments are updated in
339          * inet_gso_segment()
340          */
341         segs = skb_segment(skb, features);
342 out:
343         return segs;
344 }
345 
346 struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb,
347                                  struct udphdr *uh, udp_lookup_t lookup)
348 {
349         struct sk_buff *p, **pp = NULL;
350         struct udphdr *uh2;
351         unsigned int off = skb_gro_offset(skb);
352         int flush = 1;
353         struct sock *sk;
354 
355         if (NAPI_GRO_CB(skb)->encap_mark ||
356             (skb->ip_summed != CHECKSUM_PARTIAL &&
357              NAPI_GRO_CB(skb)->csum_cnt == 0 &&
358              !NAPI_GRO_CB(skb)->csum_valid))
359                 goto out;
360 
361         /* mark that this skb passed once through the tunnel gro layer */
362         NAPI_GRO_CB(skb)->encap_mark = 1;
363 
364         rcu_read_lock();
365         sk = (*lookup)(skb, uh->source, uh->dest);
366 
367         if (sk && udp_sk(sk)->gro_receive)
368                 goto unflush;
369         goto out_unlock;
370 
371 unflush:
372         flush = 0;
373 
374         for (p = *head; p; p = p->next) {
375                 if (!NAPI_GRO_CB(p)->same_flow)
376                         continue;
377 
378                 uh2 = (struct udphdr   *)(p->data + off);
379 
380                 /* Match ports and either checksums are either both zero
381                  * or nonzero.
382                  */
383                 if ((*(u32 *)&uh->source != *(u32 *)&uh2->source) ||
384                     (!uh->check ^ !uh2->check)) {
385                         NAPI_GRO_CB(p)->same_flow = 0;
386                         continue;
387                 }
388         }
389 
390         skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */
391         skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr));
392         pp = call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);
393 
394 out_unlock:
395         rcu_read_unlock();
396 out:
397         skb_gro_flush_final(skb, pp, flush);
398         return pp;
399 }
400 EXPORT_SYMBOL(udp_gro_receive);
401 
402 static struct sk_buff **udp4_gro_receive(struct sk_buff **head,
403                                          struct sk_buff *skb)
404 {
405         struct udphdr *uh = udp_gro_udphdr(skb);
406 
407         if (unlikely(!uh))
408                 goto flush;
409 
410         /* Don't bother verifying checksum if we're going to flush anyway. */
411         if (NAPI_GRO_CB(skb)->flush)
412                 goto skip;
413 
414         if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check,
415                                                  inet_gro_compute_pseudo))
416                 goto flush;
417         else if (uh->check)
418                 skb_gro_checksum_try_convert(skb, IPPROTO_UDP, uh->check,
419                                              inet_gro_compute_pseudo);
420 skip:
421         NAPI_GRO_CB(skb)->is_ipv6 = 0;
422         return udp_gro_receive(head, skb, uh, udp4_lib_lookup_skb);
423 
424 flush:
425         NAPI_GRO_CB(skb)->flush = 1;
426         return NULL;
427 }
428 
429 int udp_gro_complete(struct sk_buff *skb, int nhoff,
430                      udp_lookup_t lookup)
431 {
432         __be16 newlen = htons(skb->len - nhoff);
433         struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
434         int err = -ENOSYS;
435         struct sock *sk;
436 
437         uh->len = newlen;
438 
439         /* Set encapsulation before calling into inner gro_complete() functions
440          * to make them set up the inner offsets.
441          */
442         skb->encapsulation = 1;
443 
444         rcu_read_lock();
445         sk = (*lookup)(skb, uh->source, uh->dest);
446         if (sk && udp_sk(sk)->gro_complete)
447                 err = udp_sk(sk)->gro_complete(sk, skb,
448                                 nhoff + sizeof(struct udphdr));
449         rcu_read_unlock();
450 
451         if (skb->remcsum_offload)
452                 skb_shinfo(skb)->gso_type |= SKB_GSO_TUNNEL_REMCSUM;
453 
454         return err;
455 }
456 EXPORT_SYMBOL(udp_gro_complete);
457 
458 static int udp4_gro_complete(struct sk_buff *skb, int nhoff)
459 {
460         const struct iphdr *iph = ip_hdr(skb);
461         struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
462 
463         if (uh->check) {
464                 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
465                 uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr,
466                                           iph->daddr, 0);
467         } else {
468                 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
469         }
470 
471         return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb);
472 }
473 
474 static const struct net_offload udpv4_offload = {
475         .callbacks = {
476                 .gso_segment = udp4_ufo_fragment,
477                 .gro_receive  = udp4_gro_receive,
478                 .gro_complete = udp4_gro_complete,
479         },
480 };
481 
482 int __init udpv4_offload_init(void)
483 {
484         return inet_add_offload(&udpv4_offload, IPPROTO_UDP);
485 }
486 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp