~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/ipv4/inet_lro.c

Version: ~ [ linux-5.12-rc7 ] ~ [ linux-5.11.13 ] ~ [ linux-5.10.29 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.111 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.186 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.230 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.266 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.266 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *  linux/net/ipv4/inet_lro.c
  3  *
  4  *  Large Receive Offload (ipv4 / tcp)
  5  *
  6  *  (C) Copyright IBM Corp. 2007
  7  *
  8  *  Authors:
  9  *       Jan-Bernd Themann <themann@de.ibm.com>
 10  *       Christoph Raisch <raisch@de.ibm.com>
 11  *
 12  *
 13  * This program is free software; you can redistribute it and/or modify
 14  * it under the terms of the GNU General Public License as published by
 15  * the Free Software Foundation; either version 2, or (at your option)
 16  * any later version.
 17  *
 18  * This program is distributed in the hope that it will be useful,
 19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 21  * GNU General Public License for more details.
 22  *
 23  * You should have received a copy of the GNU General Public License
 24  * along with this program; if not, write to the Free Software
 25  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 26  */
 27 
 28 
 29 #include <linux/module.h>
 30 #include <linux/if_vlan.h>
 31 #include <linux/inet_lro.h>
 32 #include <net/checksum.h>
 33 
 34 MODULE_LICENSE("GPL");
 35 MODULE_AUTHOR("Jan-Bernd Themann <themann@de.ibm.com>");
 36 MODULE_DESCRIPTION("Large Receive Offload (ipv4 / tcp)");
 37 
 38 #define TCP_HDR_LEN(tcph) (tcph->doff << 2)
 39 #define IP_HDR_LEN(iph) (iph->ihl << 2)
 40 #define TCP_PAYLOAD_LENGTH(iph, tcph) \
 41         (ntohs(iph->tot_len) - IP_HDR_LEN(iph) - TCP_HDR_LEN(tcph))
 42 
 43 #define IPH_LEN_WO_OPTIONS 5
 44 #define TCPH_LEN_WO_OPTIONS 5
 45 #define TCPH_LEN_W_TIMESTAMP 8
 46 
 47 #define LRO_MAX_PG_HLEN 64
 48 
 49 #define LRO_INC_STATS(lro_mgr, attr) { lro_mgr->stats.attr++; }
 50 
 51 /*
 52  * Basic tcp checks whether packet is suitable for LRO
 53  */
 54 
 55 static int lro_tcp_ip_check(const struct iphdr *iph, const struct tcphdr *tcph,
 56                             int len, const struct net_lro_desc *lro_desc)
 57 {
 58         /* check ip header: don't aggregate padded frames */
 59         if (ntohs(iph->tot_len) != len)
 60                 return -1;
 61 
 62         if (TCP_PAYLOAD_LENGTH(iph, tcph) == 0)
 63                 return -1;
 64 
 65         if (iph->ihl != IPH_LEN_WO_OPTIONS)
 66                 return -1;
 67 
 68         if (tcph->cwr || tcph->ece || tcph->urg || !tcph->ack ||
 69             tcph->rst || tcph->syn || tcph->fin)
 70                 return -1;
 71 
 72         if (INET_ECN_is_ce(ipv4_get_dsfield(iph)))
 73                 return -1;
 74 
 75         if (tcph->doff != TCPH_LEN_WO_OPTIONS &&
 76             tcph->doff != TCPH_LEN_W_TIMESTAMP)
 77                 return -1;
 78 
 79         /* check tcp options (only timestamp allowed) */
 80         if (tcph->doff == TCPH_LEN_W_TIMESTAMP) {
 81                 __be32 *topt = (__be32 *)(tcph + 1);
 82 
 83                 if (*topt != htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16)
 84                                    | (TCPOPT_TIMESTAMP << 8)
 85                                    | TCPOLEN_TIMESTAMP))
 86                         return -1;
 87 
 88                 /* timestamp should be in right order */
 89                 topt++;
 90                 if (lro_desc && after(ntohl(lro_desc->tcp_rcv_tsval),
 91                                       ntohl(*topt)))
 92                         return -1;
 93 
 94                 /* timestamp reply should not be zero */
 95                 topt++;
 96                 if (*topt == 0)
 97                         return -1;
 98         }
 99 
100         return 0;
101 }
102 
103 static void lro_update_tcp_ip_header(struct net_lro_desc *lro_desc)
104 {
105         struct iphdr *iph = lro_desc->iph;
106         struct tcphdr *tcph = lro_desc->tcph;
107         __be32 *p;
108         __wsum tcp_hdr_csum;
109 
110         tcph->ack_seq = lro_desc->tcp_ack;
111         tcph->window = lro_desc->tcp_window;
112 
113         if (lro_desc->tcp_saw_tstamp) {
114                 p = (__be32 *)(tcph + 1);
115                 *(p+2) = lro_desc->tcp_rcv_tsecr;
116         }
117 
118         csum_replace2(&iph->check, iph->tot_len, htons(lro_desc->ip_tot_len));
119         iph->tot_len = htons(lro_desc->ip_tot_len);
120 
121         tcph->check = 0;
122         tcp_hdr_csum = csum_partial(tcph, TCP_HDR_LEN(tcph), 0);
123         lro_desc->data_csum = csum_add(lro_desc->data_csum, tcp_hdr_csum);
124         tcph->check = csum_tcpudp_magic(iph->saddr, iph->daddr,
125                                         lro_desc->ip_tot_len -
126                                         IP_HDR_LEN(iph), IPPROTO_TCP,
127                                         lro_desc->data_csum);
128 }
129 
130 static __wsum lro_tcp_data_csum(struct iphdr *iph, struct tcphdr *tcph, int len)
131 {
132         __wsum tcp_csum;
133         __wsum tcp_hdr_csum;
134         __wsum tcp_ps_hdr_csum;
135 
136         tcp_csum = ~csum_unfold(tcph->check);
137         tcp_hdr_csum = csum_partial(tcph, TCP_HDR_LEN(tcph), tcp_csum);
138 
139         tcp_ps_hdr_csum = csum_tcpudp_nofold(iph->saddr, iph->daddr,
140                                              len + TCP_HDR_LEN(tcph),
141                                              IPPROTO_TCP, 0);
142 
143         return csum_sub(csum_sub(tcp_csum, tcp_hdr_csum),
144                         tcp_ps_hdr_csum);
145 }
146 
147 static void lro_init_desc(struct net_lro_desc *lro_desc, struct sk_buff *skb,
148                           struct iphdr *iph, struct tcphdr *tcph)
149 {
150         int nr_frags;
151         __be32 *ptr;
152         u32 tcp_data_len = TCP_PAYLOAD_LENGTH(iph, tcph);
153 
154         nr_frags = skb_shinfo(skb)->nr_frags;
155         lro_desc->parent = skb;
156         lro_desc->next_frag = &(skb_shinfo(skb)->frags[nr_frags]);
157         lro_desc->iph = iph;
158         lro_desc->tcph = tcph;
159         lro_desc->tcp_next_seq = ntohl(tcph->seq) + tcp_data_len;
160         lro_desc->tcp_ack = tcph->ack_seq;
161         lro_desc->tcp_window = tcph->window;
162 
163         lro_desc->pkt_aggr_cnt = 1;
164         lro_desc->ip_tot_len = ntohs(iph->tot_len);
165 
166         if (tcph->doff == 8) {
167                 ptr = (__be32 *)(tcph+1);
168                 lro_desc->tcp_saw_tstamp = 1;
169                 lro_desc->tcp_rcv_tsval = *(ptr+1);
170                 lro_desc->tcp_rcv_tsecr = *(ptr+2);
171         }
172 
173         lro_desc->mss = tcp_data_len;
174         lro_desc->active = 1;
175 
176         lro_desc->data_csum = lro_tcp_data_csum(iph, tcph,
177                                                 tcp_data_len);
178 }
179 
180 static inline void lro_clear_desc(struct net_lro_desc *lro_desc)
181 {
182         memset(lro_desc, 0, sizeof(struct net_lro_desc));
183 }
184 
185 static void lro_add_common(struct net_lro_desc *lro_desc, struct iphdr *iph,
186                            struct tcphdr *tcph, int tcp_data_len)
187 {
188         struct sk_buff *parent = lro_desc->parent;
189         __be32 *topt;
190 
191         lro_desc->pkt_aggr_cnt++;
192         lro_desc->ip_tot_len += tcp_data_len;
193         lro_desc->tcp_next_seq += tcp_data_len;
194         lro_desc->tcp_window = tcph->window;
195         lro_desc->tcp_ack = tcph->ack_seq;
196 
197         /* don't update tcp_rcv_tsval, would not work with PAWS */
198         if (lro_desc->tcp_saw_tstamp) {
199                 topt = (__be32 *) (tcph + 1);
200                 lro_desc->tcp_rcv_tsecr = *(topt + 2);
201         }
202 
203         lro_desc->data_csum = csum_block_add(lro_desc->data_csum,
204                                              lro_tcp_data_csum(iph, tcph,
205                                                                tcp_data_len),
206                                              parent->len);
207 
208         parent->len += tcp_data_len;
209         parent->data_len += tcp_data_len;
210         if (tcp_data_len > lro_desc->mss)
211                 lro_desc->mss = tcp_data_len;
212 }
213 
214 static void lro_add_packet(struct net_lro_desc *lro_desc, struct sk_buff *skb,
215                            struct iphdr *iph, struct tcphdr *tcph)
216 {
217         struct sk_buff *parent = lro_desc->parent;
218         int tcp_data_len = TCP_PAYLOAD_LENGTH(iph, tcph);
219 
220         lro_add_common(lro_desc, iph, tcph, tcp_data_len);
221 
222         skb_pull(skb, (skb->len - tcp_data_len));
223         parent->truesize += skb->truesize;
224 
225         if (lro_desc->last_skb)
226                 lro_desc->last_skb->next = skb;
227         else
228                 skb_shinfo(parent)->frag_list = skb;
229 
230         lro_desc->last_skb = skb;
231 }
232 
233 
234 static int lro_check_tcp_conn(struct net_lro_desc *lro_desc,
235                               struct iphdr *iph,
236                               struct tcphdr *tcph)
237 {
238         if ((lro_desc->iph->saddr != iph->saddr) ||
239             (lro_desc->iph->daddr != iph->daddr) ||
240             (lro_desc->tcph->source != tcph->source) ||
241             (lro_desc->tcph->dest != tcph->dest))
242                 return -1;
243         return 0;
244 }
245 
246 static struct net_lro_desc *lro_get_desc(struct net_lro_mgr *lro_mgr,
247                                          struct net_lro_desc *lro_arr,
248                                          struct iphdr *iph,
249                                          struct tcphdr *tcph)
250 {
251         struct net_lro_desc *lro_desc = NULL;
252         struct net_lro_desc *tmp;
253         int max_desc = lro_mgr->max_desc;
254         int i;
255 
256         for (i = 0; i < max_desc; i++) {
257                 tmp = &lro_arr[i];
258                 if (tmp->active)
259                         if (!lro_check_tcp_conn(tmp, iph, tcph)) {
260                                 lro_desc = tmp;
261                                 goto out;
262                         }
263         }
264 
265         for (i = 0; i < max_desc; i++) {
266                 if (!lro_arr[i].active) {
267                         lro_desc = &lro_arr[i];
268                         goto out;
269                 }
270         }
271 
272         LRO_INC_STATS(lro_mgr, no_desc);
273 out:
274         return lro_desc;
275 }
276 
277 static void lro_flush(struct net_lro_mgr *lro_mgr,
278                       struct net_lro_desc *lro_desc)
279 {
280         if (lro_desc->pkt_aggr_cnt > 1)
281                 lro_update_tcp_ip_header(lro_desc);
282 
283         skb_shinfo(lro_desc->parent)->gso_size = lro_desc->mss;
284 
285         if (lro_mgr->features & LRO_F_NAPI)
286                 netif_receive_skb(lro_desc->parent);
287         else
288                 netif_rx(lro_desc->parent);
289 
290         LRO_INC_STATS(lro_mgr, flushed);
291         lro_clear_desc(lro_desc);
292 }
293 
294 static int __lro_proc_skb(struct net_lro_mgr *lro_mgr, struct sk_buff *skb,
295                           void *priv)
296 {
297         struct net_lro_desc *lro_desc;
298         struct iphdr *iph;
299         struct tcphdr *tcph;
300         u64 flags;
301         int vlan_hdr_len = 0;
302 
303         if (!lro_mgr->get_skb_header ||
304             lro_mgr->get_skb_header(skb, (void *)&iph, (void *)&tcph,
305                                     &flags, priv))
306                 goto out;
307 
308         if (!(flags & LRO_IPV4) || !(flags & LRO_TCP))
309                 goto out;
310 
311         lro_desc = lro_get_desc(lro_mgr, lro_mgr->lro_arr, iph, tcph);
312         if (!lro_desc)
313                 goto out;
314 
315         if ((skb->protocol == htons(ETH_P_8021Q)) &&
316             !(lro_mgr->features & LRO_F_EXTRACT_VLAN_ID))
317                 vlan_hdr_len = VLAN_HLEN;
318 
319         if (!lro_desc->active) { /* start new lro session */
320                 if (lro_tcp_ip_check(iph, tcph, skb->len - vlan_hdr_len, NULL))
321                         goto out;
322 
323                 skb->ip_summed = lro_mgr->ip_summed_aggr;
324                 lro_init_desc(lro_desc, skb, iph, tcph);
325                 LRO_INC_STATS(lro_mgr, aggregated);
326                 return 0;
327         }
328 
329         if (lro_desc->tcp_next_seq != ntohl(tcph->seq))
330                 goto out2;
331 
332         if (lro_tcp_ip_check(iph, tcph, skb->len, lro_desc))
333                 goto out2;
334 
335         lro_add_packet(lro_desc, skb, iph, tcph);
336         LRO_INC_STATS(lro_mgr, aggregated);
337 
338         if ((lro_desc->pkt_aggr_cnt >= lro_mgr->max_aggr) ||
339             lro_desc->parent->len > (0xFFFF - lro_mgr->dev->mtu))
340                 lro_flush(lro_mgr, lro_desc);
341 
342         return 0;
343 
344 out2: /* send aggregated SKBs to stack */
345         lro_flush(lro_mgr, lro_desc);
346 
347 out:
348         return 1;
349 }
350 
351 void lro_receive_skb(struct net_lro_mgr *lro_mgr,
352                      struct sk_buff *skb,
353                      void *priv)
354 {
355         if (__lro_proc_skb(lro_mgr, skb, priv)) {
356                 if (lro_mgr->features & LRO_F_NAPI)
357                         netif_receive_skb(skb);
358                 else
359                         netif_rx(skb);
360         }
361 }
362 EXPORT_SYMBOL(lro_receive_skb);
363 
364 void lro_flush_all(struct net_lro_mgr *lro_mgr)
365 {
366         int i;
367         struct net_lro_desc *lro_desc = lro_mgr->lro_arr;
368 
369         for (i = 0; i < lro_mgr->max_desc; i++) {
370                 if (lro_desc[i].active)
371                         lro_flush(lro_mgr, &lro_desc[i]);
372         }
373 }
374 EXPORT_SYMBOL(lro_flush_all);
375 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp