~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/ipv4/tcp_output.c

Version: ~ [ linux-5.4-rc3 ] ~ [ linux-5.3.6 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.79 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.149 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.196 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.196 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.75 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
  3  *              operating system.  INET is implemented using the  BSD Socket
  4  *              interface as the means of communication with the user level.
  5  *
  6  *              Implementation of the Transmission Control Protocol(TCP).
  7  *
  8  * Authors:     Ross Biro
  9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 10  *              Mark Evans, <evansmp@uhura.aston.ac.uk>
 11  *              Corey Minyard <wf-rch!minyard@relay.EU.net>
 12  *              Florian La Roche, <flla@stud.uni-sb.de>
 13  *              Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
 14  *              Linus Torvalds, <torvalds@cs.helsinki.fi>
 15  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
 16  *              Matthew Dillon, <dillon@apollo.west.oic.com>
 17  *              Arnt Gulbrandsen, <agulbra@nvg.unit.no>
 18  *              Jorge Cwik, <jorge@laser.satlink.net>
 19  */
 20 
 21 /*
 22  * Changes:     Pedro Roque     :       Retransmit queue handled by TCP.
 23  *                              :       Fragmentation on mtu decrease
 24  *                              :       Segment collapse on retransmit
 25  *                              :       AF independence
 26  *
 27  *              Linus Torvalds  :       send_delayed_ack
 28  *              David S. Miller :       Charge memory using the right skb
 29  *                                      during syn/ack processing.
 30  *              David S. Miller :       Output engine completely rewritten.
 31  *              Andrea Arcangeli:       SYNACK carry ts_recent in tsecr.
 32  *              Cacophonix Gaul :       draft-minshall-nagle-01
 33  *              J Hadi Salim    :       ECN support
 34  *
 35  */
 36 
 37 #define pr_fmt(fmt) "TCP: " fmt
 38 
 39 #include <net/tcp.h>
 40 
 41 #include <linux/compiler.h>
 42 #include <linux/gfp.h>
 43 #include <linux/module.h>
 44 #include <linux/static_key.h>
 45 
 46 #include <trace/events/tcp.h>
 47 
 48 static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
 49                            int push_one, gfp_t gfp);
 50 
 51 /* Account for new data that has been sent to the network. */
 52 static void tcp_event_new_data_sent(struct sock *sk, struct sk_buff *skb)
 53 {
 54         struct inet_connection_sock *icsk = inet_csk(sk);
 55         struct tcp_sock *tp = tcp_sk(sk);
 56         unsigned int prior_packets = tp->packets_out;
 57 
 58         tp->snd_nxt = TCP_SKB_CB(skb)->end_seq;
 59 
 60         __skb_unlink(skb, &sk->sk_write_queue);
 61         tcp_rbtree_insert(&sk->tcp_rtx_queue, skb);
 62 
 63         tp->packets_out += tcp_skb_pcount(skb);
 64         if (!prior_packets || icsk->icsk_pending == ICSK_TIME_LOSS_PROBE)
 65                 tcp_rearm_rto(sk);
 66 
 67         NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT,
 68                       tcp_skb_pcount(skb));
 69 }
 70 
 71 /* SND.NXT, if window was not shrunk or the amount of shrunk was less than one
 72  * window scaling factor due to loss of precision.
 73  * If window has been shrunk, what should we make? It is not clear at all.
 74  * Using SND.UNA we will fail to open window, SND.NXT is out of window. :-(
 75  * Anything in between SND.UNA...SND.UNA+SND.WND also can be already
 76  * invalid. OK, let's make this for now:
 77  */
 78 static inline __u32 tcp_acceptable_seq(const struct sock *sk)
 79 {
 80         const struct tcp_sock *tp = tcp_sk(sk);
 81 
 82         if (!before(tcp_wnd_end(tp), tp->snd_nxt) ||
 83             (tp->rx_opt.wscale_ok &&
 84              ((tp->snd_nxt - tcp_wnd_end(tp)) < (1 << tp->rx_opt.rcv_wscale))))
 85                 return tp->snd_nxt;
 86         else
 87                 return tcp_wnd_end(tp);
 88 }
 89 
 90 /* Calculate mss to advertise in SYN segment.
 91  * RFC1122, RFC1063, draft-ietf-tcpimpl-pmtud-01 state that:
 92  *
 93  * 1. It is independent of path mtu.
 94  * 2. Ideally, it is maximal possible segment size i.e. 65535-40.
 95  * 3. For IPv4 it is reasonable to calculate it from maximal MTU of
 96  *    attached devices, because some buggy hosts are confused by
 97  *    large MSS.
 98  * 4. We do not make 3, we advertise MSS, calculated from first
 99  *    hop device mtu, but allow to raise it to ip_rt_min_advmss.
100  *    This may be overridden via information stored in routing table.
101  * 5. Value 65535 for MSS is valid in IPv6 and means "as large as possible,
102  *    probably even Jumbo".
103  */
104 static __u16 tcp_advertise_mss(struct sock *sk)
105 {
106         struct tcp_sock *tp = tcp_sk(sk);
107         const struct dst_entry *dst = __sk_dst_get(sk);
108         int mss = tp->advmss;
109 
110         if (dst) {
111                 unsigned int metric = dst_metric_advmss(dst);
112 
113                 if (metric < mss) {
114                         mss = metric;
115                         tp->advmss = mss;
116                 }
117         }
118 
119         return (__u16)mss;
120 }
121 
122 /* RFC2861. Reset CWND after idle period longer RTO to "restart window".
123  * This is the first part of cwnd validation mechanism.
124  */
125 void tcp_cwnd_restart(struct sock *sk, s32 delta)
126 {
127         struct tcp_sock *tp = tcp_sk(sk);
128         u32 restart_cwnd = tcp_init_cwnd(tp, __sk_dst_get(sk));
129         u32 cwnd = tp->snd_cwnd;
130 
131         tcp_ca_event(sk, CA_EVENT_CWND_RESTART);
132 
133         tp->snd_ssthresh = tcp_current_ssthresh(sk);
134         restart_cwnd = min(restart_cwnd, cwnd);
135 
136         while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd)
137                 cwnd >>= 1;
138         tp->snd_cwnd = max(cwnd, restart_cwnd);
139         tp->snd_cwnd_stamp = tcp_jiffies32;
140         tp->snd_cwnd_used = 0;
141 }
142 
143 /* Congestion state accounting after a packet has been sent. */
144 static void tcp_event_data_sent(struct tcp_sock *tp,
145                                 struct sock *sk)
146 {
147         struct inet_connection_sock *icsk = inet_csk(sk);
148         const u32 now = tcp_jiffies32;
149 
150         if (tcp_packets_in_flight(tp) == 0)
151                 tcp_ca_event(sk, CA_EVENT_TX_START);
152 
153         tp->lsndtime = now;
154 
155         /* If it is a reply for ato after last received
156          * packet, enter pingpong mode.
157          */
158         if ((u32)(now - icsk->icsk_ack.lrcvtime) < icsk->icsk_ack.ato)
159                 icsk->icsk_ack.pingpong = 1;
160 }
161 
162 /* Account for an ACK we sent. */
163 static inline void tcp_event_ack_sent(struct sock *sk, unsigned int pkts)
164 {
165         tcp_dec_quickack_mode(sk, pkts);
166         inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK);
167 }
168 
169 
170 u32 tcp_default_init_rwnd(u32 mss)
171 {
172         /* Initial receive window should be twice of TCP_INIT_CWND to
173          * enable proper sending of new unsent data during fast recovery
174          * (RFC 3517, Section 4, NextSeg() rule (2)). Further place a
175          * limit when mss is larger than 1460.
176          */
177         u32 init_rwnd = TCP_INIT_CWND * 2;
178 
179         if (mss > 1460)
180                 init_rwnd = max((1460 * init_rwnd) / mss, 2U);
181         return init_rwnd;
182 }
183 
184 /* Determine a window scaling and initial window to offer.
185  * Based on the assumption that the given amount of space
186  * will be offered. Store the results in the tp structure.
187  * NOTE: for smooth operation initial space offering should
188  * be a multiple of mss if possible. We assume here that mss >= 1.
189  * This MUST be enforced by all callers.
190  */
191 void tcp_select_initial_window(const struct sock *sk, int __space, __u32 mss,
192                                __u32 *rcv_wnd, __u32 *window_clamp,
193                                int wscale_ok, __u8 *rcv_wscale,
194                                __u32 init_rcv_wnd)
195 {
196         unsigned int space = (__space < 0 ? 0 : __space);
197 
198         /* If no clamp set the clamp to the max possible scaled window */
199         if (*window_clamp == 0)
200                 (*window_clamp) = (U16_MAX << TCP_MAX_WSCALE);
201         space = min(*window_clamp, space);
202 
203         /* Quantize space offering to a multiple of mss if possible. */
204         if (space > mss)
205                 space = rounddown(space, mss);
206 
207         /* NOTE: offering an initial window larger than 32767
208          * will break some buggy TCP stacks. If the admin tells us
209          * it is likely we could be speaking with such a buggy stack
210          * we will truncate our initial window offering to 32K-1
211          * unless the remote has sent us a window scaling option,
212          * which we interpret as a sign the remote TCP is not
213          * misinterpreting the window field as a signed quantity.
214          */
215         if (sock_net(sk)->ipv4.sysctl_tcp_workaround_signed_windows)
216                 (*rcv_wnd) = min(space, MAX_TCP_WINDOW);
217         else
218                 (*rcv_wnd) = space;
219 
220         (*rcv_wscale) = 0;
221         if (wscale_ok) {
222                 /* Set window scaling on max possible window */
223                 space = max_t(u32, space, sock_net(sk)->ipv4.sysctl_tcp_rmem[2]);
224                 space = max_t(u32, space, sysctl_rmem_max);
225                 space = min_t(u32, space, *window_clamp);
226                 while (space > U16_MAX && (*rcv_wscale) < TCP_MAX_WSCALE) {
227                         space >>= 1;
228                         (*rcv_wscale)++;
229                 }
230         }
231 
232         if (mss > (1 << *rcv_wscale)) {
233                 if (!init_rcv_wnd) /* Use default unless specified otherwise */
234                         init_rcv_wnd = tcp_default_init_rwnd(mss);
235                 *rcv_wnd = min(*rcv_wnd, init_rcv_wnd * mss);
236         }
237 
238         /* Set the clamp no higher than max representable value */
239         (*window_clamp) = min_t(__u32, U16_MAX << (*rcv_wscale), *window_clamp);
240 }
241 EXPORT_SYMBOL(tcp_select_initial_window);
242 
243 /* Chose a new window to advertise, update state in tcp_sock for the
244  * socket, and return result with RFC1323 scaling applied.  The return
245  * value can be stuffed directly into th->window for an outgoing
246  * frame.
247  */
248 static u16 tcp_select_window(struct sock *sk)
249 {
250         struct tcp_sock *tp = tcp_sk(sk);
251         u32 old_win = tp->rcv_wnd;
252         u32 cur_win = tcp_receive_window(tp);
253         u32 new_win = __tcp_select_window(sk);
254 
255         /* Never shrink the offered window */
256         if (new_win < cur_win) {
257                 /* Danger Will Robinson!
258                  * Don't update rcv_wup/rcv_wnd here or else
259                  * we will not be able to advertise a zero
260                  * window in time.  --DaveM
261                  *
262                  * Relax Will Robinson.
263                  */
264                 if (new_win == 0)
265                         NET_INC_STATS(sock_net(sk),
266                                       LINUX_MIB_TCPWANTZEROWINDOWADV);
267                 new_win = ALIGN(cur_win, 1 << tp->rx_opt.rcv_wscale);
268         }
269         tp->rcv_wnd = new_win;
270         tp->rcv_wup = tp->rcv_nxt;
271 
272         /* Make sure we do not exceed the maximum possible
273          * scaled window.
274          */
275         if (!tp->rx_opt.rcv_wscale &&
276             sock_net(sk)->ipv4.sysctl_tcp_workaround_signed_windows)
277                 new_win = min(new_win, MAX_TCP_WINDOW);
278         else
279                 new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale));
280 
281         /* RFC1323 scaling applied */
282         new_win >>= tp->rx_opt.rcv_wscale;
283 
284         /* If we advertise zero window, disable fast path. */
285         if (new_win == 0) {
286                 tp->pred_flags = 0;
287                 if (old_win)
288                         NET_INC_STATS(sock_net(sk),
289                                       LINUX_MIB_TCPTOZEROWINDOWADV);
290         } else if (old_win == 0) {
291                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFROMZEROWINDOWADV);
292         }
293 
294         return new_win;
295 }
296 
297 /* Packet ECN state for a SYN-ACK */
298 static void tcp_ecn_send_synack(struct sock *sk, struct sk_buff *skb)
299 {
300         const struct tcp_sock *tp = tcp_sk(sk);
301 
302         TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_CWR;
303         if (!(tp->ecn_flags & TCP_ECN_OK))
304                 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE;
305         else if (tcp_ca_needs_ecn(sk) ||
306                  tcp_bpf_ca_needs_ecn(sk))
307                 INET_ECN_xmit(sk);
308 }
309 
310 /* Packet ECN state for a SYN.  */
311 static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb)
312 {
313         struct tcp_sock *tp = tcp_sk(sk);
314         bool bpf_needs_ecn = tcp_bpf_ca_needs_ecn(sk);
315         bool use_ecn = sock_net(sk)->ipv4.sysctl_tcp_ecn == 1 ||
316                 tcp_ca_needs_ecn(sk) || bpf_needs_ecn;
317 
318         if (!use_ecn) {
319                 const struct dst_entry *dst = __sk_dst_get(sk);
320 
321                 if (dst && dst_feature(dst, RTAX_FEATURE_ECN))
322                         use_ecn = true;
323         }
324 
325         tp->ecn_flags = 0;
326 
327         if (use_ecn) {
328                 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR;
329                 tp->ecn_flags = TCP_ECN_OK;
330                 if (tcp_ca_needs_ecn(sk) || bpf_needs_ecn)
331                         INET_ECN_xmit(sk);
332         }
333 }
334 
335 static void tcp_ecn_clear_syn(struct sock *sk, struct sk_buff *skb)
336 {
337         if (sock_net(sk)->ipv4.sysctl_tcp_ecn_fallback)
338                 /* tp->ecn_flags are cleared at a later point in time when
339                  * SYN ACK is ultimatively being received.
340                  */
341                 TCP_SKB_CB(skb)->tcp_flags &= ~(TCPHDR_ECE | TCPHDR_CWR);
342 }
343 
344 static void
345 tcp_ecn_make_synack(const struct request_sock *req, struct tcphdr *th)
346 {
347         if (inet_rsk(req)->ecn_ok)
348                 th->ece = 1;
349 }
350 
351 /* Set up ECN state for a packet on a ESTABLISHED socket that is about to
352  * be sent.
353  */
354 static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb,
355                          struct tcphdr *th, int tcp_header_len)
356 {
357         struct tcp_sock *tp = tcp_sk(sk);
358 
359         if (tp->ecn_flags & TCP_ECN_OK) {
360                 /* Not-retransmitted data segment: set ECT and inject CWR. */
361                 if (skb->len != tcp_header_len &&
362                     !before(TCP_SKB_CB(skb)->seq, tp->snd_nxt)) {
363                         INET_ECN_xmit(sk);
364                         if (tp->ecn_flags & TCP_ECN_QUEUE_CWR) {
365                                 tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR;
366                                 th->cwr = 1;
367                                 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
368                         }
369                 } else if (!tcp_ca_needs_ecn(sk)) {
370                         /* ACK or retransmitted segment: clear ECT|CE */
371                         INET_ECN_dontxmit(sk);
372                 }
373                 if (tp->ecn_flags & TCP_ECN_DEMAND_CWR)
374                         th->ece = 1;
375         }
376 }
377 
378 /* Constructs common control bits of non-data skb. If SYN/FIN is present,
379  * auto increment end seqno.
380  */
381 static void tcp_init_nondata_skb(struct sk_buff *skb, u32 seq, u8 flags)
382 {
383         skb->ip_summed = CHECKSUM_PARTIAL;
384 
385         TCP_SKB_CB(skb)->tcp_flags = flags;
386         TCP_SKB_CB(skb)->sacked = 0;
387 
388         tcp_skb_pcount_set(skb, 1);
389 
390         TCP_SKB_CB(skb)->seq = seq;
391         if (flags & (TCPHDR_SYN | TCPHDR_FIN))
392                 seq++;
393         TCP_SKB_CB(skb)->end_seq = seq;
394 }
395 
396 static inline bool tcp_urg_mode(const struct tcp_sock *tp)
397 {
398         return tp->snd_una != tp->snd_up;
399 }
400 
401 #define OPTION_SACK_ADVERTISE   (1 << 0)
402 #define OPTION_TS               (1 << 1)
403 #define OPTION_MD5              (1 << 2)
404 #define OPTION_WSCALE           (1 << 3)
405 #define OPTION_FAST_OPEN_COOKIE (1 << 8)
406 #define OPTION_SMC              (1 << 9)
407 
408 static void smc_options_write(__be32 *ptr, u16 *options)
409 {
410 #if IS_ENABLED(CONFIG_SMC)
411         if (static_branch_unlikely(&tcp_have_smc)) {
412                 if (unlikely(OPTION_SMC & *options)) {
413                         *ptr++ = htonl((TCPOPT_NOP  << 24) |
414                                        (TCPOPT_NOP  << 16) |
415                                        (TCPOPT_EXP <<  8) |
416                                        (TCPOLEN_EXP_SMC_BASE));
417                         *ptr++ = htonl(TCPOPT_SMC_MAGIC);
418                 }
419         }
420 #endif
421 }
422 
423 struct tcp_out_options {
424         u16 options;            /* bit field of OPTION_* */
425         u16 mss;                /* 0 to disable */
426         u8 ws;                  /* window scale, 0 to disable */
427         u8 num_sack_blocks;     /* number of SACK blocks to include */
428         u8 hash_size;           /* bytes in hash_location */
429         __u8 *hash_location;    /* temporary pointer, overloaded */
430         __u32 tsval, tsecr;     /* need to include OPTION_TS */
431         struct tcp_fastopen_cookie *fastopen_cookie;    /* Fast open cookie */
432 };
433 
434 /* Write previously computed TCP options to the packet.
435  *
436  * Beware: Something in the Internet is very sensitive to the ordering of
437  * TCP options, we learned this through the hard way, so be careful here.
438  * Luckily we can at least blame others for their non-compliance but from
439  * inter-operability perspective it seems that we're somewhat stuck with
440  * the ordering which we have been using if we want to keep working with
441  * those broken things (not that it currently hurts anybody as there isn't
442  * particular reason why the ordering would need to be changed).
443  *
444  * At least SACK_PERM as the first option is known to lead to a disaster
445  * (but it may well be that other scenarios fail similarly).
446  */
447 static void tcp_options_write(__be32 *ptr, struct tcp_sock *tp,
448                               struct tcp_out_options *opts)
449 {
450         u16 options = opts->options;    /* mungable copy */
451 
452         if (unlikely(OPTION_MD5 & options)) {
453                 *ptr++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
454                                (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
455                 /* overload cookie hash location */
456                 opts->hash_location = (__u8 *)ptr;
457                 ptr += 4;
458         }
459 
460         if (unlikely(opts->mss)) {
461                 *ptr++ = htonl((TCPOPT_MSS << 24) |
462                                (TCPOLEN_MSS << 16) |
463                                opts->mss);
464         }
465 
466         if (likely(OPTION_TS & options)) {
467                 if (unlikely(OPTION_SACK_ADVERTISE & options)) {
468                         *ptr++ = htonl((TCPOPT_SACK_PERM << 24) |
469                                        (TCPOLEN_SACK_PERM << 16) |
470                                        (TCPOPT_TIMESTAMP << 8) |
471                                        TCPOLEN_TIMESTAMP);
472                         options &= ~OPTION_SACK_ADVERTISE;
473                 } else {
474                         *ptr++ = htonl((TCPOPT_NOP << 24) |
475                                        (TCPOPT_NOP << 16) |
476                                        (TCPOPT_TIMESTAMP << 8) |
477                                        TCPOLEN_TIMESTAMP);
478                 }
479                 *ptr++ = htonl(opts->tsval);
480                 *ptr++ = htonl(opts->tsecr);
481         }
482 
483         if (unlikely(OPTION_SACK_ADVERTISE & options)) {
484                 *ptr++ = htonl((TCPOPT_NOP << 24) |
485                                (TCPOPT_NOP << 16) |
486                                (TCPOPT_SACK_PERM << 8) |
487                                TCPOLEN_SACK_PERM);
488         }
489 
490         if (unlikely(OPTION_WSCALE & options)) {
491                 *ptr++ = htonl((TCPOPT_NOP << 24) |
492                                (TCPOPT_WINDOW << 16) |
493                                (TCPOLEN_WINDOW << 8) |
494                                opts->ws);
495         }
496 
497         if (unlikely(opts->num_sack_blocks)) {
498                 struct tcp_sack_block *sp = tp->rx_opt.dsack ?
499                         tp->duplicate_sack : tp->selective_acks;
500                 int this_sack;
501 
502                 *ptr++ = htonl((TCPOPT_NOP  << 24) |
503                                (TCPOPT_NOP  << 16) |
504                                (TCPOPT_SACK <<  8) |
505                                (TCPOLEN_SACK_BASE + (opts->num_sack_blocks *
506                                                      TCPOLEN_SACK_PERBLOCK)));
507 
508                 for (this_sack = 0; this_sack < opts->num_sack_blocks;
509                      ++this_sack) {
510                         *ptr++ = htonl(sp[this_sack].start_seq);
511                         *ptr++ = htonl(sp[this_sack].end_seq);
512                 }
513 
514                 tp->rx_opt.dsack = 0;
515         }
516 
517         if (unlikely(OPTION_FAST_OPEN_COOKIE & options)) {
518                 struct tcp_fastopen_cookie *foc = opts->fastopen_cookie;
519                 u8 *p = (u8 *)ptr;
520                 u32 len; /* Fast Open option length */
521 
522                 if (foc->exp) {
523                         len = TCPOLEN_EXP_FASTOPEN_BASE + foc->len;
524                         *ptr = htonl((TCPOPT_EXP << 24) | (len << 16) |
525                                      TCPOPT_FASTOPEN_MAGIC);
526                         p += TCPOLEN_EXP_FASTOPEN_BASE;
527                 } else {
528                         len = TCPOLEN_FASTOPEN_BASE + foc->len;
529                         *p++ = TCPOPT_FASTOPEN;
530                         *p++ = len;
531                 }
532 
533                 memcpy(p, foc->val, foc->len);
534                 if ((len & 3) == 2) {
535                         p[foc->len] = TCPOPT_NOP;
536                         p[foc->len + 1] = TCPOPT_NOP;
537                 }
538                 ptr += (len + 3) >> 2;
539         }
540 
541         smc_options_write(ptr, &options);
542 }
543 
544 static void smc_set_option(const struct tcp_sock *tp,
545                            struct tcp_out_options *opts,
546                            unsigned int *remaining)
547 {
548 #if IS_ENABLED(CONFIG_SMC)
549         if (static_branch_unlikely(&tcp_have_smc)) {
550                 if (tp->syn_smc) {
551                         if (*remaining >= TCPOLEN_EXP_SMC_BASE_ALIGNED) {
552                                 opts->options |= OPTION_SMC;
553                                 *remaining -= TCPOLEN_EXP_SMC_BASE_ALIGNED;
554                         }
555                 }
556         }
557 #endif
558 }
559 
560 static void smc_set_option_cond(const struct tcp_sock *tp,
561                                 const struct inet_request_sock *ireq,
562                                 struct tcp_out_options *opts,
563                                 unsigned int *remaining)
564 {
565 #if IS_ENABLED(CONFIG_SMC)
566         if (static_branch_unlikely(&tcp_have_smc)) {
567                 if (tp->syn_smc && ireq->smc_ok) {
568                         if (*remaining >= TCPOLEN_EXP_SMC_BASE_ALIGNED) {
569                                 opts->options |= OPTION_SMC;
570                                 *remaining -= TCPOLEN_EXP_SMC_BASE_ALIGNED;
571                         }
572                 }
573         }
574 #endif
575 }
576 
577 /* Compute TCP options for SYN packets. This is not the final
578  * network wire format yet.
579  */
580 static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb,
581                                 struct tcp_out_options *opts,
582                                 struct tcp_md5sig_key **md5)
583 {
584         struct tcp_sock *tp = tcp_sk(sk);
585         unsigned int remaining = MAX_TCP_OPTION_SPACE;
586         struct tcp_fastopen_request *fastopen = tp->fastopen_req;
587 
588 #ifdef CONFIG_TCP_MD5SIG
589         *md5 = tp->af_specific->md5_lookup(sk, sk);
590         if (*md5) {
591                 opts->options |= OPTION_MD5;
592                 remaining -= TCPOLEN_MD5SIG_ALIGNED;
593         }
594 #else
595         *md5 = NULL;
596 #endif
597 
598         /* We always get an MSS option.  The option bytes which will be seen in
599          * normal data packets should timestamps be used, must be in the MSS
600          * advertised.  But we subtract them from tp->mss_cache so that
601          * calculations in tcp_sendmsg are simpler etc.  So account for this
602          * fact here if necessary.  If we don't do this correctly, as a
603          * receiver we won't recognize data packets as being full sized when we
604          * should, and thus we won't abide by the delayed ACK rules correctly.
605          * SACKs don't matter, we never delay an ACK when we have any of those
606          * going out.  */
607         opts->mss = tcp_advertise_mss(sk);
608         remaining -= TCPOLEN_MSS_ALIGNED;
609 
610         if (likely(sock_net(sk)->ipv4.sysctl_tcp_timestamps && !*md5)) {
611                 opts->options |= OPTION_TS;
612                 opts->tsval = tcp_skb_timestamp(skb) + tp->tsoffset;
613                 opts->tsecr = tp->rx_opt.ts_recent;
614                 remaining -= TCPOLEN_TSTAMP_ALIGNED;
615         }
616         if (likely(sock_net(sk)->ipv4.sysctl_tcp_window_scaling)) {
617                 opts->ws = tp->rx_opt.rcv_wscale;
618                 opts->options |= OPTION_WSCALE;
619                 remaining -= TCPOLEN_WSCALE_ALIGNED;
620         }
621         if (likely(sock_net(sk)->ipv4.sysctl_tcp_sack)) {
622                 opts->options |= OPTION_SACK_ADVERTISE;
623                 if (unlikely(!(OPTION_TS & opts->options)))
624                         remaining -= TCPOLEN_SACKPERM_ALIGNED;
625         }
626 
627         if (fastopen && fastopen->cookie.len >= 0) {
628                 u32 need = fastopen->cookie.len;
629 
630                 need += fastopen->cookie.exp ? TCPOLEN_EXP_FASTOPEN_BASE :
631                                                TCPOLEN_FASTOPEN_BASE;
632                 need = (need + 3) & ~3U;  /* Align to 32 bits */
633                 if (remaining >= need) {
634                         opts->options |= OPTION_FAST_OPEN_COOKIE;
635                         opts->fastopen_cookie = &fastopen->cookie;
636                         remaining -= need;
637                         tp->syn_fastopen = 1;
638                         tp->syn_fastopen_exp = fastopen->cookie.exp ? 1 : 0;
639                 }
640         }
641 
642         smc_set_option(tp, opts, &remaining);
643 
644         return MAX_TCP_OPTION_SPACE - remaining;
645 }
646 
647 /* Set up TCP options for SYN-ACKs. */
648 static unsigned int tcp_synack_options(const struct sock *sk,
649                                        struct request_sock *req,
650                                        unsigned int mss, struct sk_buff *skb,
651                                        struct tcp_out_options *opts,
652                                        const struct tcp_md5sig_key *md5,
653                                        struct tcp_fastopen_cookie *foc)
654 {
655         struct inet_request_sock *ireq = inet_rsk(req);
656         unsigned int remaining = MAX_TCP_OPTION_SPACE;
657 
658 #ifdef CONFIG_TCP_MD5SIG
659         if (md5) {
660                 opts->options |= OPTION_MD5;
661                 remaining -= TCPOLEN_MD5SIG_ALIGNED;
662 
663                 /* We can't fit any SACK blocks in a packet with MD5 + TS
664                  * options. There was discussion about disabling SACK
665                  * rather than TS in order to fit in better with old,
666                  * buggy kernels, but that was deemed to be unnecessary.
667                  */
668                 ireq->tstamp_ok &= !ireq->sack_ok;
669         }
670 #endif
671 
672         /* We always send an MSS option. */
673         opts->mss = mss;
674         remaining -= TCPOLEN_MSS_ALIGNED;
675 
676         if (likely(ireq->wscale_ok)) {
677                 opts->ws = ireq->rcv_wscale;
678                 opts->options |= OPTION_WSCALE;
679                 remaining -= TCPOLEN_WSCALE_ALIGNED;
680         }
681         if (likely(ireq->tstamp_ok)) {
682                 opts->options |= OPTION_TS;
683                 opts->tsval = tcp_skb_timestamp(skb) + tcp_rsk(req)->ts_off;
684                 opts->tsecr = req->ts_recent;
685                 remaining -= TCPOLEN_TSTAMP_ALIGNED;
686         }
687         if (likely(ireq->sack_ok)) {
688                 opts->options |= OPTION_SACK_ADVERTISE;
689                 if (unlikely(!ireq->tstamp_ok))
690                         remaining -= TCPOLEN_SACKPERM_ALIGNED;
691         }
692         if (foc != NULL && foc->len >= 0) {
693                 u32 need = foc->len;
694 
695                 need += foc->exp ? TCPOLEN_EXP_FASTOPEN_BASE :
696                                    TCPOLEN_FASTOPEN_BASE;
697                 need = (need + 3) & ~3U;  /* Align to 32 bits */
698                 if (remaining >= need) {
699                         opts->options |= OPTION_FAST_OPEN_COOKIE;
700                         opts->fastopen_cookie = foc;
701                         remaining -= need;
702                 }
703         }
704 
705         smc_set_option_cond(tcp_sk(sk), ireq, opts, &remaining);
706 
707         return MAX_TCP_OPTION_SPACE - remaining;
708 }
709 
710 /* Compute TCP options for ESTABLISHED sockets. This is not the
711  * final wire format yet.
712  */
713 static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb,
714                                         struct tcp_out_options *opts,
715                                         struct tcp_md5sig_key **md5)
716 {
717         struct tcp_sock *tp = tcp_sk(sk);
718         unsigned int size = 0;
719         unsigned int eff_sacks;
720 
721         opts->options = 0;
722 
723 #ifdef CONFIG_TCP_MD5SIG
724         *md5 = tp->af_specific->md5_lookup(sk, sk);
725         if (unlikely(*md5)) {
726                 opts->options |= OPTION_MD5;
727                 size += TCPOLEN_MD5SIG_ALIGNED;
728         }
729 #else
730         *md5 = NULL;
731 #endif
732 
733         if (likely(tp->rx_opt.tstamp_ok)) {
734                 opts->options |= OPTION_TS;
735                 opts->tsval = skb ? tcp_skb_timestamp(skb) + tp->tsoffset : 0;
736                 opts->tsecr = tp->rx_opt.ts_recent;
737                 size += TCPOLEN_TSTAMP_ALIGNED;
738         }
739 
740         eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack;
741         if (unlikely(eff_sacks)) {
742                 const unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
743                 opts->num_sack_blocks =
744                         min_t(unsigned int, eff_sacks,
745                               (remaining - TCPOLEN_SACK_BASE_ALIGNED) /
746                               TCPOLEN_SACK_PERBLOCK);
747                 size += TCPOLEN_SACK_BASE_ALIGNED +
748                         opts->num_sack_blocks * TCPOLEN_SACK_PERBLOCK;
749         }
750 
751         return size;
752 }
753 
754 
755 /* TCP SMALL QUEUES (TSQ)
756  *
757  * TSQ goal is to keep small amount of skbs per tcp flow in tx queues (qdisc+dev)
758  * to reduce RTT and bufferbloat.
759  * We do this using a special skb destructor (tcp_wfree).
760  *
761  * Its important tcp_wfree() can be replaced by sock_wfree() in the event skb
762  * needs to be reallocated in a driver.
763  * The invariant being skb->truesize subtracted from sk->sk_wmem_alloc
764  *
765  * Since transmit from skb destructor is forbidden, we use a tasklet
766  * to process all sockets that eventually need to send more skbs.
767  * We use one tasklet per cpu, with its own queue of sockets.
768  */
769 struct tsq_tasklet {
770         struct tasklet_struct   tasklet;
771         struct list_head        head; /* queue of tcp sockets */
772 };
773 static DEFINE_PER_CPU(struct tsq_tasklet, tsq_tasklet);
774 
775 static void tcp_tsq_handler(struct sock *sk)
776 {
777         if ((1 << sk->sk_state) &
778             (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_CLOSING |
779              TCPF_CLOSE_WAIT  | TCPF_LAST_ACK)) {
780                 struct tcp_sock *tp = tcp_sk(sk);
781 
782                 if (tp->lost_out > tp->retrans_out &&
783                     tp->snd_cwnd > tcp_packets_in_flight(tp)) {
784                         tcp_mstamp_refresh(tp);
785                         tcp_xmit_retransmit_queue(sk);
786                 }
787 
788                 tcp_write_xmit(sk, tcp_current_mss(sk), tp->nonagle,
789                                0, GFP_ATOMIC);
790         }
791 }
792 /*
793  * One tasklet per cpu tries to send more skbs.
794  * We run in tasklet context but need to disable irqs when
795  * transferring tsq->head because tcp_wfree() might
796  * interrupt us (non NAPI drivers)
797  */
798 static void tcp_tasklet_func(unsigned long data)
799 {
800         struct tsq_tasklet *tsq = (struct tsq_tasklet *)data;
801         LIST_HEAD(list);
802         unsigned long flags;
803         struct list_head *q, *n;
804         struct tcp_sock *tp;
805         struct sock *sk;
806 
807         local_irq_save(flags);
808         list_splice_init(&tsq->head, &list);
809         local_irq_restore(flags);
810 
811         list_for_each_safe(q, n, &list) {
812                 tp = list_entry(q, struct tcp_sock, tsq_node);
813                 list_del(&tp->tsq_node);
814 
815                 sk = (struct sock *)tp;
816                 smp_mb__before_atomic();
817                 clear_bit(TSQ_QUEUED, &sk->sk_tsq_flags);
818 
819                 if (!sk->sk_lock.owned &&
820                     test_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags)) {
821                         bh_lock_sock(sk);
822                         if (!sock_owned_by_user(sk)) {
823                                 clear_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags);
824                                 tcp_tsq_handler(sk);
825                         }
826                         bh_unlock_sock(sk);
827                 }
828 
829                 sk_free(sk);
830         }
831 }
832 
833 #define TCP_DEFERRED_ALL (TCPF_TSQ_DEFERRED |           \
834                           TCPF_WRITE_TIMER_DEFERRED |   \
835                           TCPF_DELACK_TIMER_DEFERRED |  \
836                           TCPF_MTU_REDUCED_DEFERRED)
837 /**
838  * tcp_release_cb - tcp release_sock() callback
839  * @sk: socket
840  *
841  * called from release_sock() to perform protocol dependent
842  * actions before socket release.
843  */
844 void tcp_release_cb(struct sock *sk)
845 {
846         unsigned long flags, nflags;
847 
848         /* perform an atomic operation only if at least one flag is set */
849         do {
850                 flags = sk->sk_tsq_flags;
851                 if (!(flags & TCP_DEFERRED_ALL))
852                         return;
853                 nflags = flags & ~TCP_DEFERRED_ALL;
854         } while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags);
855 
856         if (flags & TCPF_TSQ_DEFERRED)
857                 tcp_tsq_handler(sk);
858 
859         /* Here begins the tricky part :
860          * We are called from release_sock() with :
861          * 1) BH disabled
862          * 2) sk_lock.slock spinlock held
863          * 3) socket owned by us (sk->sk_lock.owned == 1)
864          *
865          * But following code is meant to be called from BH handlers,
866          * so we should keep BH disabled, but early release socket ownership
867          */
868         sock_release_ownership(sk);
869 
870         if (flags & TCPF_WRITE_TIMER_DEFERRED) {
871                 tcp_write_timer_handler(sk);
872                 __sock_put(sk);
873         }
874         if (flags & TCPF_DELACK_TIMER_DEFERRED) {
875                 tcp_delack_timer_handler(sk);
876                 __sock_put(sk);
877         }
878         if (flags & TCPF_MTU_REDUCED_DEFERRED) {
879                 inet_csk(sk)->icsk_af_ops->mtu_reduced(sk);
880                 __sock_put(sk);
881         }
882 }
883 EXPORT_SYMBOL(tcp_release_cb);
884 
885 void __init tcp_tasklet_init(void)
886 {
887         int i;
888 
889         for_each_possible_cpu(i) {
890                 struct tsq_tasklet *tsq = &per_cpu(tsq_tasklet, i);
891 
892                 INIT_LIST_HEAD(&tsq->head);
893                 tasklet_init(&tsq->tasklet,
894                              tcp_tasklet_func,
895                              (unsigned long)tsq);
896         }
897 }
898 
899 /*
900  * Write buffer destructor automatically called from kfree_skb.
901  * We can't xmit new skbs from this context, as we might already
902  * hold qdisc lock.
903  */
904 void tcp_wfree(struct sk_buff *skb)
905 {
906         struct sock *sk = skb->sk;
907         struct tcp_sock *tp = tcp_sk(sk);
908         unsigned long flags, nval, oval;
909 
910         /* Keep one reference on sk_wmem_alloc.
911          * Will be released by sk_free() from here or tcp_tasklet_func()
912          */
913         WARN_ON(refcount_sub_and_test(skb->truesize - 1, &sk->sk_wmem_alloc));
914 
915         /* If this softirq is serviced by ksoftirqd, we are likely under stress.
916          * Wait until our queues (qdisc + devices) are drained.
917          * This gives :
918          * - less callbacks to tcp_write_xmit(), reducing stress (batches)
919          * - chance for incoming ACK (processed by another cpu maybe)
920          *   to migrate this flow (skb->ooo_okay will be eventually set)
921          */
922         if (refcount_read(&sk->sk_wmem_alloc) >= SKB_TRUESIZE(1) && this_cpu_ksoftirqd() == current)
923                 goto out;
924 
925         for (oval = READ_ONCE(sk->sk_tsq_flags);; oval = nval) {
926                 struct tsq_tasklet *tsq;
927                 bool empty;
928 
929                 if (!(oval & TSQF_THROTTLED) || (oval & TSQF_QUEUED))
930                         goto out;
931 
932                 nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED | TCPF_TSQ_DEFERRED;
933                 nval = cmpxchg(&sk->sk_tsq_flags, oval, nval);
934                 if (nval != oval)
935                         continue;
936 
937                 /* queue this socket to tasklet queue */
938                 local_irq_save(flags);
939                 tsq = this_cpu_ptr(&tsq_tasklet);
940                 empty = list_empty(&tsq->head);
941                 list_add(&tp->tsq_node, &tsq->head);
942                 if (empty)
943                         tasklet_schedule(&tsq->tasklet);
944                 local_irq_restore(flags);
945                 return;
946         }
947 out:
948         sk_free(sk);
949 }
950 
951 /* Note: Called under hard irq.
952  * We can not call TCP stack right away.
953  */
954 enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer)
955 {
956         struct tcp_sock *tp = container_of(timer, struct tcp_sock, pacing_timer);
957         struct sock *sk = (struct sock *)tp;
958         unsigned long nval, oval;
959 
960         for (oval = READ_ONCE(sk->sk_tsq_flags);; oval = nval) {
961                 struct tsq_tasklet *tsq;
962                 bool empty;
963 
964                 if (oval & TSQF_QUEUED)
965                         break;
966 
967                 nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED | TCPF_TSQ_DEFERRED;
968                 nval = cmpxchg(&sk->sk_tsq_flags, oval, nval);
969                 if (nval != oval)
970                         continue;
971 
972                 if (!refcount_inc_not_zero(&sk->sk_wmem_alloc))
973                         break;
974                 /* queue this socket to tasklet queue */
975                 tsq = this_cpu_ptr(&tsq_tasklet);
976                 empty = list_empty(&tsq->head);
977                 list_add(&tp->tsq_node, &tsq->head);
978                 if (empty)
979                         tasklet_schedule(&tsq->tasklet);
980                 break;
981         }
982         return HRTIMER_NORESTART;
983 }
984 
985 /* BBR congestion control needs pacing.
986  * Same remark for SO_MAX_PACING_RATE.
987  * sch_fq packet scheduler is efficiently handling pacing,
988  * but is not always installed/used.
989  * Return true if TCP stack should pace packets itself.
990  */
991 static bool tcp_needs_internal_pacing(const struct sock *sk)
992 {
993         return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED;
994 }
995 
996 static void tcp_internal_pacing(struct sock *sk, const struct sk_buff *skb)
997 {
998         u64 len_ns;
999         u32 rate;
1000 
1001         if (!tcp_needs_internal_pacing(sk))
1002                 return;
1003         rate = sk->sk_pacing_rate;
1004         if (!rate || rate == ~0U)
1005                 return;
1006 
1007         /* Should account for header sizes as sch_fq does,
1008          * but lets make things simple.
1009          */
1010         len_ns = (u64)skb->len * NSEC_PER_SEC;
1011         do_div(len_ns, rate);
1012         hrtimer_start(&tcp_sk(sk)->pacing_timer,
1013                       ktime_add_ns(ktime_get(), len_ns),
1014                       HRTIMER_MODE_ABS_PINNED);
1015 }
1016 
1017 static void tcp_update_skb_after_send(struct tcp_sock *tp, struct sk_buff *skb)
1018 {
1019         skb->skb_mstamp = tp->tcp_mstamp;
1020         list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue);
1021 }
1022 
1023 /* This routine actually transmits TCP packets queued in by
1024  * tcp_do_sendmsg().  This is used by both the initial
1025  * transmission and possible later retransmissions.
1026  * All SKB's seen here are completely headerless.  It is our
1027  * job to build the TCP header, and pass the packet down to
1028  * IP so it can do the same plus pass the packet off to the
1029  * device.
1030  *
1031  * We are working here with either a clone of the original
1032  * SKB, or a fresh unique copy made by the retransmit engine.
1033  */
1034 static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
1035                             gfp_t gfp_mask)
1036 {
1037         const struct inet_connection_sock *icsk = inet_csk(sk);
1038         struct inet_sock *inet;
1039         struct tcp_sock *tp;
1040         struct tcp_skb_cb *tcb;
1041         struct tcp_out_options opts;
1042         unsigned int tcp_options_size, tcp_header_size;
1043         struct sk_buff *oskb = NULL;
1044         struct tcp_md5sig_key *md5;
1045         struct tcphdr *th;
1046         int err;
1047 
1048         BUG_ON(!skb || !tcp_skb_pcount(skb));
1049         tp = tcp_sk(sk);
1050 
1051         if (clone_it) {
1052                 TCP_SKB_CB(skb)->tx.in_flight = TCP_SKB_CB(skb)->end_seq
1053                         - tp->snd_una;
1054                 oskb = skb;
1055 
1056                 tcp_skb_tsorted_save(oskb) {
1057                         if (unlikely(skb_cloned(oskb)))
1058                                 skb = pskb_copy(oskb, gfp_mask);
1059                         else
1060                                 skb = skb_clone(oskb, gfp_mask);
1061                 } tcp_skb_tsorted_restore(oskb);
1062 
1063                 if (unlikely(!skb))
1064                         return -ENOBUFS;
1065         }
1066         skb->skb_mstamp = tp->tcp_mstamp;
1067 
1068         inet = inet_sk(sk);
1069         tcb = TCP_SKB_CB(skb);
1070         memset(&opts, 0, sizeof(opts));
1071 
1072         if (unlikely(tcb->tcp_flags & TCPHDR_SYN))
1073                 tcp_options_size = tcp_syn_options(sk, skb, &opts, &md5);
1074         else
1075                 tcp_options_size = tcp_established_options(sk, skb, &opts,
1076                                                            &md5);
1077         tcp_header_size = tcp_options_size + sizeof(struct tcphdr);
1078 
1079         /* if no packet is in qdisc/device queue, then allow XPS to select
1080          * another queue. We can be called from tcp_tsq_handler()
1081          * which holds one reference to sk_wmem_alloc.
1082          *
1083          * TODO: Ideally, in-flight pure ACK packets should not matter here.
1084          * One way to get this would be to set skb->truesize = 2 on them.
1085          */
1086         skb->ooo_okay = sk_wmem_alloc_get(sk) < SKB_TRUESIZE(1);
1087 
1088         /* If we had to use memory reserve to allocate this skb,
1089          * this might cause drops if packet is looped back :
1090          * Other socket might not have SOCK_MEMALLOC.
1091          * Packets not looped back do not care about pfmemalloc.
1092          */
1093         skb->pfmemalloc = 0;
1094 
1095         skb_push(skb, tcp_header_size);
1096         skb_reset_transport_header(skb);
1097 
1098         skb_orphan(skb);
1099         skb->sk = sk;
1100         skb->destructor = skb_is_tcp_pure_ack(skb) ? __sock_wfree : tcp_wfree;
1101         skb_set_hash_from_sk(skb, sk);
1102         refcount_add(skb->truesize, &sk->sk_wmem_alloc);
1103 
1104         skb_set_dst_pending_confirm(skb, sk->sk_dst_pending_confirm);
1105 
1106         /* Build TCP header and checksum it. */
1107         th = (struct tcphdr *)skb->data;
1108         th->source              = inet->inet_sport;
1109         th->dest                = inet->inet_dport;
1110         th->seq                 = htonl(tcb->seq);
1111         th->ack_seq             = htonl(tp->rcv_nxt);
1112         *(((__be16 *)th) + 6)   = htons(((tcp_header_size >> 2) << 12) |
1113                                         tcb->tcp_flags);
1114 
1115         th->check               = 0;
1116         th->urg_ptr             = 0;
1117 
1118         /* The urg_mode check is necessary during a below snd_una win probe */
1119         if (unlikely(tcp_urg_mode(tp) && before(tcb->seq, tp->snd_up))) {
1120                 if (before(tp->snd_up, tcb->seq + 0x10000)) {
1121                         th->urg_ptr = htons(tp->snd_up - tcb->seq);
1122                         th->urg = 1;
1123                 } else if (after(tcb->seq + 0xFFFF, tp->snd_nxt)) {
1124                         th->urg_ptr = htons(0xFFFF);
1125                         th->urg = 1;
1126                 }
1127         }
1128 
1129         tcp_options_write((__be32 *)(th + 1), tp, &opts);
1130         skb_shinfo(skb)->gso_type = sk->sk_gso_type;
1131         if (likely(!(tcb->tcp_flags & TCPHDR_SYN))) {
1132                 th->window      = htons(tcp_select_window(sk));
1133                 tcp_ecn_send(sk, skb, th, tcp_header_size);
1134         } else {
1135                 /* RFC1323: The window in SYN & SYN/ACK segments
1136                  * is never scaled.
1137                  */
1138                 th->window      = htons(min(tp->rcv_wnd, 65535U));
1139         }
1140 #ifdef CONFIG_TCP_MD5SIG
1141         /* Calculate the MD5 hash, as we have all we need now */
1142         if (md5) {
1143                 sk_nocaps_add(sk, NETIF_F_GSO_MASK);
1144                 tp->af_specific->calc_md5_hash(opts.hash_location,
1145                                                md5, sk, skb);
1146         }
1147 #endif
1148 
1149         icsk->icsk_af_ops->send_check(sk, skb);
1150 
1151         if (likely(tcb->tcp_flags & TCPHDR_ACK))
1152                 tcp_event_ack_sent(sk, tcp_skb_pcount(skb));
1153 
1154         if (skb->len != tcp_header_size) {
1155                 tcp_event_data_sent(tp, sk);
1156                 tp->data_segs_out += tcp_skb_pcount(skb);
1157                 tcp_internal_pacing(sk, skb);
1158         }
1159 
1160         if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq)
1161                 TCP_ADD_STATS(sock_net(sk), TCP_MIB_OUTSEGS,
1162                               tcp_skb_pcount(skb));
1163 
1164         tp->segs_out += tcp_skb_pcount(skb);
1165         /* OK, its time to fill skb_shinfo(skb)->gso_{segs|size} */
1166         skb_shinfo(skb)->gso_segs = tcp_skb_pcount(skb);
1167         skb_shinfo(skb)->gso_size = tcp_skb_mss(skb);
1168 
1169         /* Our usage of tstamp should remain private */
1170         skb->tstamp = 0;
1171 
1172         /* Cleanup our debris for IP stacks */
1173         memset(skb->cb, 0, max(sizeof(struct inet_skb_parm),
1174                                sizeof(struct inet6_skb_parm)));
1175 
1176         err = icsk->icsk_af_ops->queue_xmit(sk, skb, &inet->cork.fl);
1177 
1178         if (unlikely(err > 0)) {
1179                 tcp_enter_cwr(sk);
1180                 err = net_xmit_eval(err);
1181         }
1182         if (!err && oskb) {
1183                 tcp_update_skb_after_send(tp, oskb);
1184                 tcp_rate_skb_sent(sk, oskb);
1185         }
1186         return err;
1187 }
1188 
1189 /* This routine just queues the buffer for sending.
1190  *
1191  * NOTE: probe0 timer is not checked, do not forget tcp_push_pending_frames,
1192  * otherwise socket can stall.
1193  */
1194 static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb)
1195 {
1196         struct tcp_sock *tp = tcp_sk(sk);
1197 
1198         /* Advance write_seq and place onto the write_queue. */
1199         tp->write_seq = TCP_SKB_CB(skb)->end_seq;
1200         __skb_header_release(skb);
1201         tcp_add_write_queue_tail(sk, skb);
1202         sk->sk_wmem_queued += skb->truesize;
1203         sk_mem_charge(sk, skb->truesize);
1204 }
1205 
1206 /* Initialize TSO segments for a packet. */
1207 static void tcp_set_skb_tso_segs(struct sk_buff *skb, unsigned int mss_now)
1208 {
1209         if (skb->len <= mss_now || skb->ip_summed == CHECKSUM_NONE) {
1210                 /* Avoid the costly divide in the normal
1211                  * non-TSO case.
1212                  */
1213                 tcp_skb_pcount_set(skb, 1);
1214                 TCP_SKB_CB(skb)->tcp_gso_size = 0;
1215         } else {
1216                 tcp_skb_pcount_set(skb, DIV_ROUND_UP(skb->len, mss_now));
1217                 TCP_SKB_CB(skb)->tcp_gso_size = mss_now;
1218         }
1219 }
1220 
1221 /* Pcount in the middle of the write queue got changed, we need to do various
1222  * tweaks to fix counters
1223  */
1224 static void tcp_adjust_pcount(struct sock *sk, const struct sk_buff *skb, int decr)
1225 {
1226         struct tcp_sock *tp = tcp_sk(sk);
1227 
1228         tp->packets_out -= decr;
1229 
1230         if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
1231                 tp->sacked_out -= decr;
1232         if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS)
1233                 tp->retrans_out -= decr;
1234         if (TCP_SKB_CB(skb)->sacked & TCPCB_LOST)
1235                 tp->lost_out -= decr;
1236 
1237         /* Reno case is special. Sigh... */
1238         if (tcp_is_reno(tp) && decr > 0)
1239                 tp->sacked_out -= min_t(u32, tp->sacked_out, decr);
1240 
1241         if (tp->lost_skb_hint &&
1242             before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(tp->lost_skb_hint)->seq) &&
1243             (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED))
1244                 tp->lost_cnt_hint -= decr;
1245 
1246         tcp_verify_left_out(tp);
1247 }
1248 
1249 static bool tcp_has_tx_tstamp(const struct sk_buff *skb)
1250 {
1251         return TCP_SKB_CB(skb)->txstamp_ack ||
1252                 (skb_shinfo(skb)->tx_flags & SKBTX_ANY_TSTAMP);
1253 }
1254 
1255 static void tcp_fragment_tstamp(struct sk_buff *skb, struct sk_buff *skb2)
1256 {
1257         struct skb_shared_info *shinfo = skb_shinfo(skb);
1258 
1259         if (unlikely(tcp_has_tx_tstamp(skb)) &&
1260             !before(shinfo->tskey, TCP_SKB_CB(skb2)->seq)) {
1261                 struct skb_shared_info *shinfo2 = skb_shinfo(skb2);
1262                 u8 tsflags = shinfo->tx_flags & SKBTX_ANY_TSTAMP;
1263 
1264                 shinfo->tx_flags &= ~tsflags;
1265                 shinfo2->tx_flags |= tsflags;
1266                 swap(shinfo->tskey, shinfo2->tskey);
1267                 TCP_SKB_CB(skb2)->txstamp_ack = TCP_SKB_CB(skb)->txstamp_ack;
1268                 TCP_SKB_CB(skb)->txstamp_ack = 0;
1269         }
1270 }
1271 
1272 static void tcp_skb_fragment_eor(struct sk_buff *skb, struct sk_buff *skb2)
1273 {
1274         TCP_SKB_CB(skb2)->eor = TCP_SKB_CB(skb)->eor;
1275         TCP_SKB_CB(skb)->eor = 0;
1276 }
1277 
1278 /* Insert buff after skb on the write or rtx queue of sk.  */
1279 static void tcp_insert_write_queue_after(struct sk_buff *skb,
1280                                          struct sk_buff *buff,
1281                                          struct sock *sk,
1282                                          enum tcp_queue tcp_queue)
1283 {
1284         if (tcp_queue == TCP_FRAG_IN_WRITE_QUEUE)
1285                 __skb_queue_after(&sk->sk_write_queue, skb, buff);
1286         else
1287                 tcp_rbtree_insert(&sk->tcp_rtx_queue, buff);
1288 }
1289 
1290 /* Function to create two new TCP segments.  Shrinks the given segment
1291  * to the specified size and appends a new segment with the rest of the
1292  * packet to the list.  This won't be called frequently, I hope.
1293  * Remember, these are still headerless SKBs at this point.
1294  */
1295 int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
1296                  struct sk_buff *skb, u32 len,
1297                  unsigned int mss_now, gfp_t gfp)
1298 {
1299         struct tcp_sock *tp = tcp_sk(sk);
1300         struct sk_buff *buff;
1301         int nsize, old_factor;
1302         int nlen;
1303         u8 flags;
1304 
1305         if (WARN_ON(len > skb->len))
1306                 return -EINVAL;
1307 
1308         nsize = skb_headlen(skb) - len;
1309         if (nsize < 0)
1310                 nsize = 0;
1311 
1312         if (skb_unclone(skb, gfp))
1313                 return -ENOMEM;
1314 
1315         /* Get a new skb... force flag on. */
1316         buff = sk_stream_alloc_skb(sk, nsize, gfp, true);
1317         if (!buff)
1318                 return -ENOMEM; /* We'll just try again later. */
1319 
1320         sk->sk_wmem_queued += buff->truesize;
1321         sk_mem_charge(sk, buff->truesize);
1322         nlen = skb->len - len - nsize;
1323         buff->truesize += nlen;
1324         skb->truesize -= nlen;
1325 
1326         /* Correct the sequence numbers. */
1327         TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
1328         TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
1329         TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
1330 
1331         /* PSH and FIN should only be set in the second packet. */
1332         flags = TCP_SKB_CB(skb)->tcp_flags;
1333         TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
1334         TCP_SKB_CB(buff)->tcp_flags = flags;
1335         TCP_SKB_CB(buff)->sacked = TCP_SKB_CB(skb)->sacked;
1336         tcp_skb_fragment_eor(skb, buff);
1337 
1338         if (!skb_shinfo(skb)->nr_frags && skb->ip_summed != CHECKSUM_PARTIAL) {
1339                 /* Copy and checksum data tail into the new buffer. */
1340                 buff->csum = csum_partial_copy_nocheck(skb->data + len,
1341                                                        skb_put(buff, nsize),
1342                                                        nsize, 0);
1343 
1344                 skb_trim(skb, len);
1345 
1346                 skb->csum = csum_block_sub(skb->csum, buff->csum, len);
1347         } else {
1348                 skb->ip_summed = CHECKSUM_PARTIAL;
1349                 skb_split(skb, buff, len);
1350         }
1351 
1352         buff->ip_summed = skb->ip_summed;
1353 
1354         buff->tstamp = skb->tstamp;
1355         tcp_fragment_tstamp(skb, buff);
1356 
1357         old_factor = tcp_skb_pcount(skb);
1358 
1359         /* Fix up tso_factor for both original and new SKB.  */
1360         tcp_set_skb_tso_segs(skb, mss_now);
1361         tcp_set_skb_tso_segs(buff, mss_now);
1362 
1363         /* Update delivered info for the new segment */
1364         TCP_SKB_CB(buff)->tx = TCP_SKB_CB(skb)->tx;
1365 
1366         /* If this packet has been sent out already, we must
1367          * adjust the various packet counters.
1368          */
1369         if (!before(tp->snd_nxt, TCP_SKB_CB(buff)->end_seq)) {
1370                 int diff = old_factor - tcp_skb_pcount(skb) -
1371                         tcp_skb_pcount(buff);
1372 
1373                 if (diff)
1374                         tcp_adjust_pcount(sk, skb, diff);
1375         }
1376 
1377         /* Link BUFF into the send queue. */
1378         __skb_header_release(buff);
1379         tcp_insert_write_queue_after(skb, buff, sk, tcp_queue);
1380         if (tcp_queue == TCP_FRAG_IN_RTX_QUEUE)
1381                 list_add(&buff->tcp_tsorted_anchor, &skb->tcp_tsorted_anchor);
1382 
1383         return 0;
1384 }
1385 
1386 /* This is similar to __pskb_pull_tail(). The difference is that pulled
1387  * data is not copied, but immediately discarded.
1388  */
1389 static int __pskb_trim_head(struct sk_buff *skb, int len)
1390 {
1391         struct skb_shared_info *shinfo;
1392         int i, k, eat;
1393 
1394         eat = min_t(int, len, skb_headlen(skb));
1395         if (eat) {
1396                 __skb_pull(skb, eat);
1397                 len -= eat;
1398                 if (!len)
1399                         return 0;
1400         }
1401         eat = len;
1402         k = 0;
1403         shinfo = skb_shinfo(skb);
1404         for (i = 0; i < shinfo->nr_frags; i++) {
1405                 int size = skb_frag_size(&shinfo->frags[i]);
1406 
1407                 if (size <= eat) {
1408                         skb_frag_unref(skb, i);
1409                         eat -= size;
1410                 } else {
1411                         shinfo->frags[k] = shinfo->frags[i];
1412                         if (eat) {
1413                                 shinfo->frags[k].page_offset += eat;
1414                                 skb_frag_size_sub(&shinfo->frags[k], eat);
1415                                 eat = 0;
1416                         }
1417                         k++;
1418                 }
1419         }
1420         shinfo->nr_frags = k;
1421 
1422         skb->data_len -= len;
1423         skb->len = skb->data_len;
1424         return len;
1425 }
1426 
1427 /* Remove acked data from a packet in the transmit queue. */
1428 int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)
1429 {
1430         u32 delta_truesize;
1431 
1432         if (skb_unclone(skb, GFP_ATOMIC))
1433                 return -ENOMEM;
1434 
1435         delta_truesize = __pskb_trim_head(skb, len);
1436 
1437         TCP_SKB_CB(skb)->seq += len;
1438         skb->ip_summed = CHECKSUM_PARTIAL;
1439 
1440         if (delta_truesize) {
1441                 skb->truesize      -= delta_truesize;
1442                 sk->sk_wmem_queued -= delta_truesize;
1443                 sk_mem_uncharge(sk, delta_truesize);
1444                 sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
1445         }
1446 
1447         /* Any change of skb->len requires recalculation of tso factor. */
1448         if (tcp_skb_pcount(skb) > 1)
1449                 tcp_set_skb_tso_segs(skb, tcp_skb_mss(skb));
1450 
1451         return 0;
1452 }
1453 
1454 /* Calculate MSS not accounting any TCP options.  */
1455 static inline int __tcp_mtu_to_mss(struct sock *sk, int pmtu)
1456 {
1457         const struct tcp_sock *tp = tcp_sk(sk);
1458         const struct inet_connection_sock *icsk = inet_csk(sk);
1459         int mss_now;
1460 
1461         /* Calculate base mss without TCP options:
1462            It is MMS_S - sizeof(tcphdr) of rfc1122
1463          */
1464         mss_now = pmtu - icsk->icsk_af_ops->net_header_len - sizeof(struct tcphdr);
1465 
1466         /* IPv6 adds a frag_hdr in case RTAX_FEATURE_ALLFRAG is set */
1467         if (icsk->icsk_af_ops->net_frag_header_len) {
1468                 const struct dst_entry *dst = __sk_dst_get(sk);
1469 
1470                 if (dst && dst_allfrag(dst))
1471                         mss_now -= icsk->icsk_af_ops->net_frag_header_len;
1472         }
1473 
1474         /* Clamp it (mss_clamp does not include tcp options) */
1475         if (mss_now > tp->rx_opt.mss_clamp)
1476                 mss_now = tp->rx_opt.mss_clamp;
1477 
1478         /* Now subtract optional transport overhead */
1479         mss_now -= icsk->icsk_ext_hdr_len;
1480 
1481         /* Then reserve room for full set of TCP options and 8 bytes of data */
1482         if (mss_now < 48)
1483                 mss_now = 48;
1484         return mss_now;
1485 }
1486 
1487 /* Calculate MSS. Not accounting for SACKs here.  */
1488 int tcp_mtu_to_mss(struct sock *sk, int pmtu)
1489 {
1490         /* Subtract TCP options size, not including SACKs */
1491         return __tcp_mtu_to_mss(sk, pmtu) -
1492                (tcp_sk(sk)->tcp_header_len - sizeof(struct tcphdr));
1493 }
1494 
1495 /* Inverse of above */
1496 int tcp_mss_to_mtu(struct sock *sk, int mss)
1497 {
1498         const struct tcp_sock *tp = tcp_sk(sk);
1499         const struct inet_connection_sock *icsk = inet_csk(sk);
1500         int mtu;
1501 
1502         mtu = mss +
1503               tp->tcp_header_len +
1504               icsk->icsk_ext_hdr_len +
1505               icsk->icsk_af_ops->net_header_len;
1506 
1507         /* IPv6 adds a frag_hdr in case RTAX_FEATURE_ALLFRAG is set */
1508         if (icsk->icsk_af_ops->net_frag_header_len) {
1509                 const struct dst_entry *dst = __sk_dst_get(sk);
1510 
1511                 if (dst && dst_allfrag(dst))
1512                         mtu += icsk->icsk_af_ops->net_frag_header_len;
1513         }
1514         return mtu;
1515 }
1516 EXPORT_SYMBOL(tcp_mss_to_mtu);
1517 
1518 /* MTU probing init per socket */
1519 void tcp_mtup_init(struct sock *sk)
1520 {
1521         struct tcp_sock *tp = tcp_sk(sk);
1522         struct inet_connection_sock *icsk = inet_csk(sk);
1523         struct net *net = sock_net(sk);
1524 
1525         icsk->icsk_mtup.enabled = net->ipv4.sysctl_tcp_mtu_probing > 1;
1526         icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + sizeof(struct tcphdr) +
1527                                icsk->icsk_af_ops->net_header_len;
1528         icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, net->ipv4.sysctl_tcp_base_mss);
1529         icsk->icsk_mtup.probe_size = 0;
1530         if (icsk->icsk_mtup.enabled)
1531                 icsk->icsk_mtup.probe_timestamp = tcp_jiffies32;
1532 }
1533 EXPORT_SYMBOL(tcp_mtup_init);
1534 
1535 /* This function synchronize snd mss to current pmtu/exthdr set.
1536 
1537    tp->rx_opt.user_mss is mss set by user by TCP_MAXSEG. It does NOT counts
1538    for TCP options, but includes only bare TCP header.
1539 
1540    tp->rx_opt.mss_clamp is mss negotiated at connection setup.
1541    It is minimum of user_mss and mss received with SYN.
1542    It also does not include TCP options.
1543 
1544    inet_csk(sk)->icsk_pmtu_cookie is last pmtu, seen by this function.
1545 
1546    tp->mss_cache is current effective sending mss, including
1547    all tcp options except for SACKs. It is evaluated,
1548    taking into account current pmtu, but never exceeds
1549    tp->rx_opt.mss_clamp.
1550 
1551    NOTE1. rfc1122 clearly states that advertised MSS
1552    DOES NOT include either tcp or ip options.
1553 
1554    NOTE2. inet_csk(sk)->icsk_pmtu_cookie and tp->mss_cache
1555    are READ ONLY outside this function.         --ANK (980731)
1556  */
1557 unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu)
1558 {
1559         struct tcp_sock *tp = tcp_sk(sk);
1560         struct inet_connection_sock *icsk = inet_csk(sk);
1561         int mss_now;
1562 
1563         if (icsk->icsk_mtup.search_high > pmtu)
1564                 icsk->icsk_mtup.search_high = pmtu;
1565 
1566         mss_now = tcp_mtu_to_mss(sk, pmtu);
1567         mss_now = tcp_bound_to_half_wnd(tp, mss_now);
1568 
1569         /* And store cached results */
1570         icsk->icsk_pmtu_cookie = pmtu;
1571         if (icsk->icsk_mtup.enabled)
1572                 mss_now = min(mss_now, tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_low));
1573         tp->mss_cache = mss_now;
1574 
1575         return mss_now;
1576 }
1577 EXPORT_SYMBOL(tcp_sync_mss);
1578 
1579 /* Compute the current effective MSS, taking SACKs and IP options,
1580  * and even PMTU discovery events into account.
1581  */
1582 unsigned int tcp_current_mss(struct sock *sk)
1583 {
1584         const struct tcp_sock *tp = tcp_sk(sk);
1585         const struct dst_entry *dst = __sk_dst_get(sk);
1586         u32 mss_now;
1587         unsigned int header_len;
1588         struct tcp_out_options opts;
1589         struct tcp_md5sig_key *md5;
1590 
1591         mss_now = tp->mss_cache;
1592 
1593         if (dst) {
1594                 u32 mtu = dst_mtu(dst);
1595                 if (mtu != inet_csk(sk)->icsk_pmtu_cookie)
1596                         mss_now = tcp_sync_mss(sk, mtu);
1597         }
1598 
1599         header_len = tcp_established_options(sk, NULL, &opts, &md5) +
1600                      sizeof(struct tcphdr);
1601         /* The mss_cache is sized based on tp->tcp_header_len, which assumes
1602          * some common options. If this is an odd packet (because we have SACK
1603          * blocks etc) then our calculated header_len will be different, and
1604          * we have to adjust mss_now correspondingly */
1605         if (header_len != tp->tcp_header_len) {
1606                 int delta = (int) header_len - tp->tcp_header_len;
1607                 mss_now -= delta;
1608         }
1609 
1610         return mss_now;
1611 }
1612 
1613 /* RFC2861, slow part. Adjust cwnd, after it was not full during one rto.
1614  * As additional protections, we do not touch cwnd in retransmission phases,
1615  * and if application hit its sndbuf limit recently.
1616  */
1617 static void tcp_cwnd_application_limited(struct sock *sk)
1618 {
1619         struct tcp_sock *tp = tcp_sk(sk);
1620 
1621         if (inet_csk(sk)->icsk_ca_state == TCP_CA_Open &&
1622             sk->sk_socket && !test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
1623                 /* Limited by application or receiver window. */
1624                 u32 init_win = tcp_init_cwnd(tp, __sk_dst_get(sk));
1625                 u32 win_used = max(tp->snd_cwnd_used, init_win);
1626                 if (win_used < tp->snd_cwnd) {
1627                         tp->snd_ssthresh = tcp_current_ssthresh(sk);
1628                         tp->snd_cwnd = (tp->snd_cwnd + win_used) >> 1;
1629                 }
1630                 tp->snd_cwnd_used = 0;
1631         }
1632         tp->snd_cwnd_stamp = tcp_jiffies32;
1633 }
1634 
1635 static void tcp_cwnd_validate(struct sock *sk, bool is_cwnd_limited)
1636 {
1637         const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
1638         struct tcp_sock *tp = tcp_sk(sk);
1639 
1640         /* Track the maximum number of outstanding packets in each
1641          * window, and remember whether we were cwnd-limited then.
1642          */
1643         if (!before(tp->snd_una, tp->max_packets_seq) ||
1644             tp->packets_out > tp->max_packets_out) {
1645                 tp->max_packets_out = tp->packets_out;
1646                 tp->max_packets_seq = tp->snd_nxt;
1647                 tp->is_cwnd_limited = is_cwnd_limited;
1648         }
1649 
1650         if (tcp_is_cwnd_limited(sk)) {
1651                 /* Network is feed fully. */
1652                 tp->snd_cwnd_used = 0;
1653                 tp->snd_cwnd_stamp = tcp_jiffies32;
1654         } else {
1655                 /* Network starves. */
1656                 if (tp->packets_out > tp->snd_cwnd_used)
1657                         tp->snd_cwnd_used = tp->packets_out;
1658 
1659                 if (sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle &&
1660                     (s32)(tcp_jiffies32 - tp->snd_cwnd_stamp) >= inet_csk(sk)->icsk_rto &&
1661                     !ca_ops->cong_control)
1662                         tcp_cwnd_application_limited(sk);
1663 
1664                 /* The following conditions together indicate the starvation
1665                  * is caused by insufficient sender buffer:
1666                  * 1) just sent some data (see tcp_write_xmit)
1667                  * 2) not cwnd limited (this else condition)
1668                  * 3) no more data to send (tcp_write_queue_empty())
1669                  * 4) application is hitting buffer limit (SOCK_NOSPACE)
1670                  */
1671                 if (tcp_write_queue_empty(sk) && sk->sk_socket &&
1672                     test_bit(SOCK_NOSPACE, &sk->sk_socket->flags) &&
1673                     (1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
1674                         tcp_chrono_start(sk, TCP_CHRONO_SNDBUF_LIMITED);
1675         }
1676 }
1677 
1678 /* Minshall's variant of the Nagle send check. */
1679 static bool tcp_minshall_check(const struct tcp_sock *tp)
1680 {
1681         return after(tp->snd_sml, tp->snd_una) &&
1682                 !after(tp->snd_sml, tp->snd_nxt);
1683 }
1684 
1685 /* Update snd_sml if this skb is under mss
1686  * Note that a TSO packet might end with a sub-mss segment
1687  * The test is really :
1688  * if ((skb->len % mss) != 0)
1689  *        tp->snd_sml = TCP_SKB_CB(skb)->end_seq;
1690  * But we can avoid doing the divide again given we already have
1691  *  skb_pcount = skb->len / mss_now
1692  */
1693 static void tcp_minshall_update(struct tcp_sock *tp, unsigned int mss_now,
1694                                 const struct sk_buff *skb)
1695 {
1696         if (skb->len < tcp_skb_pcount(skb) * mss_now)
1697                 tp->snd_sml = TCP_SKB_CB(skb)->end_seq;
1698 }
1699 
1700 /* Return false, if packet can be sent now without violation Nagle's rules:
1701  * 1. It is full sized. (provided by caller in %partial bool)
1702  * 2. Or it contains FIN. (already checked by caller)
1703  * 3. Or TCP_CORK is not set, and TCP_NODELAY is set.
1704  * 4. Or TCP_CORK is not set, and all sent packets are ACKed.
1705  *    With Minshall's modification: all sent small packets are ACKed.
1706  */
1707 static bool tcp_nagle_check(bool partial, const struct tcp_sock *tp,
1708                             int nonagle)
1709 {
1710         return partial &&
1711                 ((nonagle & TCP_NAGLE_CORK) ||
1712                  (!nonagle && tp->packets_out && tcp_minshall_check(tp)));
1713 }
1714 
1715 /* Return how many segs we'd like on a TSO packet,
1716  * to send one TSO packet per ms
1717  */
1718 u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now,
1719                      int min_tso_segs)
1720 {
1721         u32 bytes, segs;
1722 
1723         bytes = min(sk->sk_pacing_rate >> sk->sk_pacing_shift,
1724                     sk->sk_gso_max_size - 1 - MAX_TCP_HEADER);
1725 
1726         /* Goal is to send at least one packet per ms,
1727          * not one big TSO packet every 100 ms.
1728          * This preserves ACK clocking and is consistent
1729          * with tcp_tso_should_defer() heuristic.
1730          */
1731         segs = max_t(u32, bytes / mss_now, min_tso_segs);
1732 
1733         return segs;
1734 }
1735 EXPORT_SYMBOL(tcp_tso_autosize);
1736 
1737 /* Return the number of segments we want in the skb we are transmitting.
1738  * See if congestion control module wants to decide; otherwise, autosize.
1739  */
1740 static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now)
1741 {
1742         const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
1743         u32 tso_segs = ca_ops->tso_segs_goal ? ca_ops->tso_segs_goal(sk) : 0;
1744 
1745         if (!tso_segs)
1746                 tso_segs = tcp_tso_autosize(sk, mss_now,
1747                                 sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs);
1748         return min_t(u32, tso_segs, sk->sk_gso_max_segs);
1749 }
1750 
1751 /* Returns the portion of skb which can be sent right away */
1752 static unsigned int tcp_mss_split_point(const struct sock *sk,
1753                                         const struct sk_buff *skb,
1754                                         unsigned int mss_now,
1755                                         unsigned int max_segs,
1756                                         int nonagle)
1757 {
1758         const struct tcp_sock *tp = tcp_sk(sk);
1759         u32 partial, needed, window, max_len;
1760 
1761         window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
1762         max_len = mss_now * max_segs;
1763 
1764         if (likely(max_len <= window && skb != tcp_write_queue_tail(sk)))
1765                 return max_len;
1766 
1767         needed = min(skb->len, window);
1768 
1769         if (max_len <= needed)
1770                 return max_len;
1771 
1772         partial = needed % mss_now;
1773         /* If last segment is not a full MSS, check if Nagle rules allow us
1774          * to include this last segment in this skb.
1775          * Otherwise, we'll split the skb at last MSS boundary
1776          */
1777         if (tcp_nagle_check(partial != 0, tp, nonagle))
1778                 return needed - partial;
1779 
1780         return needed;
1781 }
1782 
1783 /* Can at least one segment of SKB be sent right now, according to the
1784  * congestion window rules?  If so, return how many segments are allowed.
1785  */
1786 static inline unsigned int tcp_cwnd_test(const struct tcp_sock *tp,
1787                                          const struct sk_buff *skb)
1788 {
1789         u32 in_flight, cwnd, halfcwnd;
1790 
1791         /* Don't be strict about the congestion window for the final FIN.  */
1792         if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) &&
1793             tcp_skb_pcount(skb) == 1)
1794                 return 1;
1795 
1796         in_flight = tcp_packets_in_flight(tp);
1797         cwnd = tp->snd_cwnd;
1798         if (in_flight >= cwnd)
1799                 return 0;
1800 
1801         /* For better scheduling, ensure we have at least
1802          * 2 GSO packets in flight.
1803          */
1804         halfcwnd = max(cwnd >> 1, 1U);
1805         return min(halfcwnd, cwnd - in_flight);
1806 }
1807 
1808 /* Initialize TSO state of a skb.
1809  * This must be invoked the first time we consider transmitting
1810  * SKB onto the wire.
1811  */
1812 static int tcp_init_tso_segs(struct sk_buff *skb, unsigned int mss_now)
1813 {
1814         int tso_segs = tcp_skb_pcount(skb);
1815 
1816         if (!tso_segs || (tso_segs > 1 && tcp_skb_mss(skb) != mss_now)) {
1817                 tcp_set_skb_tso_segs(skb, mss_now);
1818                 tso_segs = tcp_skb_pcount(skb);
1819         }
1820         return tso_segs;
1821 }
1822 
1823 
1824 /* Return true if the Nagle test allows this packet to be
1825  * sent now.
1826  */
1827 static inline bool tcp_nagle_test(const struct tcp_sock *tp, const struct sk_buff *skb,
1828                                   unsigned int cur_mss, int nonagle)
1829 {
1830         /* Nagle rule does not apply to frames, which sit in the middle of the
1831          * write_queue (they have no chances to get new data).
1832          *
1833          * This is implemented in the callers, where they modify the 'nonagle'
1834          * argument based upon the location of SKB in the send queue.
1835          */
1836         if (nonagle & TCP_NAGLE_PUSH)
1837                 return true;
1838 
1839         /* Don't use the nagle rule for urgent data (or for the final FIN). */
1840         if (tcp_urg_mode(tp) || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
1841                 return true;
1842 
1843         if (!tcp_nagle_check(skb->len < cur_mss, tp, nonagle))
1844                 return true;
1845 
1846         return false;
1847 }
1848 
1849 /* Does at least the first segment of SKB fit into the send window? */
1850 static bool tcp_snd_wnd_test(const struct tcp_sock *tp,
1851                              const struct sk_buff *skb,
1852                              unsigned int cur_mss)
1853 {
1854         u32 end_seq = TCP_SKB_CB(skb)->end_seq;
1855 
1856         if (skb->len > cur_mss)
1857                 end_seq = TCP_SKB_CB(skb)->seq + cur_mss;
1858 
1859         return !after(end_seq, tcp_wnd_end(tp));
1860 }
1861 
1862 /* Trim TSO SKB to LEN bytes, put the remaining data into a new packet
1863  * which is put after SKB on the list.  It is very much like
1864  * tcp_fragment() except that it may make several kinds of assumptions
1865  * in order to speed up the splitting operation.  In particular, we
1866  * know that all the data is in scatter-gather pages, and that the
1867  * packet has never been sent out before (and thus is not cloned).
1868  */
1869 static int tso_fragment(struct sock *sk, enum tcp_queue tcp_queue,
1870                         struct sk_buff *skb, unsigned int len,
1871                         unsigned int mss_now, gfp_t gfp)
1872 {
1873         struct sk_buff *buff;
1874         int nlen = skb->len - len;
1875         u8 flags;
1876 
1877         /* All of a TSO frame must be composed of paged data.  */
1878         if (skb->len != skb->data_len)
1879                 return tcp_fragment(sk, tcp_queue, skb, len, mss_now, gfp);
1880 
1881         buff = sk_stream_alloc_skb(sk, 0, gfp, true);
1882         if (unlikely(!buff))
1883                 return -ENOMEM;
1884 
1885         sk->sk_wmem_queued += buff->truesize;
1886         sk_mem_charge(sk, buff->truesize);
1887         buff->truesize += nlen;
1888         skb->truesize -= nlen;
1889 
1890         /* Correct the sequence numbers. */
1891         TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
1892         TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
1893         TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
1894 
1895         /* PSH and FIN should only be set in the second packet. */
1896         flags = TCP_SKB_CB(skb)->tcp_flags;
1897         TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
1898         TCP_SKB_CB(buff)->tcp_flags = flags;
1899 
1900         /* This packet was never sent out yet, so no SACK bits. */
1901         TCP_SKB_CB(buff)->sacked = 0;
1902 
1903         tcp_skb_fragment_eor(skb, buff);
1904 
1905         buff->ip_summed = skb->ip_summed = CHECKSUM_PARTIAL;
1906         skb_split(skb, buff, len);
1907         tcp_fragment_tstamp(skb, buff);
1908 
1909         /* Fix up tso_factor for both original and new SKB.  */
1910         tcp_set_skb_tso_segs(skb, mss_now);
1911         tcp_set_skb_tso_segs(buff, mss_now);
1912 
1913         /* Link BUFF into the send queue. */
1914         __skb_header_release(buff);
1915         tcp_insert_write_queue_after(skb, buff, sk, tcp_queue);
1916 
1917         return 0;
1918 }
1919 
1920 /* Try to defer sending, if possible, in order to minimize the amount
1921  * of TSO splitting we do.  View it as a kind of TSO Nagle test.
1922  *
1923  * This algorithm is from John Heffner.
1924  */
1925 static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb,
1926                                  bool *is_cwnd_limited, u32 max_segs)
1927 {
1928         const struct inet_connection_sock *icsk = inet_csk(sk);
1929         u32 age, send_win, cong_win, limit, in_flight;
1930         struct tcp_sock *tp = tcp_sk(sk);
1931         struct sk_buff *head;
1932         int win_divisor;
1933 
1934         if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
1935                 goto send_now;
1936 
1937         if (icsk->icsk_ca_state >= TCP_CA_Recovery)
1938                 goto send_now;
1939 
1940         /* Avoid bursty behavior by allowing defer
1941          * only if the last write was recent.
1942          */
1943         if ((s32)(tcp_jiffies32 - tp->lsndtime) > 0)
1944                 goto send_now;
1945 
1946         in_flight = tcp_packets_in_flight(tp);
1947 
1948         BUG_ON(tcp_skb_pcount(skb) <= 1 || (tp->snd_cwnd <= in_flight));
1949 
1950         send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
1951 
1952         /* From in_flight test above, we know that cwnd > in_flight.  */
1953         cong_win = (tp->snd_cwnd - in_flight) * tp->mss_cache;
1954 
1955         limit = min(send_win, cong_win);
1956 
1957         /* If a full-sized TSO skb can be sent, do it. */
1958         if (limit >= max_segs * tp->mss_cache)
1959                 goto send_now;
1960 
1961         /* Middle in queue won't get any more data, full sendable already? */
1962         if ((skb != tcp_write_queue_tail(sk)) && (limit >= skb->len))
1963                 goto send_now;
1964 
1965         win_divisor = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tso_win_divisor);
1966         if (win_divisor) {
1967                 u32 chunk = min(tp->snd_wnd, tp->snd_cwnd * tp->mss_cache);
1968 
1969                 /* If at least some fraction of a window is available,
1970                  * just use it.
1971                  */
1972                 chunk /= win_divisor;
1973                 if (limit >= chunk)
1974                         goto send_now;
1975         } else {
1976                 /* Different approach, try not to defer past a single
1977                  * ACK.  Receiver should ACK every other full sized
1978                  * frame, so if we have space for more than 3 frames
1979                  * then send now.
1980                  */
1981                 if (limit > tcp_max_tso_deferred_mss(tp) * tp->mss_cache)
1982                         goto send_now;
1983         }
1984 
1985         /* TODO : use tsorted_sent_queue ? */
1986         head = tcp_rtx_queue_head(sk);
1987         if (!head)
1988                 goto send_now;
1989         age = tcp_stamp_us_delta(tp->tcp_mstamp, head->skb_mstamp);
1990         /* If next ACK is likely to come too late (half srtt), do not defer */
1991         if (age < (tp->srtt_us >> 4))
1992                 goto send_now;
1993 
1994         /* Ok, it looks like it is advisable to defer. */
1995 
1996         if (cong_win < send_win && cong_win <= skb->len)
1997                 *is_cwnd_limited = true;
1998 
1999         return true;
2000 
2001 send_now:
2002         return false;
2003 }
2004 
2005 static inline void tcp_mtu_check_reprobe(struct sock *sk)
2006 {
2007         struct inet_connection_sock *icsk = inet_csk(sk);
2008         struct tcp_sock *tp = tcp_sk(sk);
2009         struct net *net = sock_net(sk);
2010         u32 interval;
2011         s32 delta;
2012 
2013         interval = net->ipv4.sysctl_tcp_probe_interval;
2014         delta = tcp_jiffies32 - icsk->icsk_mtup.probe_timestamp;
2015         if (unlikely(delta >= interval * HZ)) {
2016                 int mss = tcp_current_mss(sk);
2017 
2018                 /* Update current search range */
2019                 icsk->icsk_mtup.probe_size = 0;
2020                 icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp +
2021                         sizeof(struct tcphdr) +
2022                         icsk->icsk_af_ops->net_header_len;
2023                 icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, mss);
2024 
2025                 /* Update probe time stamp */
2026                 icsk->icsk_mtup.probe_timestamp = tcp_jiffies32;
2027         }
2028 }
2029 
2030 static bool tcp_can_coalesce_send_queue_head(struct sock *sk, int len)
2031 {
2032         struct sk_buff *skb, *next;
2033 
2034         skb = tcp_send_head(sk);
2035         tcp_for_write_queue_from_safe(skb, next, sk) {
2036                 if (len <= skb->len)
2037                         break;
2038 
2039                 if (unlikely(TCP_SKB_CB(skb)->eor))
2040                         return false;
2041 
2042                 len -= skb->len;
2043         }
2044 
2045         return true;
2046 }
2047 
2048 /* Create a new MTU probe if we are ready.
2049  * MTU probe is regularly attempting to increase the path MTU by
2050  * deliberately sending larger packets.  This discovers routing
2051  * changes resulting in larger path MTUs.
2052  *
2053  * Returns 0 if we should wait to probe (no cwnd available),
2054  *         1 if a probe was sent,
2055  *         -1 otherwise
2056  */
2057 static int tcp_mtu_probe(struct sock *sk)
2058 {
2059         struct inet_connection_sock *icsk = inet_csk(sk);
2060         struct tcp_sock *tp = tcp_sk(sk);
2061         struct sk_buff *skb, *nskb, *next;
2062         struct net *net = sock_net(sk);
2063         int probe_size;
2064         int size_needed;
2065         int copy, len;
2066         int mss_now;
2067         int interval;
2068 
2069         /* Not currently probing/verifying,
2070          * not in recovery,
2071          * have enough cwnd, and
2072          * not SACKing (the variable headers throw things off)
2073          */
2074         if (likely(!icsk->icsk_mtup.enabled ||
2075                    icsk->icsk_mtup.probe_size ||
2076                    inet_csk(sk)->icsk_ca_state != TCP_CA_Open ||
2077                    tp->snd_cwnd < 11 ||
2078                    tp->rx_opt.num_sacks || tp->rx_opt.dsack))
2079                 return -1;
2080 
2081         /* Use binary search for probe_size between tcp_mss_base,
2082          * and current mss_clamp. if (search_high - search_low)
2083          * smaller than a threshold, backoff from probing.
2084          */
2085         mss_now = tcp_current_mss(sk);
2086         probe_size = tcp_mtu_to_mss(sk, (icsk->icsk_mtup.search_high +
2087                                     icsk->icsk_mtup.search_low) >> 1);
2088         size_needed = probe_size + (tp->reordering + 1) * tp->mss_cache;
2089         interval = icsk->icsk_mtup.search_high - icsk->icsk_mtup.search_low;
2090         /* When misfortune happens, we are reprobing actively,
2091          * and then reprobe timer has expired. We stick with current
2092          * probing process by not resetting search range to its orignal.
2093          */
2094         if (probe_size > tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_high) ||
2095                 interval < net->ipv4.sysctl_tcp_probe_threshold) {
2096                 /* Check whether enough time has elaplased for
2097                  * another round of probing.
2098                  */
2099                 tcp_mtu_check_reprobe(sk);
2100                 return -1;
2101         }
2102 
2103         /* Have enough data in the send queue to probe? */
2104         if (tp->write_seq - tp->snd_nxt < size_needed)
2105                 return -1;
2106 
2107         if (tp->snd_wnd < size_needed)
2108                 return -1;
2109         if (after(tp->snd_nxt + size_needed, tcp_wnd_end(tp)))
2110                 return 0;
2111 
2112         /* Do we need to wait to drain cwnd? With none in flight, don't stall */
2113         if (tcp_packets_in_flight(tp) + 2 > tp->snd_cwnd) {
2114                 if (!tcp_packets_in_flight(tp))
2115                         return -1;
2116                 else
2117                         return 0;
2118         }
2119 
2120         if (!tcp_can_coalesce_send_queue_head(sk, probe_size))
2121                 return -1;
2122 
2123         /* We're allowed to probe.  Build it now. */
2124         nskb = sk_stream_alloc_skb(sk, probe_size, GFP_ATOMIC, false);
2125         if (!nskb)
2126                 return -1;
2127         sk->sk_wmem_queued += nskb->truesize;
2128         sk_mem_charge(sk, nskb->truesize);
2129 
2130         skb = tcp_send_head(sk);
2131 
2132         TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq;
2133         TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size;
2134         TCP_SKB_CB(nskb)->tcp_flags = TCPHDR_ACK;
2135         TCP_SKB_CB(nskb)->sacked = 0;
2136         nskb->csum = 0;
2137         nskb->ip_summed = skb->ip_summed;
2138 
2139         tcp_insert_write_queue_before(nskb, skb, sk);
2140         tcp_highest_sack_replace(sk, skb, nskb);
2141 
2142         len = 0;
2143         tcp_for_write_queue_from_safe(skb, next, sk) {
2144                 copy = min_t(int, skb->len, probe_size - len);
2145                 if (nskb->ip_summed) {
2146                         skb_copy_bits(skb, 0, skb_put(nskb, copy), copy);
2147                 } else {
2148                         __wsum csum = skb_copy_and_csum_bits(skb, 0,
2149                                                              skb_put(nskb, copy),
2150                                                              copy, 0);
2151                         nskb->csum = csum_block_add(nskb->csum, csum, len);
2152                 }
2153 
2154                 if (skb->len <= copy) {
2155                         /* We've eaten all the data from this skb.
2156                          * Throw it away. */
2157                         TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
2158                         /* If this is the last SKB we copy and eor is set
2159                          * we need to propagate it to the new skb.
2160                          */
2161                         TCP_SKB_CB(nskb)->eor = TCP_SKB_CB(skb)->eor;
2162                         tcp_unlink_write_queue(skb, sk);
2163                         sk_wmem_free_skb(sk, skb);
2164                 } else {
2165                         TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags &
2166                                                    ~(TCPHDR_FIN|TCPHDR_PSH);
2167                         if (!skb_shinfo(skb)->nr_frags) {
2168                                 skb_pull(skb, copy);
2169                                 if (skb->ip_summed != CHECKSUM_PARTIAL)
2170                                         skb->csum = csum_partial(skb->data,
2171                                                                  skb->len, 0);
2172                         } else {
2173                                 __pskb_trim_head(skb, copy);
2174                                 tcp_set_skb_tso_segs(skb, mss_now);
2175                         }
2176                         TCP_SKB_CB(skb)->seq += copy;
2177                 }
2178 
2179                 len += copy;
2180 
2181                 if (len >= probe_size)
2182                         break;
2183         }
2184         tcp_init_tso_segs(nskb, nskb->len);
2185 
2186         /* We're ready to send.  If this fails, the probe will
2187          * be resegmented into mss-sized pieces by tcp_write_xmit().
2188          */
2189         if (!tcp_transmit_skb(sk, nskb, 1, GFP_ATOMIC)) {
2190                 /* Decrement cwnd here because we are sending
2191                  * effectively two packets. */
2192                 tp->snd_cwnd--;
2193                 tcp_event_new_data_sent(sk, nskb);
2194 
2195                 icsk->icsk_mtup.probe_size = tcp_mss_to_mtu(sk, nskb->len);
2196                 tp->mtu_probe.probe_seq_start = TCP_SKB_CB(nskb)->seq;
2197                 tp->mtu_probe.probe_seq_end = TCP_SKB_CB(nskb)->end_seq;
2198 
2199                 return 1;
2200         }
2201 
2202         return -1;
2203 }
2204 
2205 static bool tcp_pacing_check(const struct sock *sk)
2206 {
2207         return tcp_needs_internal_pacing(sk) &&
2208                hrtimer_active(&tcp_sk(sk)->pacing_timer);
2209 }
2210 
2211 /* TCP Small Queues :
2212  * Control number of packets in qdisc/devices to two packets / or ~1 ms.
2213  * (These limits are doubled for retransmits)
2214  * This allows for :
2215  *  - better RTT estimation and ACK scheduling
2216  *  - faster recovery
2217  *  - high rates
2218  * Alas, some drivers / subsystems require a fair amount
2219  * of queued bytes to ensure line rate.
2220  * One example is wifi aggregation (802.11 AMPDU)
2221  */
2222 static bool tcp_small_queue_check(struct sock *sk, const struct sk_buff *skb,
2223                                   unsigned int factor)
2224 {
2225         unsigned int limit;
2226 
2227         limit = max(2 * skb->truesize, sk->sk_pacing_rate >> sk->sk_pacing_shift);
2228         limit = min_t(u32, limit,
2229                       sock_net(sk)->ipv4.sysctl_tcp_limit_output_bytes);
2230         limit <<= factor;
2231 
2232         if (refcount_read(&sk->sk_wmem_alloc) > limit) {
2233                 /* Always send skb if rtx queue is empty.
2234                  * No need to wait for TX completion to call us back,
2235                  * after softirq/tasklet schedule.
2236                  * This helps when TX completions are delayed too much.
2237                  */
2238                 if (tcp_rtx_queue_empty(sk))
2239                         return false;
2240 
2241                 set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
2242                 /* It is possible TX completion already happened
2243                  * before we set TSQ_THROTTLED, so we must
2244                  * test again the condition.
2245                  */
2246                 smp_mb__after_atomic();
2247                 if (refcount_read(&sk->sk_wmem_alloc) > limit)
2248                         return true;
2249         }
2250         return false;
2251 }
2252 
2253 static void tcp_chrono_set(struct tcp_sock *tp, const enum tcp_chrono new)
2254 {
2255         const u32 now = tcp_jiffies32;
2256         enum tcp_chrono old = tp->chrono_type;
2257 
2258         if (old > TCP_CHRONO_UNSPEC)
2259                 tp->chrono_stat[old - 1] += now - tp->chrono_start;
2260         tp->chrono_start = now;
2261         tp->chrono_type = new;
2262 }
2263 
2264 void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type)
2265 {
2266         struct tcp_sock *tp = tcp_sk(sk);
2267 
2268         /* If there are multiple conditions worthy of tracking in a
2269          * chronograph then the highest priority enum takes precedence
2270          * over the other conditions. So that if something "more interesting"
2271          * starts happening, stop the previous chrono and start a new one.
2272          */
2273         if (type > tp->chrono_type)
2274                 tcp_chrono_set(tp, type);
2275 }
2276 
2277 void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type)
2278 {
2279         struct tcp_sock *tp = tcp_sk(sk);
2280 
2281 
2282         /* There are multiple conditions worthy of tracking in a
2283          * chronograph, so that the highest priority enum takes
2284          * precedence over the other conditions (see tcp_chrono_start).
2285          * If a condition stops, we only stop chrono tracking if
2286          * it's the "most interesting" or current chrono we are
2287          * tracking and starts busy chrono if we have pending data.
2288          */
2289         if (tcp_rtx_and_write_queues_empty(sk))
2290                 tcp_chrono_set(tp, TCP_CHRONO_UNSPEC);
2291         else if (type == tp->chrono_type)
2292                 tcp_chrono_set(tp, TCP_CHRONO_BUSY);
2293 }
2294 
2295 /* This routine writes packets to the network.  It advances the
2296  * send_head.  This happens as incoming acks open up the remote
2297  * window for us.
2298  *
2299  * LARGESEND note: !tcp_urg_mode is overkill, only frames between
2300  * snd_up-64k-mss .. snd_up cannot be large. However, taking into
2301  * account rare use of URG, this is not a big flaw.
2302  *
2303  * Send at most one packet when push_one > 0. Temporarily ignore
2304  * cwnd limit to force at most one packet out when push_one == 2.
2305 
2306  * Returns true, if no segments are in flight and we have queued segments,
2307  * but cannot send anything now because of SWS or another problem.
2308  */
2309 static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
2310                            int push_one, gfp_t gfp)
2311 {
2312         struct tcp_sock *tp = tcp_sk(sk);
2313         struct sk_buff *skb;
2314         unsigned int tso_segs, sent_pkts;
2315         int cwnd_quota;
2316         int result;
2317         bool is_cwnd_limited = false, is_rwnd_limited = false;
2318         u32 max_segs;
2319 
2320         sent_pkts = 0;
2321 
2322         tcp_mstamp_refresh(tp);
2323         if (!push_one) {
2324                 /* Do MTU probing. */
2325                 result = tcp_mtu_probe(sk);
2326                 if (!result) {
2327                         return false;
2328                 } else if (result > 0) {
2329                         sent_pkts = 1;
2330                 }
2331         }
2332 
2333         max_segs = tcp_tso_segs(sk, mss_now);
2334         while ((skb = tcp_send_head(sk))) {
2335                 unsigned int limit;
2336 
2337                 if (tcp_pacing_check(sk))
2338                         break;
2339 
2340                 tso_segs = tcp_init_tso_segs(skb, mss_now);
2341                 BUG_ON(!tso_segs);
2342 
2343                 if (unlikely(tp->repair) && tp->repair_queue == TCP_SEND_QUEUE) {
2344                         /* "skb_mstamp" is used as a start point for the retransmit timer */
2345                         tcp_update_skb_after_send(tp, skb);
2346                         goto repair; /* Skip network transmission */
2347                 }
2348 
2349                 cwnd_quota = tcp_cwnd_test(tp, skb);
2350                 if (!cwnd_quota) {
2351                         if (push_one == 2)
2352                                 /* Force out a loss probe pkt. */
2353                                 cwnd_quota = 1;
2354                         else
2355                                 break;
2356                 }
2357 
2358                 if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now))) {
2359                         is_rwnd_limited = true;
2360                         break;
2361                 }
2362 
2363                 if (tso_segs == 1) {
2364                         if (unlikely(!tcp_nagle_test(tp, skb, mss_now,
2365                                                      (tcp_skb_is_last(sk, skb) ?
2366                                                       nonagle : TCP_NAGLE_PUSH))))
2367                                 break;
2368                 } else {
2369                         if (!push_one &&
2370                             tcp_tso_should_defer(sk, skb, &is_cwnd_limited,
2371                                                  max_segs))
2372                                 break;
2373                 }
2374 
2375                 limit = mss_now;
2376                 if (tso_segs > 1 && !tcp_urg_mode(tp))
2377                         limit = tcp_mss_split_point(sk, skb, mss_now,
2378                                                     min_t(unsigned int,
2379                                                           cwnd_quota,
2380                                                           max_segs),
2381                                                     nonagle);
2382 
2383                 if (skb->len > limit &&
2384                     unlikely(tso_fragment(sk, TCP_FRAG_IN_WRITE_QUEUE,
2385                                           skb, limit, mss_now, gfp)))
2386                         break;
2387 
2388                 if (test_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags))
2389                         clear_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags);
2390                 if (tcp_small_queue_check(sk, skb, 0))
2391                         break;
2392 
2393                 if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp)))
2394                         break;
2395 
2396 repair:
2397                 /* Advance the send_head.  This one is sent out.
2398                  * This call will increment packets_out.
2399                  */
2400                 tcp_event_new_data_sent(sk, skb);
2401 
2402                 tcp_minshall_update(tp, mss_now, skb);
2403                 sent_pkts += tcp_skb_pcount(skb);
2404 
2405                 if (push_one)
2406                         break;
2407         }
2408 
2409         if (is_rwnd_limited)
2410                 tcp_chrono_start(sk, TCP_CHRONO_RWND_LIMITED);
2411         else
2412                 tcp_chrono_stop(sk, TCP_CHRONO_RWND_LIMITED);
2413 
2414         if (likely(sent_pkts)) {
2415                 if (tcp_in_cwnd_reduction(sk))
2416                         tp->prr_out += sent_pkts;
2417 
2418                 /* Send one loss probe per tail loss episode. */
2419                 if (push_one != 2)
2420                         tcp_schedule_loss_probe(sk, false);
2421                 is_cwnd_limited |= (tcp_packets_in_flight(tp) >= tp->snd_cwnd);
2422                 tcp_cwnd_validate(sk, is_cwnd_limited);
2423                 return false;
2424         }
2425         return !tp->packets_out && !tcp_write_queue_empty(sk);
2426 }
2427 
2428 bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto)
2429 {
2430         struct inet_connection_sock *icsk = inet_csk(sk);
2431         struct tcp_sock *tp = tcp_sk(sk);
2432         u32 timeout, rto_delta_us;
2433         int early_retrans;
2434 
2435         /* Don't do any loss probe on a Fast Open connection before 3WHS
2436          * finishes.
2437          */
2438         if (tp->fastopen_rsk)
2439                 return false;
2440 
2441         early_retrans = sock_net(sk)->ipv4.sysctl_tcp_early_retrans;
2442         /* Schedule a loss probe in 2*RTT for SACK capable connections
2443          * not in loss recovery, that are either limited by cwnd or application.
2444          */
2445         if ((early_retrans != 3 && early_retrans != 4) ||
2446             !tp->packets_out || !tcp_is_sack(tp) ||
2447             (icsk->icsk_ca_state != TCP_CA_Open &&
2448              icsk->icsk_ca_state != TCP_CA_CWR))
2449                 return false;
2450 
2451         /* Probe timeout is 2*rtt. Add minimum RTO to account
2452          * for delayed ack when there's one outstanding packet. If no RTT
2453          * sample is available then probe after TCP_TIMEOUT_INIT.
2454          */
2455         if (tp->srtt_us) {
2456                 timeout = usecs_to_jiffies(tp->srtt_us >> 2);
2457                 if (tp->packets_out == 1)
2458                         timeout += TCP_RTO_MIN;
2459                 else
2460                         timeout += TCP_TIMEOUT_MIN;
2461         } else {
2462                 timeout = TCP_TIMEOUT_INIT;
2463         }
2464 
2465         /* If the RTO formula yields an earlier time, then use that time. */
2466         rto_delta_us = advancing_rto ?
2467                         jiffies_to_usecs(inet_csk(sk)->icsk_rto) :
2468                         tcp_rto_delta_us(sk);  /* How far in future is RTO? */
2469         if (rto_delta_us > 0)
2470                 timeout = min_t(u32, timeout, usecs_to_jiffies(rto_delta_us));
2471 
2472         inet_csk_reset_xmit_timer(sk, ICSK_TIME_LOSS_PROBE, timeout,
2473                                   TCP_RTO_MAX);
2474         return true;
2475 }
2476 
2477 /* Thanks to skb fast clones, we can detect if a prior transmit of
2478  * a packet is still in a qdisc or driver queue.
2479  * In this case, there is very little point doing a retransmit !
2480  */
2481 static bool skb_still_in_host_queue(const struct sock *sk,
2482                                     const struct sk_buff *skb)
2483 {
2484         if (unlikely(skb_fclone_busy(sk, skb))) {
2485                 NET_INC_STATS(sock_net(sk),
2486                               LINUX_MIB_TCPSPURIOUS_RTX_HOSTQUEUES);
2487                 return true;
2488         }
2489         return false;
2490 }
2491 
2492 /* When probe timeout (PTO) fires, try send a new segment if possible, else
2493  * retransmit the last segment.
2494  */
2495 void tcp_send_loss_probe(struct sock *sk)
2496 {
2497         struct tcp_sock *tp = tcp_sk(sk);
2498         struct sk_buff *skb;
2499         int pcount;
2500         int mss = tcp_current_mss(sk);
2501 
2502         skb = tcp_send_head(sk);
2503         if (skb && tcp_snd_wnd_test(tp, skb, mss)) {
2504                 pcount = tp->packets_out;
2505                 tcp_write_xmit(sk, mss, TCP_NAGLE_OFF, 2, GFP_ATOMIC);
2506                 if (tp->packets_out > pcount)
2507                         goto probe_sent;
2508                 goto rearm_timer;
2509         }
2510         skb = skb_rb_last(&sk->tcp_rtx_queue);
2511 
2512         /* At most one outstanding TLP retransmission. */
2513         if (tp->tlp_high_seq)
2514                 goto rearm_timer;
2515 
2516         /* Retransmit last segment. */
2517         if (WARN_ON(!skb))
2518                 goto rearm_timer;
2519 
2520         if (skb_still_in_host_queue(sk, skb))
2521                 goto rearm_timer;
2522 
2523         pcount = tcp_skb_pcount(skb);
2524         if (WARN_ON(!pcount))
2525                 goto rearm_timer;
2526 
2527         if ((pcount > 1) && (skb->len > (pcount - 1) * mss)) {
2528                 if (unlikely(tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb,
2529                                           (pcount - 1) * mss, mss,
2530                                           GFP_ATOMIC)))
2531                         goto rearm_timer;
2532                 skb = skb_rb_next(skb);
2533         }
2534 
2535         if (WARN_ON(!skb || !tcp_skb_pcount(skb)))
2536                 goto rearm_timer;
2537 
2538         if (__tcp_retransmit_skb(sk, skb, 1))
2539                 goto rearm_timer;
2540 
2541         /* Record snd_nxt for loss detection. */
2542         tp->tlp_high_seq = tp->snd_nxt;
2543 
2544 probe_sent:
2545         NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPLOSSPROBES);
2546         /* Reset s.t. tcp_rearm_rto will restart timer from now */
2547         inet_csk(sk)->icsk_pending = 0;
2548 rearm_timer:
2549         tcp_rearm_rto(sk);
2550 }
2551 
2552 /* Push out any pending frames which were held back due to
2553  * TCP_CORK or attempt at coalescing tiny packets.
2554  * The socket must be locked by the caller.
2555  */
2556 void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
2557                                int nonagle)
2558 {
2559         /* If we are closed, the bytes will have to remain here.
2560          * In time closedown will finish, we empty the write queue and
2561          * all will be happy.
2562          */
2563         if (unlikely(sk->sk_state == TCP_CLOSE))
2564                 return;
2565 
2566         if (tcp_write_xmit(sk, cur_mss, nonagle, 0,
2567                            sk_gfp_mask(sk, GFP_ATOMIC)))
2568                 tcp_check_probe_timer(sk);
2569 }
2570 
2571 /* Send _single_ skb sitting at the send head. This function requires
2572  * true push pending frames to setup probe timer etc.
2573  */
2574 void tcp_push_one(struct sock *sk, unsigned int mss_now)
2575 {
2576         struct sk_buff *skb = tcp_send_head(sk);
2577 
2578         BUG_ON(!skb || skb->len < mss_now);
2579 
2580         tcp_write_xmit(sk, mss_now, TCP_NAGLE_PUSH, 1, sk->sk_allocation);
2581 }
2582 
2583 /* This function returns the amount that we can raise the
2584  * usable window based on the following constraints
2585  *
2586  * 1. The window can never be shrunk once it is offered (RFC 793)
2587  * 2. We limit memory per socket
2588  *
2589  * RFC 1122:
2590  * "the suggested [SWS] avoidance algorithm for the receiver is to keep
2591  *  RECV.NEXT + RCV.WIN fixed until:
2592  *  RCV.BUFF - RCV.USER - RCV.WINDOW >= min(1/2 RCV.BUFF, MSS)"
2593  *
2594  * i.e. don't raise the right edge of the window until you can raise
2595  * it at least MSS bytes.
2596  *
2597  * Unfortunately, the recommended algorithm breaks header prediction,
2598  * since header prediction assumes th->window stays fixed.
2599  *
2600  * Strictly speaking, keeping th->window fixed violates the receiver
2601  * side SWS prevention criteria. The problem is that under this rule
2602  * a stream of single byte packets will cause the right side of the
2603  * window to always advance by a single byte.
2604  *
2605  * Of course, if the sender implements sender side SWS prevention
2606  * then this will not be a problem.
2607  *
2608  * BSD seems to make the following compromise:
2609  *
2610  *      If the free space is less than the 1/4 of the maximum
2611  *      space available and the free space is less than 1/2 mss,
2612  *      then set the window to 0.
2613  *      [ Actually, bsd uses MSS and 1/4 of maximal _window_ ]
2614  *      Otherwise, just prevent the window from shrinking
2615  *      and from being larger than the largest representable value.
2616  *
2617  * This prevents incremental opening of the window in the regime
2618  * where TCP is limited by the speed of the reader side taking
2619  * data out of the TCP receive queue. It does nothing about
2620  * those cases where the window is constrained on the sender side
2621  * because the pipeline is full.
2622  *
2623  * BSD also seems to "accidentally" limit itself to windows that are a
2624  * multiple of MSS, at least until the free space gets quite small.
2625  * This would appear to be a side effect of the mbuf implementation.
2626  * Combining these two algorithms results in the observed behavior
2627  * of having a fixed window size at almost all times.
2628  *
2629  * Below we obtain similar behavior by forcing the offered window to
2630  * a multiple of the mss when it is feasible to do so.
2631  *
2632  * Note, we don't "adjust" for TIMESTAMP or SACK option bytes.
2633  * Regular options like TIMESTAMP are taken into account.
2634  */
2635 u32 __tcp_select_window(struct sock *sk)
2636 {
2637         struct inet_connection_sock *icsk = inet_csk(sk);
2638         struct tcp_sock *tp = tcp_sk(sk);
2639         /* MSS for the peer's data.  Previous versions used mss_clamp
2640          * here.  I don't know if the value based on our guesses
2641          * of peer's MSS is better for the performance.  It's more correct
2642          * but may be worse for the performance because of rcv_mss
2643          * fluctuations.  --SAW  1998/11/1
2644          */
2645         int mss = icsk->icsk_ack.rcv_mss;
2646         int free_space = tcp_space(sk);
2647         int allowed_space = tcp_full_space(sk);
2648         int full_space = min_t(int, tp->window_clamp, allowed_space);
2649         int window;
2650 
2651         if (unlikely(mss > full_space)) {
2652                 mss = full_space;
2653                 if (mss <= 0)
2654                         return 0;
2655         }
2656         if (free_space < (full_space >> 1)) {
2657                 icsk->icsk_ack.quick = 0;
2658 
2659                 if (tcp_under_memory_pressure(sk))
2660                         tp->rcv_ssthresh = min(tp->rcv_ssthresh,
2661                                                4U * tp->advmss);
2662 
2663                 /* free_space might become our new window, make sure we don't
2664                  * increase it due to wscale.
2665                  */
2666                 free_space = round_down(free_space, 1 << tp->rx_opt.rcv_wscale);
2667 
2668                 /* if free space is less than mss estimate, or is below 1/16th
2669                  * of the maximum allowed, try to move to zero-window, else
2670                  * tcp_clamp_window() will grow rcv buf up to tcp_rmem[2], and
2671                  * new incoming data is dropped due to memory limits.
2672                  * With large window, mss test triggers way too late in order
2673                  * to announce zero window in time before rmem limit kicks in.
2674                  */
2675                 if (free_space < (allowed_space >> 4) || free_space < mss)
2676                         return 0;
2677         }
2678 
2679         if (free_space > tp->rcv_ssthresh)
2680                 free_space = tp->rcv_ssthresh;
2681 
2682         /* Don't do rounding if we are using window scaling, since the
2683          * scaled window will not line up with the MSS boundary anyway.
2684          */
2685         if (tp->rx_opt.rcv_wscale) {
2686                 window = free_space;
2687 
2688                 /* Advertise enough space so that it won't get scaled away.
2689                  * Import case: prevent zero window announcement if
2690                  * 1<<rcv_wscale > mss.
2691                  */
2692                 window = ALIGN(window, (1 << tp->rx_opt.rcv_wscale));
2693         } else {
2694                 window = tp->rcv_wnd;
2695                 /* Get the largest window that is a nice multiple of mss.
2696                  * Window clamp already applied above.
2697                  * If our current window offering is within 1 mss of the
2698                  * free space we just keep it. This prevents the divide
2699                  * and multiply from happening most of the time.
2700                  * We also don't do any window rounding when the free space
2701                  * is too small.
2702                  */
2703                 if (window <= free_space - mss || window > free_space)
2704                         window = rounddown(free_space, mss);
2705                 else if (mss == full_space &&
2706                          free_space > window + (full_space >> 1))
2707                         window = free_space;
2708         }
2709 
2710         return window;
2711 }
2712 
2713 void tcp_skb_collapse_tstamp(struct sk_buff *skb,
2714                              const struct sk_buff *next_skb)
2715 {
2716         if (unlikely(tcp_has_tx_tstamp(next_skb))) {
2717                 const struct skb_shared_info *next_shinfo =
2718                         skb_shinfo(next_skb);
2719                 struct skb_shared_info *shinfo = skb_shinfo(skb);
2720 
2721                 shinfo->tx_flags |= next_shinfo->tx_flags & SKBTX_ANY_TSTAMP;
2722                 shinfo->tskey = next_shinfo->tskey;
2723                 TCP_SKB_CB(skb)->txstamp_ack |=
2724                         TCP_SKB_CB(next_skb)->txstamp_ack;
2725         }
2726 }
2727 
2728 /* Collapses two adjacent SKB's during retransmission. */
2729 static bool tcp_collapse_retrans(struct sock *sk, struct sk_buff *skb)
2730 {
2731         struct tcp_sock *tp = tcp_sk(sk);
2732         struct sk_buff *next_skb = skb_rb_next(skb);
2733         int skb_size, next_skb_size;
2734 
2735         skb_size = skb->len;
2736         next_skb_size = next_skb->len;
2737 
2738         BUG_ON(tcp_skb_pcount(skb) != 1 || tcp_skb_pcount(next_skb) != 1);
2739 
2740         if (next_skb_size) {
2741                 if (next_skb_size <= skb_availroom(skb))
2742                         skb_copy_bits(next_skb, 0, skb_put(skb, next_skb_size),
2743                                       next_skb_size);
2744                 else if (!skb_shift(skb, next_skb, next_skb_size))
2745                         return false;
2746         }
2747         tcp_highest_sack_replace(sk, next_skb, skb);
2748 
2749         if (next_skb->ip_summed == CHECKSUM_PARTIAL)
2750                 skb->ip_summed = CHECKSUM_PARTIAL;
2751 
2752         if (skb->ip_summed != CHECKSUM_PARTIAL)
2753                 skb->csum = csum_block_add(skb->csum, next_skb->csum, skb_size);
2754 
2755         /* Update sequence range on original skb. */
2756         TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq;
2757 
2758         /* Merge over control information. This moves PSH/FIN etc. over */
2759         TCP_SKB_CB(skb)->tcp_flags |= TCP_SKB_CB(next_skb)->tcp_flags;
2760 
2761         /* All done, get rid of second SKB and account for it so
2762          * packet counting does not break.
2763          */
2764         TCP_SKB_CB(skb)->sacked |= TCP_SKB_CB(next_skb)->sacked & TCPCB_EVER_RETRANS;
2765         TCP_SKB_CB(skb)->eor = TCP_SKB_CB(next_skb)->eor;
2766 
2767         /* changed transmit queue under us so clear hints */
2768         tcp_clear_retrans_hints_partial(tp);
2769         if (next_skb == tp->retransmit_skb_hint)
2770                 tp->retransmit_skb_hint = skb;
2771 
2772         tcp_adjust_pcount(sk, next_skb, tcp_skb_pcount(next_skb));
2773 
2774         tcp_skb_collapse_tstamp(skb, next_skb);
2775 
2776         tcp_rtx_queue_unlink_and_free(next_skb, sk);
2777         return true;
2778 }
2779 
2780 /* Check if coalescing SKBs is legal. */
2781 static bool tcp_can_collapse(const struct sock *sk, const struct sk_buff *skb)
2782 {
2783         if (tcp_skb_pcount(skb) > 1)
2784                 return false;
2785         if (skb_cloned(skb))
2786                 return false;
2787         /* Some heuristics for collapsing over SACK'd could be invented */
2788         if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
2789                 return false;
2790 
2791         return true;
2792 }
2793 
2794 /* Collapse packets in the retransmit queue to make to create
2795  * less packets on the wire. This is only done on retransmission.
2796  */
2797 static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *to,
2798                                      int space)
2799 {
2800         struct tcp_sock *tp = tcp_sk(sk);
2801         struct sk_buff *skb = to, *tmp;
2802         bool first = true;
2803 
2804         if (!sock_net(sk)->ipv4.sysctl_tcp_retrans_collapse)
2805                 return;
2806         if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
2807                 return;
2808 
2809         skb_rbtree_walk_from_safe(skb, tmp) {
2810                 if (!tcp_can_collapse(sk, skb))
2811                         break;
2812 
2813                 if (!tcp_skb_can_collapse_to(to))
2814                         break;
2815 
2816                 space -= skb->len;
2817 
2818                 if (first) {
2819                         first = false;
2820                         continue;
2821                 }
2822 
2823                 if (space < 0)
2824                         break;
2825 
2826                 if (after(TCP_SKB_CB(skb)->end_seq, tcp_wnd_end(tp)))
2827                         break;
2828 
2829                 if (!tcp_collapse_retrans(sk, to))
2830                         break;
2831         }
2832 }
2833 
2834 /* This retransmits one SKB.  Policy decisions and retransmit queue
2835  * state updates are done by the caller.  Returns non-zero if an
2836  * error occurred which prevented the send.
2837  */
2838 int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
2839 {
2840         struct inet_connection_sock *icsk = inet_csk(sk);
2841         struct tcp_sock *tp = tcp_sk(sk);
2842         unsigned int cur_mss;
2843         int diff, len, err;
2844 
2845 
2846         /* Inconclusive MTU probe */
2847         if (icsk->icsk_mtup.probe_size)
2848                 icsk->icsk_mtup.probe_size = 0;
2849 
2850         /* Do not sent more than we queued. 1/4 is reserved for possible
2851          * copying overhead: fragmentation, tunneling, mangling etc.
2852          */
2853         if (refcount_read(&sk->sk_wmem_alloc) >
2854             min_t(u32, sk->sk_wmem_queued + (sk->sk_wmem_queued >> 2),
2855                   sk->sk_sndbuf))
2856                 return -EAGAIN;
2857 
2858         if (skb_still_in_host_queue(sk, skb))
2859                 return -EBUSY;
2860 
2861         if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) {
2862                 if (before(TCP_SKB_CB(skb)->end_seq, tp->snd_una))
2863                         BUG();
2864                 if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
2865                         return -ENOMEM;
2866         }
2867 
2868         if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk))
2869                 return -EHOSTUNREACH; /* Routing failure or similar. */
2870 
2871         cur_mss = tcp_current_mss(sk);
2872 
2873         /* If receiver has shrunk his window, and skb is out of
2874          * new window, do not retransmit it. The exception is the
2875          * case, when window is shrunk to zero. In this case
2876          * our retransmit serves as a zero window probe.
2877          */
2878         if (!before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp)) &&
2879             TCP_SKB_CB(skb)->seq != tp->snd_una)
2880                 return -EAGAIN;
2881 
2882         len = cur_mss * segs;
2883         if (skb->len > len) {
2884                 if (tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb, len,
2885                                  cur_mss, GFP_ATOMIC))
2886                         return -ENOMEM; /* We'll try again later. */
2887         } else {
2888                 if (skb_unclone(skb, GFP_ATOMIC))
2889                         return -ENOMEM;
2890 
2891                 diff = tcp_skb_pcount(skb);
2892                 tcp_set_skb_tso_segs(skb, cur_mss);
2893                 diff -= tcp_skb_pcount(skb);
2894                 if (diff)
2895                         tcp_adjust_pcount(sk, skb, diff);
2896                 if (skb->len < cur_mss)
2897                         tcp_retrans_try_collapse(sk, skb, cur_mss);
2898         }
2899 
2900         /* RFC3168, section 6.1.1.1. ECN fallback */
2901         if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN_ECN) == TCPHDR_SYN_ECN)
2902                 tcp_ecn_clear_syn(sk, skb);
2903 
2904         /* Update global and local TCP statistics. */
2905         segs = tcp_skb_pcount(skb);
2906         TCP_ADD_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS, segs);
2907         if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
2908                 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS);
2909         tp->total_retrans += segs;
2910 
2911         /* make sure skb->data is aligned on arches that require it
2912          * and check if ack-trimming & collapsing extended the headroom
2913          * beyond what csum_start can cover.
2914          */
2915         if (unlikely((NET_IP_ALIGN && ((unsigned long)skb->data & 3)) ||
2916                      skb_headroom(skb) >= 0xFFFF)) {
2917                 struct sk_buff *nskb;
2918 
2919                 tcp_skb_tsorted_save(skb) {
2920                         nskb = __pskb_copy(skb, MAX_TCP_HEADER, GFP_ATOMIC);
2921                         err = nskb ? tcp_transmit_skb(sk, nskb, 0, GFP_ATOMIC) :
2922                                      -ENOBUFS;
2923                 } tcp_skb_tsorted_restore(skb);
2924 
2925                 if (!err) {
2926                         tcp_update_skb_after_send(tp, skb);
2927                         tcp_rate_skb_sent(sk, skb);
2928                 }
2929         } else {
2930                 err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
2931         }
2932 
2933         if (likely(!err)) {
2934                 TCP_SKB_CB(skb)->sacked |= TCPCB_EVER_RETRANS;
2935                 trace_tcp_retransmit_skb(sk, skb);
2936         } else if (err != -EBUSY) {
2937                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRETRANSFAIL);
2938         }
2939         return err;
2940 }
2941 
2942 int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
2943 {
2944         struct tcp_sock *tp = tcp_sk(sk);
2945         int err = __tcp_retransmit_skb(sk, skb, segs);
2946 
2947         if (err == 0) {
2948 #if FASTRETRANS_DEBUG > 0
2949                 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) {
2950                         net_dbg_ratelimited("retrans_out leaked\n");
2951                 }
2952 #endif
2953                 TCP_SKB_CB(skb)->sacked |= TCPCB_RETRANS;
2954                 tp->retrans_out += tcp_skb_pcount(skb);
2955 
2956                 /* Save stamp of the first retransmit. */
2957                 if (!tp->retrans_stamp)
2958                         tp->retrans_stamp = tcp_skb_timestamp(skb);
2959 
2960         }
2961 
2962         if (tp->undo_retrans < 0)
2963                 tp->undo_retrans = 0;
2964         tp->undo_retrans += tcp_skb_pcount(skb);
2965         return err;
2966 }
2967 
2968 /* This gets called after a retransmit timeout, and the initially
2969  * retransmitted data is acknowledged.  It tries to continue
2970  * resending the rest of the retransmit queue, until either
2971  * we've sent it all or the congestion window limit is reached.
2972  */
2973 void tcp_xmit_retransmit_queue(struct sock *sk)
2974 {
2975         const struct inet_connection_sock *icsk = inet_csk(sk);
2976         struct sk_buff *skb, *rtx_head, *hole = NULL;
2977         struct tcp_sock *tp = tcp_sk(sk);
2978         u32 max_segs;
2979         int mib_idx;
2980 
2981         if (!tp->packets_out)
2982                 return;
2983 
2984         rtx_head = tcp_rtx_queue_head(sk);
2985         skb = tp->retransmit_skb_hint ?: rtx_head;
2986         max_segs = tcp_tso_segs(sk, tcp_current_mss(sk));
2987         skb_rbtree_walk_from(skb) {
2988                 __u8 sacked;
2989                 int segs;
2990 
2991                 if (tcp_pacing_check(sk))
2992                         break;
2993 
2994                 /* we could do better than to assign each time */
2995                 if (!hole)
2996                         tp->retransmit_skb_hint = skb;
2997 
2998                 segs = tp->snd_cwnd - tcp_packets_in_flight(tp);
2999                 if (segs <= 0)
3000                         return;
3001                 sacked = TCP_SKB_CB(skb)->sacked;
3002                 /* In case tcp_shift_skb_data() have aggregated large skbs,
3003                  * we need to make sure not sending too bigs TSO packets
3004                  */
3005                 segs = min_t(int, segs, max_segs);
3006 
3007                 if (tp->retrans_out >= tp->lost_out) {
3008                         break;
3009                 } else if (!(sacked & TCPCB_LOST)) {
3010                         if (!hole && !(sacked & (TCPCB_SACKED_RETRANS|TCPCB_SACKED_ACKED)))
3011                                 hole = skb;
3012                         continue;
3013 
3014                 } else {
3015                         if (icsk->icsk_ca_state != TCP_CA_Loss)
3016                                 mib_idx = LINUX_MIB_TCPFASTRETRANS;
3017                         else
3018                                 mib_idx = LINUX_MIB_TCPSLOWSTARTRETRANS;
3019                 }
3020 
3021                 if (sacked & (TCPCB_SACKED_ACKED|TCPCB_SACKED_RETRANS))
3022                         continue;
3023 
3024                 if (tcp_small_queue_check(sk, skb, 1))
3025                         return;
3026 
3027                 if (tcp_retransmit_skb(sk, skb, segs))
3028                         return;
3029 
3030                 NET_ADD_STATS(sock_net(sk), mib_idx, tcp_skb_pcount(skb));
3031 
3032                 if (tcp_in_cwnd_reduction(sk))
3033                         tp->prr_out += tcp_skb_pcount(skb);
3034 
3035                 if (skb == rtx_head &&
3036                     icsk->icsk_pending != ICSK_TIME_REO_TIMEOUT)
3037                         inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
3038                                                   inet_csk(sk)->icsk_rto,
3039                                                   TCP_RTO_MAX);
3040         }
3041 }
3042 
3043 /* We allow to exceed memory limits for FIN packets to expedite
3044  * connection tear down and (memory) recovery.
3045  * Otherwise tcp_send_fin() could be tempted to either delay FIN
3046  * or even be forced to close flow without any FIN.
3047  * In general, we want to allow one skb per socket to avoid hangs
3048  * with edge trigger epoll()
3049  */
3050 void sk_forced_mem_schedule(struct sock *sk, int size)
3051 {
3052         int amt;
3053 
3054         if (size <= sk->sk_forward_alloc)
3055                 return;
3056         amt = sk_mem_pages(size);
3057         sk->sk_forward_alloc += amt * SK_MEM_QUANTUM;
3058         sk_memory_allocated_add(sk, amt);
3059 
3060         if (mem_cgroup_sockets_enabled && sk->sk_memcg)
3061                 mem_cgroup_charge_skmem(sk->sk_memcg, amt);
3062 }
3063 
3064 /* Send a FIN. The caller locks the socket for us.
3065  * We should try to send a FIN packet really hard, but eventually give up.
3066  */
3067 void tcp_send_fin(struct sock *sk)
3068 {
3069         struct sk_buff *skb, *tskb = tcp_write_queue_tail(sk);
3070         struct tcp_sock *tp = tcp_sk(sk);
3071 
3072         /* Optimization, tack on the FIN if we have one skb in write queue and
3073          * this skb was not yet sent, or we are under memory pressure.
3074          * Note: in the latter case, FIN packet will be sent after a timeout,
3075          * as TCP stack thinks it has already been transmitted.
3076          */
3077         if (!tskb && tcp_under_memory_pressure(sk))
3078                 tskb = skb_rb_last(&sk->tcp_rtx_queue);
3079 
3080         if (tskb) {
3081 coalesce:
3082                 TCP_SKB_CB(tskb)->tcp_flags |= TCPHDR_FIN;
3083                 TCP_SKB_CB(tskb)->end_seq++;
3084                 tp->write_seq++;
3085                 if (tcp_write_queue_empty(sk)) {
3086                         /* This means tskb was already sent.
3087                          * Pretend we included the FIN on previous transmit.
3088                          * We need to set tp->snd_nxt to the value it would have
3089                          * if FIN had been sent. This is because retransmit path
3090                          * does not change tp->snd_nxt.
3091                          */
3092                         tp->snd_nxt++;
3093                         return;
3094                 }
3095         } else {
3096                 skb = alloc_skb_fclone(MAX_TCP_HEADER, sk->sk_allocation);
3097                 if (unlikely(!skb)) {
3098                         if (tskb)
3099                                 goto coalesce;
3100                         return;
3101                 }
3102                 INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
3103                 skb_reserve(skb, MAX_TCP_HEADER);
3104                 sk_forced_mem_schedule(sk, skb->truesize);
3105                 /* FIN eats a sequence byte, write_seq advanced by tcp_queue_skb(). */
3106                 tcp_init_nondata_skb(skb, tp->write_seq,
3107                                      TCPHDR_ACK | TCPHDR_FIN);
3108                 tcp_queue_skb(sk, skb);
3109         }
3110         __tcp_push_pending_frames(sk, tcp_current_mss(sk), TCP_NAGLE_OFF);
3111 }
3112 
3113 /* We get here when a process closes a file descriptor (either due to
3114  * an explicit close() or as a byproduct of exit()'ing) and there
3115  * was unread data in the receive queue.  This behavior is recommended
3116  * by RFC 2525, section 2.17.  -DaveM
3117  */
3118 void tcp_send_active_reset(struct sock *sk, gfp_t priority)
3119 {
3120         struct sk_buff *skb;
3121 
3122         TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTRSTS);
3123 
3124         /* NOTE: No TCP options attached and we never retransmit this. */
3125         skb = alloc_skb(MAX_TCP_HEADER, priority);
3126         if (!skb) {
3127                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
3128                 return;
3129         }
3130 
3131         /* Reserve space for headers and prepare control bits. */
3132         skb_reserve(skb, MAX_TCP_HEADER);
3133         tcp_init_nondata_skb(skb, tcp_acceptable_seq(sk),
3134                              TCPHDR_ACK | TCPHDR_RST);
3135         tcp_mstamp_refresh(tcp_sk(sk));
3136         /* Send it off. */
3137         if (tcp_transmit_skb(sk, skb, 0, priority))
3138                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
3139 
3140         /* skb of trace_tcp_send_reset() keeps the skb that caused RST,
3141          * skb here is different to the troublesome skb, so use NULL
3142          */
3143         trace_tcp_send_reset(sk, NULL);
3144 }
3145 
3146 /* Send a crossed SYN-ACK during socket establishment.
3147  * WARNING: This routine must only be called when we have already sent
3148  * a SYN packet that crossed the incoming SYN that caused this routine
3149  * to get called. If this assumption fails then the initial rcv_wnd
3150  * and rcv_wscale values will not be correct.
3151  */
3152 int tcp_send_synack(struct sock *sk)
3153 {
3154         struct sk_buff *skb;
3155 
3156         skb = tcp_rtx_queue_head(sk);
3157         if (!skb || !(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
3158                 pr_err("%s: wrong queue state\n", __func__);
3159                 return -EFAULT;
3160         }
3161         if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK)) {
3162                 if (skb_cloned(skb)) {
3163                         struct sk_buff *nskb;
3164 
3165                         tcp_skb_tsorted_save(skb) {
3166                                 nskb = skb_copy(skb, GFP_ATOMIC);
3167                         } tcp_skb_tsorted_restore(skb);
3168                         if (!nskb)
3169                                 return -ENOMEM;
3170                         INIT_LIST_HEAD(&nskb->tcp_tsorted_anchor);
3171                         tcp_rtx_queue_unlink_and_free(skb, sk);
3172                         __skb_header_release(nskb);
3173                         tcp_rbtree_insert(&sk->tcp_rtx_queue, nskb);
3174                         sk->sk_wmem_queued += nskb->truesize;
3175                         sk_mem_charge(sk, nskb->truesize);
3176                         skb = nskb;
3177                 }
3178 
3179                 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ACK;
3180                 tcp_ecn_send_synack(sk, skb);
3181         }
3182         return tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
3183 }
3184 
3185 /**
3186  * tcp_make_synack - Prepare a SYN-ACK.
3187  * sk: listener socket
3188  * dst: dst entry attached to the SYNACK
3189  * req: request_sock pointer
3190  *
3191  * Allocate one skb and build a SYNACK packet.
3192  * @dst is consumed : Caller should not use it again.
3193  */
3194 struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
3195                                 struct request_sock *req,
3196                                 struct tcp_fastopen_cookie *foc,
3197                                 enum tcp_synack_type synack_type)
3198 {
3199         struct inet_request_sock *ireq = inet_rsk(req);
3200         const struct tcp_sock *tp = tcp_sk(sk);
3201         struct tcp_md5sig_key *md5 = NULL;
3202         struct tcp_out_options opts;
3203         struct sk_buff *skb;
3204         int tcp_header_size;
3205         struct tcphdr *th;
3206         int mss;
3207 
3208         skb = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
3209         if (unlikely(!skb)) {
3210                 dst_release(dst);
3211                 return NULL;
3212         }
3213         /* Reserve space for headers. */
3214         skb_reserve(skb, MAX_TCP_HEADER);
3215 
3216         switch (synack_type) {
3217         case TCP_SYNACK_NORMAL:
3218                 skb_set_owner_w(skb, req_to_sk(req));
3219                 break;
3220         case TCP_SYNACK_COOKIE:
3221                 /* Under synflood, we do not attach skb to a socket,
3222                  * to avoid false sharing.
3223                  */
3224                 break;
3225         case TCP_SYNACK_FASTOPEN:
3226                 /* sk is a const pointer, because we want to express multiple
3227                  * cpu might call us concurrently.
3228                  * sk->sk_wmem_alloc in an atomic, we can promote to rw.
3229                  */
3230                 skb_set_owner_w(skb, (struct sock *)sk);
3231                 break;
3232         }
3233         skb_dst_set(skb, dst);
3234 
3235         mss = tcp_mss_clamp(tp, dst_metric_advmss(dst));
3236 
3237         memset(&opts, 0, sizeof(opts));
3238 #ifdef CONFIG_SYN_COOKIES
3239         if (unlikely(req->cookie_ts))
3240                 skb->skb_mstamp = cookie_init_timestamp(req);
3241         else
3242 #endif
3243                 skb->skb_mstamp = tcp_clock_us();
3244 
3245 #ifdef CONFIG_TCP_MD5SIG
3246         rcu_read_lock();
3247         md5 = tcp_rsk(req)->af_specific->req_md5_lookup(sk, req_to_sk(req));
3248 #endif
3249         skb_set_hash(skb, tcp_rsk(req)->txhash, PKT_HASH_TYPE_L4);
3250         tcp_header_size = tcp_synack_options(sk, req, mss, skb, &opts, md5,
3251                                              foc) + sizeof(*th);
3252 
3253         skb_push(skb, tcp_header_size);
3254         skb_reset_transport_header(skb);
3255 
3256         th = (struct tcphdr *)skb->data;
3257         memset(th, 0, sizeof(struct tcphdr));
3258         th->syn = 1;
3259         th->ack = 1;
3260         tcp_ecn_make_synack(req, th);
3261         th->source = htons(ireq->ir_num);
3262         th->dest = ireq->ir_rmt_port;
3263         skb->mark = ireq->ir_mark;
3264         skb->ip_summed = CHECKSUM_PARTIAL;
3265         th->seq = htonl(tcp_rsk(req)->snt_isn);
3266         /* XXX data is queued and acked as is. No buffer/window check */
3267         th->ack_seq = htonl(tcp_rsk(req)->rcv_nxt);
3268 
3269         /* RFC1323: The window in SYN & SYN/ACK segments is never scaled. */
3270         th->window = htons(min(req->rsk_rcv_wnd, 65535U));
3271         tcp_options_write((__be32 *)(th + 1), NULL, &opts);
3272         th->doff = (tcp_header_size >> 2);
3273         __TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTSEGS);
3274 
3275 #ifdef CONFIG_TCP_MD5SIG
3276         /* Okay, we have all we need - do the md5 hash if needed */
3277         if (md5)
3278                 tcp_rsk(req)->af_specific->calc_md5_hash(opts.hash_location,
3279                                                md5, req_to_sk(req), skb);
3280         rcu_read_unlock();
3281 #endif
3282 
3283         /* Do not fool tcpdump (if any), clean our debris */
3284         skb->tstamp = 0;
3285         return skb;
3286 }
3287 EXPORT_SYMBOL(tcp_make_synack);
3288 
3289 static void tcp_ca_dst_init(struct sock *sk, const struct dst_entry *dst)
3290 {
3291         struct inet_connection_sock *icsk = inet_csk(sk);
3292         const struct tcp_congestion_ops *ca;
3293         u32 ca_key = dst_metric(dst, RTAX_CC_ALGO);
3294 
3295         if (ca_key == TCP_CA_UNSPEC)
3296                 return;
3297 
3298         rcu_read_lock();
3299         ca = tcp_ca_find_key(ca_key);
3300         if (likely(ca && try_module_get(ca->owner))) {
3301                 module_put(icsk->icsk_ca_ops->owner);
3302                 icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst);
3303                 icsk->icsk_ca_ops = ca;
3304         }
3305         rcu_read_unlock();
3306 }
3307 
3308 /* Do all connect socket setups that can be done AF independent. */
3309 static void tcp_connect_init(struct sock *sk)
3310 {
3311         const struct dst_entry *dst = __sk_dst_get(sk);
3312         struct tcp_sock *tp = tcp_sk(sk);
3313         __u8 rcv_wscale;
3314         u32 rcv_wnd;
3315 
3316         /* We'll fix this up when we get a response from the other end.
3317          * See tcp_input.c:tcp_rcv_state_process case TCP_SYN_SENT.
3318          */
3319         tp->tcp_header_len = sizeof(struct tcphdr);
3320         if (sock_net(sk)->ipv4.sysctl_tcp_timestamps)
3321                 tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
3322 
3323 #ifdef CONFIG_TCP_MD5SIG
3324         if (tp->af_specific->md5_lookup(sk, sk))
3325                 tp->tcp_header_len += TCPOLEN_MD5SIG_ALIGNED;
3326 #endif
3327 
3328         /* If user gave his TCP_MAXSEG, record it to clamp */
3329         if (tp->rx_opt.user_mss)
3330                 tp->rx_opt.mss_clamp = tp->rx_opt.user_mss;
3331         tp->max_window = 0;
3332         tcp_mtup_init(sk);
3333         tcp_sync_mss(sk, dst_mtu(dst));
3334 
3335         tcp_ca_dst_init(sk, dst);
3336 
3337         if (!tp->window_clamp)
3338                 tp->window_clamp = dst_metric(dst, RTAX_WINDOW);
3339         tp->advmss = tcp_mss_clamp(tp, dst_metric_advmss(dst));
3340 
3341         tcp_initialize_rcv_mss(sk);
3342 
3343         /* limit the window selection if the user enforce a smaller rx buffer */
3344         if (sk->sk_userlocks & SOCK_RCVBUF_LOCK &&
3345             (tp->window_clamp > tcp_full_space(sk) || tp->window_clamp == 0))
3346                 tp->window_clamp = tcp_full_space(sk);
3347 
3348         rcv_wnd = tcp_rwnd_init_bpf(sk);
3349         if (rcv_wnd == 0)
3350                 rcv_wnd = dst_metric(dst, RTAX_INITRWND);
3351 
3352         tcp_select_initial_window(sk, tcp_full_space(sk),
3353                                   tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0),
3354                                   &tp->rcv_wnd,
3355                                   &tp->window_clamp,
3356                                   sock_net(sk)->ipv4.sysctl_tcp_window_scaling,
3357                                   &rcv_wscale,
3358                                   rcv_wnd);
3359 
3360         tp->rx_opt.rcv_wscale = rcv_wscale;
3361         tp->rcv_ssthresh = tp->rcv_wnd;
3362 
3363         sk->sk_err = 0;
3364         sock_reset_flag(sk, SOCK_DONE);
3365         tp->snd_wnd = 0;
3366         tcp_init_wl(tp, 0);
3367         tp->snd_una = tp->write_seq;
3368         tp->snd_sml = tp->write_seq;
3369         tp->snd_up = tp->write_seq;
3370         tp->snd_nxt = tp->write_seq;
3371 
3372         if (likely(!tp->repair))
3373                 tp->rcv_nxt = 0;
3374         else
3375                 tp->rcv_tstamp = tcp_jiffies32;
3376         tp->rcv_wup = tp->rcv_nxt;
3377         tp->copied_seq = tp->rcv_nxt;
3378 
3379         inet_csk(sk)->icsk_rto = tcp_timeout_init(sk);
3380         inet_csk(sk)->icsk_retransmits = 0;
3381         tcp_clear_retrans(tp);
3382 }
3383 
3384 static void tcp_connect_queue_skb(struct sock *sk, struct sk_buff *skb)
3385 {
3386         struct tcp_sock *tp = tcp_sk(sk);
3387         struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
3388 
3389         tcb->end_seq += skb->len;
3390         __skb_header_release(skb);
3391         sk->sk_wmem_queued += skb->truesize;
3392         sk_mem_charge(sk, skb->truesize);
3393         tp->write_seq = tcb->end_seq;
3394         tp->packets_out += tcp_skb_pcount(skb);
3395 }
3396 
3397 /* Build and send a SYN with data and (cached) Fast Open cookie. However,
3398  * queue a data-only packet after the regular SYN, such that regular SYNs
3399  * are retransmitted on timeouts. Also if the remote SYN-ACK acknowledges
3400  * only the SYN sequence, the data are retransmitted in the first ACK.
3401  * If cookie is not cached or other error occurs, falls back to send a
3402  * regular SYN with Fast Open cookie request option.
3403  */
3404 static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn)
3405 {
3406         struct tcp_sock *tp = tcp_sk(sk);
3407         struct tcp_fastopen_request *fo = tp->fastopen_req;
3408         int space, err = 0;
3409         struct sk_buff *syn_data;
3410 
3411         tp->rx_opt.mss_clamp = tp->advmss;  /* If MSS is not cached */
3412         if (!tcp_fastopen_cookie_check(sk, &tp->rx_opt.mss_clamp, &fo->cookie))
3413                 goto fallback;
3414 
3415         /* MSS for SYN-data is based on cached MSS and bounded by PMTU and
3416          * user-MSS. Reserve maximum option space for middleboxes that add
3417          * private TCP options. The cost is reduced data space in SYN :(
3418          */
3419         tp->rx_opt.mss_clamp = tcp_mss_clamp(tp, tp->rx_opt.mss_clamp);
3420 
3421         space = __tcp_mtu_to_mss(sk, inet_csk(sk)->icsk_pmtu_cookie) -
3422                 MAX_TCP_OPTION_SPACE;
3423 
3424         space = min_t(size_t, space, fo->size);
3425 
3426         /* limit to order-0 allocations */
3427         space = min_t(size_t, space, SKB_MAX_HEAD(MAX_TCP_HEADER));
3428 
3429         syn_data = sk_stream_alloc_skb(sk, space, sk->sk_allocation, false);
3430         if (!syn_data)
3431                 goto fallback;
3432         syn_data->ip_summed = CHECKSUM_PARTIAL;
3433         memcpy(syn_data->cb, syn->cb, sizeof(syn->cb));
3434         if (space) {
3435                 int copied = copy_from_iter(skb_put(syn_data, space), space,
3436                                             &fo->data->msg_iter);
3437                 if (unlikely(!copied)) {
3438                         tcp_skb_tsorted_anchor_cleanup(syn_data);
3439                         kfree_skb(syn_data);
3440                         goto fallback;
3441                 }
3442                 if (copied != space) {
3443                         skb_trim(syn_data, copied);
3444                         space = copied;
3445                 }
3446         }
3447         /* No more data pending in inet_wait_for_connect() */
3448         if (space == fo->size)
3449                 fo->data = NULL;
3450         fo->copied = space;
3451 
3452         tcp_connect_queue_skb(sk, syn_data);
3453         if (syn_data->len)
3454                 tcp_chrono_start(sk, TCP_CHRONO_BUSY);
3455 
3456         err = tcp_transmit_skb(sk, syn_data, 1, sk->sk_allocation);
3457 
3458         syn->skb_mstamp = syn_data->skb_mstamp;
3459 
3460         /* Now full SYN+DATA was cloned and sent (or not),
3461          * remove the SYN from the original skb (syn_data)
3462          * we keep in write queue in case of a retransmit, as we
3463          * also have the SYN packet (with no data) in the same queue.
3464          */
3465         TCP_SKB_CB(syn_data)->seq++;
3466         TCP_SKB_CB(syn_data)->tcp_flags = TCPHDR_ACK | TCPHDR_PSH;
3467         if (!err) {
3468                 tp->syn_data = (fo->copied > 0);
3469                 tcp_rbtree_insert(&sk->tcp_rtx_queue, syn_data);
3470                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT);
3471                 goto done;
3472         }
3473 
3474         /* data was not sent, put it in write_queue */
3475         __skb_queue_tail(&sk->sk_write_queue, syn_data);
3476         tp->packets_out -= tcp_skb_pcount(syn_data);
3477 
3478 fallback:
3479         /* Send a regular SYN with Fast Open cookie request option */
3480         if (fo->cookie.len > 0)
3481                 fo->cookie.len = 0;
3482         err = tcp_transmit_skb(sk, syn, 1, sk->sk_allocation);
3483         if (err)
3484                 tp->syn_fastopen = 0;
3485 done:
3486         fo->cookie.len = -1;  /* Exclude Fast Open option for SYN retries */
3487         return err;
3488 }
3489 
3490 /* Build a SYN and send it off. */
3491 int tcp_connect(struct sock *sk)
3492 {
3493         struct tcp_sock *tp = tcp_sk(sk);
3494         struct sk_buff *buff;
3495         int err;
3496 
3497         tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB);
3498 
3499         if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk))
3500                 return -EHOSTUNREACH; /* Routing failure or similar. */
3501 
3502         tcp_connect_init(sk);
3503 
3504         if (unlikely(tp->repair)) {
3505                 tcp_finish_connect(sk, NULL);
3506                 return 0;
3507         }
3508 
3509         buff = sk_stream_alloc_skb(sk, 0, sk->sk_allocation, true);
3510         if (unlikely(!buff))
3511                 return -ENOBUFS;
3512 
3513         tcp_init_nondata_skb(buff, tp->write_seq++, TCPHDR_SYN);
3514         tcp_mstamp_refresh(tp);
3515         tp->retrans_stamp = tcp_time_stamp(tp);
3516         tcp_connect_queue_skb(sk, buff);
3517         tcp_ecn_send_syn(sk, buff);
3518         tcp_rbtree_insert(&sk->tcp_rtx_queue, buff);
3519 
3520         /* Send off SYN; include data in Fast Open. */
3521         err = tp->fastopen_req ? tcp_send_syn_data(sk, buff) :
3522               tcp_transmit_skb(sk, buff, 1, sk->sk_allocation);
3523         if (err == -ECONNREFUSED)
3524                 return err;
3525 
3526         /* We change tp->snd_nxt after the tcp_transmit_skb() call
3527          * in order to make this packet get counted in tcpOutSegs.
3528          */
3529         tp->snd_nxt = tp->write_seq;
3530         tp->pushed_seq = tp->write_seq;
3531         buff = tcp_send_head(sk);
3532         if (unlikely(buff)) {
3533                 tp->snd_nxt     = TCP_SKB_CB(buff)->seq;
3534                 tp->pushed_seq  = TCP_SKB_CB(buff)->seq;
3535         }
3536         TCP_INC_STATS(sock_net(sk), TCP_MIB_ACTIVEOPENS);
3537 
3538         /* Timer for repeating the SYN until an answer. */
3539         inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
3540                                   inet_csk(sk)->icsk_rto, TCP_RTO_MAX);
3541         return 0;
3542 }
3543 EXPORT_SYMBOL(tcp_connect);
3544 
3545 /* Send out a delayed ack, the caller does the policy checking
3546  * to see if we should even be here.  See tcp_input.c:tcp_ack_snd_check()
3547  * for details.
3548  */
3549 void tcp_send_delayed_ack(struct sock *sk)
3550 {
3551         struct inet_connection_sock *icsk = inet_csk(sk);
3552         int ato = icsk->icsk_ack.ato;
3553         unsigned long timeout;
3554 
3555         tcp_ca_event(sk, CA_EVENT_DELAYED_ACK);
3556 
3557         if (ato > TCP_DELACK_MIN) {
3558                 const struct tcp_sock *tp = tcp_sk(sk);
3559                 int max_ato = HZ / 2;
3560 
3561                 if (icsk->icsk_ack.pingpong ||
3562                     (icsk->icsk_ack.pending & ICSK_ACK_PUSHED))
3563                         max_ato = TCP_DELACK_MAX;
3564 
3565                 /* Slow path, intersegment interval is "high". */
3566 
3567                 /* If some rtt estimate is known, use it to bound delayed ack.
3568                  * Do not use inet_csk(sk)->icsk_rto here, use results of rtt measurements
3569                  * directly.
3570                  */
3571                 if (tp->srtt_us) {
3572                         int rtt = max_t(int, usecs_to_jiffies(tp->srtt_us >> 3),
3573                                         TCP_DELACK_MIN);
3574 
3575                         if (rtt < max_ato)
3576                                 max_ato = rtt;
3577                 }
3578 
3579                 ato = min(ato, max_ato);
3580         }
3581 
3582         /* Stay within the limit we were given */
3583         timeout = jiffies + ato;
3584 
3585         /* Use new timeout only if there wasn't a older one earlier. */
3586         if (icsk->icsk_ack.pending & ICSK_ACK_TIMER) {
3587                 /* If delack timer was blocked or is about to expire,
3588                  * send ACK now.
3589                  */
3590                 if (icsk->icsk_ack.blocked ||
3591                     time_before_eq(icsk->icsk_ack.timeout, jiffies + (ato >> 2))) {
3592                         tcp_send_ack(sk);
3593                         return;
3594                 }
3595 
3596                 if (!time_before(timeout, icsk->icsk_ack.timeout))
3597                         timeout = icsk->icsk_ack.timeout;
3598         }
3599         icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER;
3600         icsk->icsk_ack.timeout = timeout;
3601         sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout);
3602 }
3603 
3604 /* This routine sends an ack and also updates the window. */
3605 void tcp_send_ack(struct sock *sk)
3606 {
3607         struct sk_buff *buff;
3608 
3609         /* If we have been reset, we may not send again. */
3610         if (sk->sk_state == TCP_CLOSE)
3611                 return;
3612 
3613         tcp_ca_event(sk, CA_EVENT_NON_DELAYED_ACK);
3614 
3615         /* We are not putting this on the write queue, so
3616          * tcp_transmit_skb() will set the ownership to this
3617          * sock.
3618          */
3619         buff = alloc_skb(MAX_TCP_HEADER,
3620                          sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN));
3621         if (unlikely(!buff)) {
3622                 inet_csk_schedule_ack(sk);
3623                 inet_csk(sk)->icsk_ack.ato = TCP_ATO_MIN;
3624                 inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK,
3625                                           TCP_DELACK_MAX, TCP_RTO_MAX);
3626                 return;
3627         }
3628 
3629         /* Reserve space for headers and prepare control bits. */
3630         skb_reserve(buff, MAX_TCP_HEADER);
3631         tcp_init_nondata_skb(buff, tcp_acceptable_seq(sk), TCPHDR_ACK);
3632 
3633         /* We do not want pure acks influencing TCP Small Queues or fq/pacing
3634          * too much.
3635          * SKB_TRUESIZE(max(1 .. 66, MAX_TCP_HEADER)) is unfortunately ~784
3636          */
3637         skb_set_tcp_pure_ack(buff);
3638 
3639         /* Send it off, this clears delayed acks for us. */
3640         tcp_transmit_skb(sk, buff, 0, (__force gfp_t)0);
3641 }
3642 EXPORT_SYMBOL_GPL(tcp_send_ack);
3643 
3644 /* This routine sends a packet with an out of date sequence
3645  * number. It assumes the other end will try to ack it.
3646  *
3647  * Question: what should we make while urgent mode?
3648  * 4.4BSD forces sending single byte of data. We cannot send
3649  * out of window data, because we have SND.NXT==SND.MAX...
3650  *
3651  * Current solution: to send TWO zero-length segments in urgent mode:
3652  * one is with SEG.SEQ=SND.UNA to deliver urgent pointer, another is
3653  * out-of-date with SND.UNA-1 to probe window.
3654  */
3655 static int tcp_xmit_probe_skb(struct sock *sk, int urgent, int mib)
3656 {
3657         struct tcp_sock *tp = tcp_sk(sk);
3658         struct sk_buff *skb;
3659 
3660         /* We don't queue it, tcp_transmit_skb() sets ownership. */
3661         skb = alloc_skb(MAX_TCP_HEADER,
3662                         sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN));
3663         if (!skb)
3664                 return -1;
3665 
3666         /* Reserve space for headers and set control bits. */
3667         skb_reserve(skb, MAX_TCP_HEADER);
3668         /* Use a previous sequence.  This should cause the other
3669          * end to send an ack.  Don't queue or clone SKB, just
3670          * send it.
3671          */
3672         tcp_init_nondata_skb(skb, tp->snd_una - !urgent, TCPHDR_ACK);
3673         NET_INC_STATS(sock_net(sk), mib);
3674         return tcp_transmit_skb(sk, skb, 0, (__force gfp_t)0);
3675 }
3676 
3677 /* Called from setsockopt( ... TCP_REPAIR ) */
3678 void tcp_send_window_probe(struct sock *sk)
3679 {
3680         if (sk->sk_state == TCP_ESTABLISHED) {
3681                 tcp_sk(sk)->snd_wl1 = tcp_sk(sk)->rcv_nxt - 1;
3682                 tcp_mstamp_refresh(tcp_sk(sk));
3683                 tcp_xmit_probe_skb(sk, 0, LINUX_MIB_TCPWINPROBE);
3684         }
3685 }
3686 
3687 /* Initiate keepalive or window probe from timer. */
3688 int tcp_write_wakeup(struct sock *sk, int mib)
3689 {
3690         struct tcp_sock *tp = tcp_sk(sk);
3691         struct sk_buff *skb;
3692 
3693         if (sk->sk_state == TCP_CLOSE)
3694                 return -1;
3695 
3696         skb = tcp_send_head(sk);
3697         if (skb && before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp))) {
3698                 int err;
3699                 unsigned int mss = tcp_current_mss(sk);
3700                 unsigned int seg_size = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
3701 
3702                 if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq))
3703                         tp->pushed_seq = TCP_SKB_CB(skb)->end_seq;
3704 
3705                 /* We are probing the opening of a window
3706                  * but the window size is != 0
3707                  * must have been a result SWS avoidance ( sender )
3708                  */
3709                 if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq ||
3710                     skb->len > mss) {
3711                         seg_size = min(seg_size, mss);
3712                         TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
3713                         if (tcp_fragment(sk, TCP_FRAG_IN_WRITE_QUEUE,
3714                                          skb, seg_size, mss, GFP_ATOMIC))
3715                                 return -1;
3716                 } else if (!tcp_skb_pcount(skb))
3717                         tcp_set_skb_tso_segs(skb, mss);
3718 
3719                 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
3720                 err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
3721                 if (!err)
3722                         tcp_event_new_data_sent(sk, skb);
3723                 return err;
3724         } else {
3725                 if (between(tp->snd_up, tp->snd_una + 1, tp->snd_una + 0xFFFF))
3726                         tcp_xmit_probe_skb(sk, 1, mib);
3727                 return tcp_xmit_probe_skb(sk, 0, mib);
3728         }
3729 }
3730 
3731 /* A window probe timeout has occurred.  If window is not closed send
3732  * a partial packet else a zero probe.
3733  */
3734 void tcp_send_probe0(struct sock *sk)
3735 {
3736         struct inet_connection_sock *icsk = inet_csk(sk);
3737         struct tcp_sock *tp = tcp_sk(sk);
3738         struct net *net = sock_net(sk);
3739         unsigned long probe_max;
3740         int err;
3741 
3742         err = tcp_write_wakeup(sk, LINUX_MIB_TCPWINPROBE);
3743 
3744         if (tp->packets_out || tcp_write_queue_empty(sk)) {
3745                 /* Cancel probe timer, if it is not required. */
3746                 icsk->icsk_probes_out = 0;
3747                 icsk->icsk_backoff = 0;
3748                 return;
3749         }
3750 
3751         if (err <= 0) {
3752                 if (icsk->icsk_backoff < net->ipv4.sysctl_tcp_retries2)
3753                         icsk->icsk_backoff++;
3754                 icsk->icsk_probes_out++;
3755                 probe_max = TCP_RTO_MAX;
3756         } else {
3757                 /* If packet was not sent due to local congestion,
3758                  * do not backoff and do not remember icsk_probes_out.
3759                  * Let local senders to fight for local resources.
3760                  *
3761                  * Use accumulated backoff yet.
3762                  */
3763                 if (!icsk->icsk_probes_out)
3764                         icsk->icsk_probes_out = 1;
3765                 probe_max = TCP_RESOURCE_PROBE_INTERVAL;
3766         }
3767         inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
3768                                   tcp_probe0_when(sk, probe_max),
3769                                   TCP_RTO_MAX);
3770 }
3771 
3772 int tcp_rtx_synack(const struct sock *sk, struct request_sock *req)
3773 {
3774         const struct tcp_request_sock_ops *af_ops = tcp_rsk(req)->af_specific;
3775         struct flowi fl;
3776         int res;
3777 
3778         tcp_rsk(req)->txhash = net_tx_rndhash();
3779         res = af_ops->send_synack(sk, NULL, &fl, req, NULL, TCP_SYNACK_NORMAL);
3780         if (!res) {
3781                 __TCP_INC_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS);
3782                 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS);
3783                 if (unlikely(tcp_passive_fastopen(sk)))
3784                         tcp_sk(sk)->total_retrans++;
3785                 trace_tcp_retransmit_synack(sk, req);
3786         }
3787         return res;
3788 }
3789 EXPORT_SYMBOL(tcp_rtx_synack);
3790 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp