~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/ipv4/tcp_metrics.c

Version: ~ [ linux-5.6 ] ~ [ linux-5.5.13 ] ~ [ linux-5.4.28 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.113 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.174 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.217 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.217 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.82 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 #include <linux/rcupdate.h>
  2 #include <linux/spinlock.h>
  3 #include <linux/jiffies.h>
  4 #include <linux/module.h>
  5 #include <linux/cache.h>
  6 #include <linux/slab.h>
  7 #include <linux/init.h>
  8 #include <linux/tcp.h>
  9 #include <linux/hash.h>
 10 #include <linux/tcp_metrics.h>
 11 #include <linux/vmalloc.h>
 12 
 13 #include <net/inet_connection_sock.h>
 14 #include <net/net_namespace.h>
 15 #include <net/request_sock.h>
 16 #include <net/inetpeer.h>
 17 #include <net/sock.h>
 18 #include <net/ipv6.h>
 19 #include <net/dst.h>
 20 #include <net/tcp.h>
 21 #include <net/genetlink.h>
 22 
 23 int sysctl_tcp_nometrics_save __read_mostly;
 24 
 25 static struct tcp_metrics_block *__tcp_get_metrics(const struct inetpeer_addr *addr,
 26                                                    struct net *net, unsigned int hash);
 27 
 28 struct tcp_fastopen_metrics {
 29         u16     mss;
 30         u16     syn_loss:10;            /* Recurring Fast Open SYN losses */
 31         unsigned long   last_syn_loss;  /* Last Fast Open SYN loss */
 32         struct  tcp_fastopen_cookie     cookie;
 33 };
 34 
 35 struct tcp_metrics_block {
 36         struct tcp_metrics_block __rcu  *tcpm_next;
 37         struct inetpeer_addr            tcpm_addr;
 38         unsigned long                   tcpm_stamp;
 39         u32                             tcpm_ts;
 40         u32                             tcpm_ts_stamp;
 41         u32                             tcpm_lock;
 42         u32                             tcpm_vals[TCP_METRIC_MAX + 1];
 43         struct tcp_fastopen_metrics     tcpm_fastopen;
 44 
 45         struct rcu_head                 rcu_head;
 46 };
 47 
 48 static bool tcp_metric_locked(struct tcp_metrics_block *tm,
 49                               enum tcp_metric_index idx)
 50 {
 51         return tm->tcpm_lock & (1 << idx);
 52 }
 53 
 54 static u32 tcp_metric_get(struct tcp_metrics_block *tm,
 55                           enum tcp_metric_index idx)
 56 {
 57         return tm->tcpm_vals[idx];
 58 }
 59 
 60 static u32 tcp_metric_get_jiffies(struct tcp_metrics_block *tm,
 61                                   enum tcp_metric_index idx)
 62 {
 63         return msecs_to_jiffies(tm->tcpm_vals[idx]);
 64 }
 65 
 66 static void tcp_metric_set(struct tcp_metrics_block *tm,
 67                            enum tcp_metric_index idx,
 68                            u32 val)
 69 {
 70         tm->tcpm_vals[idx] = val;
 71 }
 72 
 73 static void tcp_metric_set_msecs(struct tcp_metrics_block *tm,
 74                                  enum tcp_metric_index idx,
 75                                  u32 val)
 76 {
 77         tm->tcpm_vals[idx] = jiffies_to_msecs(val);
 78 }
 79 
 80 static bool addr_same(const struct inetpeer_addr *a,
 81                       const struct inetpeer_addr *b)
 82 {
 83         const struct in6_addr *a6, *b6;
 84 
 85         if (a->family != b->family)
 86                 return false;
 87         if (a->family == AF_INET)
 88                 return a->addr.a4 == b->addr.a4;
 89 
 90         a6 = (const struct in6_addr *) &a->addr.a6[0];
 91         b6 = (const struct in6_addr *) &b->addr.a6[0];
 92 
 93         return ipv6_addr_equal(a6, b6);
 94 }
 95 
 96 struct tcpm_hash_bucket {
 97         struct tcp_metrics_block __rcu  *chain;
 98 };
 99 
100 static DEFINE_SPINLOCK(tcp_metrics_lock);
101 
102 static void tcpm_suck_dst(struct tcp_metrics_block *tm, struct dst_entry *dst,
103                           bool fastopen_clear)
104 {
105         u32 val;
106 
107         tm->tcpm_stamp = jiffies;
108 
109         val = 0;
110         if (dst_metric_locked(dst, RTAX_RTT))
111                 val |= 1 << TCP_METRIC_RTT;
112         if (dst_metric_locked(dst, RTAX_RTTVAR))
113                 val |= 1 << TCP_METRIC_RTTVAR;
114         if (dst_metric_locked(dst, RTAX_SSTHRESH))
115                 val |= 1 << TCP_METRIC_SSTHRESH;
116         if (dst_metric_locked(dst, RTAX_CWND))
117                 val |= 1 << TCP_METRIC_CWND;
118         if (dst_metric_locked(dst, RTAX_REORDERING))
119                 val |= 1 << TCP_METRIC_REORDERING;
120         tm->tcpm_lock = val;
121 
122         tm->tcpm_vals[TCP_METRIC_RTT] = dst_metric_raw(dst, RTAX_RTT);
123         tm->tcpm_vals[TCP_METRIC_RTTVAR] = dst_metric_raw(dst, RTAX_RTTVAR);
124         tm->tcpm_vals[TCP_METRIC_SSTHRESH] = dst_metric_raw(dst, RTAX_SSTHRESH);
125         tm->tcpm_vals[TCP_METRIC_CWND] = dst_metric_raw(dst, RTAX_CWND);
126         tm->tcpm_vals[TCP_METRIC_REORDERING] = dst_metric_raw(dst, RTAX_REORDERING);
127         tm->tcpm_ts = 0;
128         tm->tcpm_ts_stamp = 0;
129         if (fastopen_clear) {
130                 tm->tcpm_fastopen.mss = 0;
131                 tm->tcpm_fastopen.syn_loss = 0;
132                 tm->tcpm_fastopen.cookie.len = 0;
133         }
134 }
135 
136 #define TCP_METRICS_TIMEOUT             (60 * 60 * HZ)
137 
138 static void tcpm_check_stamp(struct tcp_metrics_block *tm, struct dst_entry *dst)
139 {
140         if (tm && unlikely(time_after(jiffies, tm->tcpm_stamp + TCP_METRICS_TIMEOUT)))
141                 tcpm_suck_dst(tm, dst, false);
142 }
143 
144 #define TCP_METRICS_RECLAIM_DEPTH       5
145 #define TCP_METRICS_RECLAIM_PTR         (struct tcp_metrics_block *) 0x1UL
146 
147 static struct tcp_metrics_block *tcpm_new(struct dst_entry *dst,
148                                           struct inetpeer_addr *addr,
149                                           unsigned int hash)
150 {
151         struct tcp_metrics_block *tm;
152         struct net *net;
153         bool reclaim = false;
154 
155         spin_lock_bh(&tcp_metrics_lock);
156         net = dev_net(dst->dev);
157 
158         /* While waiting for the spin-lock the cache might have been populated
159          * with this entry and so we have to check again.
160          */
161         tm = __tcp_get_metrics(addr, net, hash);
162         if (tm == TCP_METRICS_RECLAIM_PTR) {
163                 reclaim = true;
164                 tm = NULL;
165         }
166         if (tm) {
167                 tcpm_check_stamp(tm, dst);
168                 goto out_unlock;
169         }
170 
171         if (unlikely(reclaim)) {
172                 struct tcp_metrics_block *oldest;
173 
174                 oldest = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain);
175                 for (tm = rcu_dereference(oldest->tcpm_next); tm;
176                      tm = rcu_dereference(tm->tcpm_next)) {
177                         if (time_before(tm->tcpm_stamp, oldest->tcpm_stamp))
178                                 oldest = tm;
179                 }
180                 tm = oldest;
181         } else {
182                 tm = kmalloc(sizeof(*tm), GFP_ATOMIC);
183                 if (!tm)
184                         goto out_unlock;
185         }
186         tm->tcpm_addr = *addr;
187 
188         tcpm_suck_dst(tm, dst, true);
189 
190         if (likely(!reclaim)) {
191                 tm->tcpm_next = net->ipv4.tcp_metrics_hash[hash].chain;
192                 rcu_assign_pointer(net->ipv4.tcp_metrics_hash[hash].chain, tm);
193         }
194 
195 out_unlock:
196         spin_unlock_bh(&tcp_metrics_lock);
197         return tm;
198 }
199 
200 static struct tcp_metrics_block *tcp_get_encode(struct tcp_metrics_block *tm, int depth)
201 {
202         if (tm)
203                 return tm;
204         if (depth > TCP_METRICS_RECLAIM_DEPTH)
205                 return TCP_METRICS_RECLAIM_PTR;
206         return NULL;
207 }
208 
209 static struct tcp_metrics_block *__tcp_get_metrics(const struct inetpeer_addr *addr,
210                                                    struct net *net, unsigned int hash)
211 {
212         struct tcp_metrics_block *tm;
213         int depth = 0;
214 
215         for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm;
216              tm = rcu_dereference(tm->tcpm_next)) {
217                 if (addr_same(&tm->tcpm_addr, addr))
218                         break;
219                 depth++;
220         }
221         return tcp_get_encode(tm, depth);
222 }
223 
224 static struct tcp_metrics_block *__tcp_get_metrics_req(struct request_sock *req,
225                                                        struct dst_entry *dst)
226 {
227         struct tcp_metrics_block *tm;
228         struct inetpeer_addr addr;
229         unsigned int hash;
230         struct net *net;
231 
232         addr.family = req->rsk_ops->family;
233         switch (addr.family) {
234         case AF_INET:
235                 addr.addr.a4 = inet_rsk(req)->rmt_addr;
236                 hash = (__force unsigned int) addr.addr.a4;
237                 break;
238         case AF_INET6:
239                 *(struct in6_addr *)addr.addr.a6 = inet6_rsk(req)->rmt_addr;
240                 hash = ipv6_addr_hash(&inet6_rsk(req)->rmt_addr);
241                 break;
242         default:
243                 return NULL;
244         }
245 
246         net = dev_net(dst->dev);
247         hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log);
248 
249         for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm;
250              tm = rcu_dereference(tm->tcpm_next)) {
251                 if (addr_same(&tm->tcpm_addr, &addr))
252                         break;
253         }
254         tcpm_check_stamp(tm, dst);
255         return tm;
256 }
257 
258 static struct tcp_metrics_block *__tcp_get_metrics_tw(struct inet_timewait_sock *tw)
259 {
260         struct inet6_timewait_sock *tw6;
261         struct tcp_metrics_block *tm;
262         struct inetpeer_addr addr;
263         unsigned int hash;
264         struct net *net;
265 
266         addr.family = tw->tw_family;
267         switch (addr.family) {
268         case AF_INET:
269                 addr.addr.a4 = tw->tw_daddr;
270                 hash = (__force unsigned int) addr.addr.a4;
271                 break;
272         case AF_INET6:
273                 tw6 = inet6_twsk((struct sock *)tw);
274                 *(struct in6_addr *)addr.addr.a6 = tw6->tw_v6_daddr;
275                 hash = ipv6_addr_hash(&tw6->tw_v6_daddr);
276                 break;
277         default:
278                 return NULL;
279         }
280 
281         net = twsk_net(tw);
282         hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log);
283 
284         for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm;
285              tm = rcu_dereference(tm->tcpm_next)) {
286                 if (addr_same(&tm->tcpm_addr, &addr))
287                         break;
288         }
289         return tm;
290 }
291 
292 static struct tcp_metrics_block *tcp_get_metrics(struct sock *sk,
293                                                  struct dst_entry *dst,
294                                                  bool create)
295 {
296         struct tcp_metrics_block *tm;
297         struct inetpeer_addr addr;
298         unsigned int hash;
299         struct net *net;
300 
301         addr.family = sk->sk_family;
302         switch (addr.family) {
303         case AF_INET:
304                 addr.addr.a4 = inet_sk(sk)->inet_daddr;
305                 hash = (__force unsigned int) addr.addr.a4;
306                 break;
307         case AF_INET6:
308                 *(struct in6_addr *)addr.addr.a6 = inet6_sk(sk)->daddr;
309                 hash = ipv6_addr_hash(&inet6_sk(sk)->daddr);
310                 break;
311         default:
312                 return NULL;
313         }
314 
315         net = dev_net(dst->dev);
316         hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log);
317 
318         tm = __tcp_get_metrics(&addr, net, hash);
319         if (tm == TCP_METRICS_RECLAIM_PTR)
320                 tm = NULL;
321         if (!tm && create)
322                 tm = tcpm_new(dst, &addr, hash);
323         else
324                 tcpm_check_stamp(tm, dst);
325 
326         return tm;
327 }
328 
329 /* Save metrics learned by this TCP session.  This function is called
330  * only, when TCP finishes successfully i.e. when it enters TIME-WAIT
331  * or goes from LAST-ACK to CLOSE.
332  */
333 void tcp_update_metrics(struct sock *sk)
334 {
335         const struct inet_connection_sock *icsk = inet_csk(sk);
336         struct dst_entry *dst = __sk_dst_get(sk);
337         struct tcp_sock *tp = tcp_sk(sk);
338         struct tcp_metrics_block *tm;
339         unsigned long rtt;
340         u32 val;
341         int m;
342 
343         if (sysctl_tcp_nometrics_save || !dst)
344                 return;
345 
346         if (dst->flags & DST_HOST)
347                 dst_confirm(dst);
348 
349         rcu_read_lock();
350         if (icsk->icsk_backoff || !tp->srtt) {
351                 /* This session failed to estimate rtt. Why?
352                  * Probably, no packets returned in time.  Reset our
353                  * results.
354                  */
355                 tm = tcp_get_metrics(sk, dst, false);
356                 if (tm && !tcp_metric_locked(tm, TCP_METRIC_RTT))
357                         tcp_metric_set(tm, TCP_METRIC_RTT, 0);
358                 goto out_unlock;
359         } else
360                 tm = tcp_get_metrics(sk, dst, true);
361 
362         if (!tm)
363                 goto out_unlock;
364 
365         rtt = tcp_metric_get_jiffies(tm, TCP_METRIC_RTT);
366         m = rtt - tp->srtt;
367 
368         /* If newly calculated rtt larger than stored one, store new
369          * one. Otherwise, use EWMA. Remember, rtt overestimation is
370          * always better than underestimation.
371          */
372         if (!tcp_metric_locked(tm, TCP_METRIC_RTT)) {
373                 if (m <= 0)
374                         rtt = tp->srtt;
375                 else
376                         rtt -= (m >> 3);
377                 tcp_metric_set_msecs(tm, TCP_METRIC_RTT, rtt);
378         }
379 
380         if (!tcp_metric_locked(tm, TCP_METRIC_RTTVAR)) {
381                 unsigned long var;
382 
383                 if (m < 0)
384                         m = -m;
385 
386                 /* Scale deviation to rttvar fixed point */
387                 m >>= 1;
388                 if (m < tp->mdev)
389                         m = tp->mdev;
390 
391                 var = tcp_metric_get_jiffies(tm, TCP_METRIC_RTTVAR);
392                 if (m >= var)
393                         var = m;
394                 else
395                         var -= (var - m) >> 2;
396 
397                 tcp_metric_set_msecs(tm, TCP_METRIC_RTTVAR, var);
398         }
399 
400         if (tcp_in_initial_slowstart(tp)) {
401                 /* Slow start still did not finish. */
402                 if (!tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) {
403                         val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH);
404                         if (val && (tp->snd_cwnd >> 1) > val)
405                                 tcp_metric_set(tm, TCP_METRIC_SSTHRESH,
406                                                tp->snd_cwnd >> 1);
407                 }
408                 if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) {
409                         val = tcp_metric_get(tm, TCP_METRIC_CWND);
410                         if (tp->snd_cwnd > val)
411                                 tcp_metric_set(tm, TCP_METRIC_CWND,
412                                                tp->snd_cwnd);
413                 }
414         } else if (tp->snd_cwnd > tp->snd_ssthresh &&
415                    icsk->icsk_ca_state == TCP_CA_Open) {
416                 /* Cong. avoidance phase, cwnd is reliable. */
417                 if (!tcp_metric_locked(tm, TCP_METRIC_SSTHRESH))
418                         tcp_metric_set(tm, TCP_METRIC_SSTHRESH,
419                                        max(tp->snd_cwnd >> 1, tp->snd_ssthresh));
420                 if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) {
421                         val = tcp_metric_get(tm, TCP_METRIC_CWND);
422                         tcp_metric_set(tm, TCP_METRIC_CWND, (val + tp->snd_cwnd) >> 1);
423                 }
424         } else {
425                 /* Else slow start did not finish, cwnd is non-sense,
426                  * ssthresh may be also invalid.
427                  */
428                 if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) {
429                         val = tcp_metric_get(tm, TCP_METRIC_CWND);
430                         tcp_metric_set(tm, TCP_METRIC_CWND,
431                                        (val + tp->snd_ssthresh) >> 1);
432                 }
433                 if (!tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) {
434                         val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH);
435                         if (val && tp->snd_ssthresh > val)
436                                 tcp_metric_set(tm, TCP_METRIC_SSTHRESH,
437                                                tp->snd_ssthresh);
438                 }
439                 if (!tcp_metric_locked(tm, TCP_METRIC_REORDERING)) {
440                         val = tcp_metric_get(tm, TCP_METRIC_REORDERING);
441                         if (val < tp->reordering &&
442                             tp->reordering != sysctl_tcp_reordering)
443                                 tcp_metric_set(tm, TCP_METRIC_REORDERING,
444                                                tp->reordering);
445                 }
446         }
447         tm->tcpm_stamp = jiffies;
448 out_unlock:
449         rcu_read_unlock();
450 }
451 
452 /* Initialize metrics on socket. */
453 
454 void tcp_init_metrics(struct sock *sk)
455 {
456         struct dst_entry *dst = __sk_dst_get(sk);
457         struct tcp_sock *tp = tcp_sk(sk);
458         struct tcp_metrics_block *tm;
459         u32 val;
460 
461         if (dst == NULL)
462                 goto reset;
463 
464         dst_confirm(dst);
465 
466         rcu_read_lock();
467         tm = tcp_get_metrics(sk, dst, true);
468         if (!tm) {
469                 rcu_read_unlock();
470                 goto reset;
471         }
472 
473         if (tcp_metric_locked(tm, TCP_METRIC_CWND))
474                 tp->snd_cwnd_clamp = tcp_metric_get(tm, TCP_METRIC_CWND);
475 
476         val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH);
477         if (val) {
478                 tp->snd_ssthresh = val;
479                 if (tp->snd_ssthresh > tp->snd_cwnd_clamp)
480                         tp->snd_ssthresh = tp->snd_cwnd_clamp;
481         } else {
482                 /* ssthresh may have been reduced unnecessarily during.
483                  * 3WHS. Restore it back to its initial default.
484                  */
485                 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
486         }
487         val = tcp_metric_get(tm, TCP_METRIC_REORDERING);
488         if (val && tp->reordering != val) {
489                 tcp_disable_fack(tp);
490                 tcp_disable_early_retrans(tp);
491                 tp->reordering = val;
492         }
493 
494         val = tcp_metric_get(tm, TCP_METRIC_RTT);
495         if (val == 0 || tp->srtt == 0) {
496                 rcu_read_unlock();
497                 goto reset;
498         }
499         /* Initial rtt is determined from SYN,SYN-ACK.
500          * The segment is small and rtt may appear much
501          * less than real one. Use per-dst memory
502          * to make it more realistic.
503          *
504          * A bit of theory. RTT is time passed after "normal" sized packet
505          * is sent until it is ACKed. In normal circumstances sending small
506          * packets force peer to delay ACKs and calculation is correct too.
507          * The algorithm is adaptive and, provided we follow specs, it
508          * NEVER underestimate RTT. BUT! If peer tries to make some clever
509          * tricks sort of "quick acks" for time long enough to decrease RTT
510          * to low value, and then abruptly stops to do it and starts to delay
511          * ACKs, wait for troubles.
512          */
513         val = msecs_to_jiffies(val);
514         if (val > tp->srtt) {
515                 tp->srtt = val;
516                 tp->rtt_seq = tp->snd_nxt;
517         }
518         val = tcp_metric_get_jiffies(tm, TCP_METRIC_RTTVAR);
519         if (val > tp->mdev) {
520                 tp->mdev = val;
521                 tp->mdev_max = tp->rttvar = max(tp->mdev, tcp_rto_min(sk));
522         }
523         rcu_read_unlock();
524 
525         tcp_set_rto(sk);
526 reset:
527         if (tp->srtt == 0) {
528                 /* RFC6298: 5.7 We've failed to get a valid RTT sample from
529                  * 3WHS. This is most likely due to retransmission,
530                  * including spurious one. Reset the RTO back to 3secs
531                  * from the more aggressive 1sec to avoid more spurious
532                  * retransmission.
533                  */
534                 tp->mdev = tp->mdev_max = tp->rttvar = TCP_TIMEOUT_FALLBACK;
535                 inet_csk(sk)->icsk_rto = TCP_TIMEOUT_FALLBACK;
536         }
537         /* Cut cwnd down to 1 per RFC5681 if SYN or SYN-ACK has been
538          * retransmitted. In light of RFC6298 more aggressive 1sec
539          * initRTO, we only reset cwnd when more than 1 SYN/SYN-ACK
540          * retransmission has occurred.
541          */
542         if (tp->total_retrans > 1)
543                 tp->snd_cwnd = 1;
544         else
545                 tp->snd_cwnd = tcp_init_cwnd(tp, dst);
546         tp->snd_cwnd_stamp = tcp_time_stamp;
547 }
548 
549 bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst, bool paws_check)
550 {
551         struct tcp_metrics_block *tm;
552         bool ret;
553 
554         if (!dst)
555                 return false;
556 
557         rcu_read_lock();
558         tm = __tcp_get_metrics_req(req, dst);
559         if (paws_check) {
560                 if (tm &&
561                     (u32)get_seconds() - tm->tcpm_ts_stamp < TCP_PAWS_MSL &&
562                     (s32)(tm->tcpm_ts - req->ts_recent) > TCP_PAWS_WINDOW)
563                         ret = false;
564                 else
565                         ret = true;
566         } else {
567                 if (tm && tcp_metric_get(tm, TCP_METRIC_RTT) && tm->tcpm_ts_stamp)
568                         ret = true;
569                 else
570                         ret = false;
571         }
572         rcu_read_unlock();
573 
574         return ret;
575 }
576 EXPORT_SYMBOL_GPL(tcp_peer_is_proven);
577 
578 void tcp_fetch_timewait_stamp(struct sock *sk, struct dst_entry *dst)
579 {
580         struct tcp_metrics_block *tm;
581 
582         rcu_read_lock();
583         tm = tcp_get_metrics(sk, dst, true);
584         if (tm) {
585                 struct tcp_sock *tp = tcp_sk(sk);
586 
587                 if ((u32)get_seconds() - tm->tcpm_ts_stamp <= TCP_PAWS_MSL) {
588                         tp->rx_opt.ts_recent_stamp = tm->tcpm_ts_stamp;
589                         tp->rx_opt.ts_recent = tm->tcpm_ts;
590                 }
591         }
592         rcu_read_unlock();
593 }
594 EXPORT_SYMBOL_GPL(tcp_fetch_timewait_stamp);
595 
596 /* VJ's idea. Save last timestamp seen from this destination and hold
597  * it at least for normal timewait interval to use for duplicate
598  * segment detection in subsequent connections, before they enter
599  * synchronized state.
600  */
601 bool tcp_remember_stamp(struct sock *sk)
602 {
603         struct dst_entry *dst = __sk_dst_get(sk);
604         bool ret = false;
605 
606         if (dst) {
607                 struct tcp_metrics_block *tm;
608 
609                 rcu_read_lock();
610                 tm = tcp_get_metrics(sk, dst, true);
611                 if (tm) {
612                         struct tcp_sock *tp = tcp_sk(sk);
613 
614                         if ((s32)(tm->tcpm_ts - tp->rx_opt.ts_recent) <= 0 ||
615                             ((u32)get_seconds() - tm->tcpm_ts_stamp > TCP_PAWS_MSL &&
616                              tm->tcpm_ts_stamp <= (u32)tp->rx_opt.ts_recent_stamp)) {
617                                 tm->tcpm_ts_stamp = (u32)tp->rx_opt.ts_recent_stamp;
618                                 tm->tcpm_ts = tp->rx_opt.ts_recent;
619                         }
620                         ret = true;
621                 }
622                 rcu_read_unlock();
623         }
624         return ret;
625 }
626 
627 bool tcp_tw_remember_stamp(struct inet_timewait_sock *tw)
628 {
629         struct tcp_metrics_block *tm;
630         bool ret = false;
631 
632         rcu_read_lock();
633         tm = __tcp_get_metrics_tw(tw);
634         if (tm) {
635                 const struct tcp_timewait_sock *tcptw;
636                 struct sock *sk = (struct sock *) tw;
637 
638                 tcptw = tcp_twsk(sk);
639                 if ((s32)(tm->tcpm_ts - tcptw->tw_ts_recent) <= 0 ||
640                     ((u32)get_seconds() - tm->tcpm_ts_stamp > TCP_PAWS_MSL &&
641                      tm->tcpm_ts_stamp <= (u32)tcptw->tw_ts_recent_stamp)) {
642                         tm->tcpm_ts_stamp = (u32)tcptw->tw_ts_recent_stamp;
643                         tm->tcpm_ts        = tcptw->tw_ts_recent;
644                 }
645                 ret = true;
646         }
647         rcu_read_unlock();
648 
649         return ret;
650 }
651 
652 static DEFINE_SEQLOCK(fastopen_seqlock);
653 
654 void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
655                             struct tcp_fastopen_cookie *cookie,
656                             int *syn_loss, unsigned long *last_syn_loss)
657 {
658         struct tcp_metrics_block *tm;
659 
660         rcu_read_lock();
661         tm = tcp_get_metrics(sk, __sk_dst_get(sk), false);
662         if (tm) {
663                 struct tcp_fastopen_metrics *tfom = &tm->tcpm_fastopen;
664                 unsigned int seq;
665 
666                 do {
667                         seq = read_seqbegin(&fastopen_seqlock);
668                         if (tfom->mss)
669                                 *mss = tfom->mss;
670                         *cookie = tfom->cookie;
671                         *syn_loss = tfom->syn_loss;
672                         *last_syn_loss = *syn_loss ? tfom->last_syn_loss : 0;
673                 } while (read_seqretry(&fastopen_seqlock, seq));
674         }
675         rcu_read_unlock();
676 }
677 
678 void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
679                             struct tcp_fastopen_cookie *cookie, bool syn_lost)
680 {
681         struct dst_entry *dst = __sk_dst_get(sk);
682         struct tcp_metrics_block *tm;
683 
684         if (!dst)
685                 return;
686         rcu_read_lock();
687         tm = tcp_get_metrics(sk, dst, true);
688         if (tm) {
689                 struct tcp_fastopen_metrics *tfom = &tm->tcpm_fastopen;
690 
691                 write_seqlock_bh(&fastopen_seqlock);
692                 tfom->mss = mss;
693                 if (cookie->len > 0)
694                         tfom->cookie = *cookie;
695                 if (syn_lost) {
696                         ++tfom->syn_loss;
697                         tfom->last_syn_loss = jiffies;
698                 } else
699                         tfom->syn_loss = 0;
700                 write_sequnlock_bh(&fastopen_seqlock);
701         }
702         rcu_read_unlock();
703 }
704 
705 static struct genl_family tcp_metrics_nl_family = {
706         .id             = GENL_ID_GENERATE,
707         .hdrsize        = 0,
708         .name           = TCP_METRICS_GENL_NAME,
709         .version        = TCP_METRICS_GENL_VERSION,
710         .maxattr        = TCP_METRICS_ATTR_MAX,
711         .netnsok        = true,
712 };
713 
714 static struct nla_policy tcp_metrics_nl_policy[TCP_METRICS_ATTR_MAX + 1] = {
715         [TCP_METRICS_ATTR_ADDR_IPV4]    = { .type = NLA_U32, },
716         [TCP_METRICS_ATTR_ADDR_IPV6]    = { .type = NLA_BINARY,
717                                             .len = sizeof(struct in6_addr), },
718         /* Following attributes are not received for GET/DEL,
719          * we keep them for reference
720          */
721 #if 0
722         [TCP_METRICS_ATTR_AGE]          = { .type = NLA_MSECS, },
723         [TCP_METRICS_ATTR_TW_TSVAL]     = { .type = NLA_U32, },
724         [TCP_METRICS_ATTR_TW_TS_STAMP]  = { .type = NLA_S32, },
725         [TCP_METRICS_ATTR_VALS]         = { .type = NLA_NESTED, },
726         [TCP_METRICS_ATTR_FOPEN_MSS]    = { .type = NLA_U16, },
727         [TCP_METRICS_ATTR_FOPEN_SYN_DROPS]      = { .type = NLA_U16, },
728         [TCP_METRICS_ATTR_FOPEN_SYN_DROP_TS]    = { .type = NLA_MSECS, },
729         [TCP_METRICS_ATTR_FOPEN_COOKIE] = { .type = NLA_BINARY,
730                                             .len = TCP_FASTOPEN_COOKIE_MAX, },
731 #endif
732 };
733 
734 /* Add attributes, caller cancels its header on failure */
735 static int tcp_metrics_fill_info(struct sk_buff *msg,
736                                  struct tcp_metrics_block *tm)
737 {
738         struct nlattr *nest;
739         int i;
740 
741         switch (tm->tcpm_addr.family) {
742         case AF_INET:
743                 if (nla_put_be32(msg, TCP_METRICS_ATTR_ADDR_IPV4,
744                                 tm->tcpm_addr.addr.a4) < 0)
745                         goto nla_put_failure;
746                 break;
747         case AF_INET6:
748                 if (nla_put(msg, TCP_METRICS_ATTR_ADDR_IPV6, 16,
749                             tm->tcpm_addr.addr.a6) < 0)
750                         goto nla_put_failure;
751                 break;
752         default:
753                 return -EAFNOSUPPORT;
754         }
755 
756         if (nla_put_msecs(msg, TCP_METRICS_ATTR_AGE,
757                           jiffies - tm->tcpm_stamp) < 0)
758                 goto nla_put_failure;
759         if (tm->tcpm_ts_stamp) {
760                 if (nla_put_s32(msg, TCP_METRICS_ATTR_TW_TS_STAMP,
761                                 (s32) (get_seconds() - tm->tcpm_ts_stamp)) < 0)
762                         goto nla_put_failure;
763                 if (nla_put_u32(msg, TCP_METRICS_ATTR_TW_TSVAL,
764                                 tm->tcpm_ts) < 0)
765                         goto nla_put_failure;
766         }
767 
768         {
769                 int n = 0;
770 
771                 nest = nla_nest_start(msg, TCP_METRICS_ATTR_VALS);
772                 if (!nest)
773                         goto nla_put_failure;
774                 for (i = 0; i < TCP_METRIC_MAX + 1; i++) {
775                         if (!tm->tcpm_vals[i])
776                                 continue;
777                         if (nla_put_u32(msg, i + 1, tm->tcpm_vals[i]) < 0)
778                                 goto nla_put_failure;
779                         n++;
780                 }
781                 if (n)
782                         nla_nest_end(msg, nest);
783                 else
784                         nla_nest_cancel(msg, nest);
785         }
786 
787         {
788                 struct tcp_fastopen_metrics tfom_copy[1], *tfom;
789                 unsigned int seq;
790 
791                 do {
792                         seq = read_seqbegin(&fastopen_seqlock);
793                         tfom_copy[0] = tm->tcpm_fastopen;
794                 } while (read_seqretry(&fastopen_seqlock, seq));
795 
796                 tfom = tfom_copy;
797                 if (tfom->mss &&
798                     nla_put_u16(msg, TCP_METRICS_ATTR_FOPEN_MSS,
799                                 tfom->mss) < 0)
800                         goto nla_put_failure;
801                 if (tfom->syn_loss &&
802                     (nla_put_u16(msg, TCP_METRICS_ATTR_FOPEN_SYN_DROPS,
803                                 tfom->syn_loss) < 0 ||
804                      nla_put_msecs(msg, TCP_METRICS_ATTR_FOPEN_SYN_DROP_TS,
805                                 jiffies - tfom->last_syn_loss) < 0))
806                         goto nla_put_failure;
807                 if (tfom->cookie.len > 0 &&
808                     nla_put(msg, TCP_METRICS_ATTR_FOPEN_COOKIE,
809                             tfom->cookie.len, tfom->cookie.val) < 0)
810                         goto nla_put_failure;
811         }
812 
813         return 0;
814 
815 nla_put_failure:
816         return -EMSGSIZE;
817 }
818 
819 static int tcp_metrics_dump_info(struct sk_buff *skb,
820                                  struct netlink_callback *cb,
821                                  struct tcp_metrics_block *tm)
822 {
823         void *hdr;
824 
825         hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
826                           &tcp_metrics_nl_family, NLM_F_MULTI,
827                           TCP_METRICS_CMD_GET);
828         if (!hdr)
829                 return -EMSGSIZE;
830 
831         if (tcp_metrics_fill_info(skb, tm) < 0)
832                 goto nla_put_failure;
833 
834         return genlmsg_end(skb, hdr);
835 
836 nla_put_failure:
837         genlmsg_cancel(skb, hdr);
838         return -EMSGSIZE;
839 }
840 
841 static int tcp_metrics_nl_dump(struct sk_buff *skb,
842                                struct netlink_callback *cb)
843 {
844         struct net *net = sock_net(skb->sk);
845         unsigned int max_rows = 1U << net->ipv4.tcp_metrics_hash_log;
846         unsigned int row, s_row = cb->args[0];
847         int s_col = cb->args[1], col = s_col;
848 
849         for (row = s_row; row < max_rows; row++, s_col = 0) {
850                 struct tcp_metrics_block *tm;
851                 struct tcpm_hash_bucket *hb = net->ipv4.tcp_metrics_hash + row;
852 
853                 rcu_read_lock();
854                 for (col = 0, tm = rcu_dereference(hb->chain); tm;
855                      tm = rcu_dereference(tm->tcpm_next), col++) {
856                         if (col < s_col)
857                                 continue;
858                         if (tcp_metrics_dump_info(skb, cb, tm) < 0) {
859                                 rcu_read_unlock();
860                                 goto done;
861                         }
862                 }
863                 rcu_read_unlock();
864         }
865 
866 done:
867         cb->args[0] = row;
868         cb->args[1] = col;
869         return skb->len;
870 }
871 
872 static int parse_nl_addr(struct genl_info *info, struct inetpeer_addr *addr,
873                          unsigned int *hash, int optional)
874 {
875         struct nlattr *a;
876 
877         a = info->attrs[TCP_METRICS_ATTR_ADDR_IPV4];
878         if (a) {
879                 addr->family = AF_INET;
880                 addr->addr.a4 = nla_get_be32(a);
881                 *hash = (__force unsigned int) addr->addr.a4;
882                 return 0;
883         }
884         a = info->attrs[TCP_METRICS_ATTR_ADDR_IPV6];
885         if (a) {
886                 if (nla_len(a) != sizeof(struct in6_addr))
887                         return -EINVAL;
888                 addr->family = AF_INET6;
889                 memcpy(addr->addr.a6, nla_data(a), sizeof(addr->addr.a6));
890                 *hash = ipv6_addr_hash((struct in6_addr *) addr->addr.a6);
891                 return 0;
892         }
893         return optional ? 1 : -EAFNOSUPPORT;
894 }
895 
896 static int tcp_metrics_nl_cmd_get(struct sk_buff *skb, struct genl_info *info)
897 {
898         struct tcp_metrics_block *tm;
899         struct inetpeer_addr addr;
900         unsigned int hash;
901         struct sk_buff *msg;
902         struct net *net = genl_info_net(info);
903         void *reply;
904         int ret;
905 
906         ret = parse_nl_addr(info, &addr, &hash, 0);
907         if (ret < 0)
908                 return ret;
909 
910         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
911         if (!msg)
912                 return -ENOMEM;
913 
914         reply = genlmsg_put_reply(msg, info, &tcp_metrics_nl_family, 0,
915                                   info->genlhdr->cmd);
916         if (!reply)
917                 goto nla_put_failure;
918 
919         hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log);
920         ret = -ESRCH;
921         rcu_read_lock();
922         for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm;
923              tm = rcu_dereference(tm->tcpm_next)) {
924                 if (addr_same(&tm->tcpm_addr, &addr)) {
925                         ret = tcp_metrics_fill_info(msg, tm);
926                         break;
927                 }
928         }
929         rcu_read_unlock();
930         if (ret < 0)
931                 goto out_free;
932 
933         genlmsg_end(msg, reply);
934         return genlmsg_reply(msg, info);
935 
936 nla_put_failure:
937         ret = -EMSGSIZE;
938 
939 out_free:
940         nlmsg_free(msg);
941         return ret;
942 }
943 
944 #define deref_locked_genl(p)    \
945         rcu_dereference_protected(p, lockdep_genl_is_held() && \
946                                      lockdep_is_held(&tcp_metrics_lock))
947 
948 #define deref_genl(p)   rcu_dereference_protected(p, lockdep_genl_is_held())
949 
950 static int tcp_metrics_flush_all(struct net *net)
951 {
952         unsigned int max_rows = 1U << net->ipv4.tcp_metrics_hash_log;
953         struct tcpm_hash_bucket *hb = net->ipv4.tcp_metrics_hash;
954         struct tcp_metrics_block *tm;
955         unsigned int row;
956 
957         for (row = 0; row < max_rows; row++, hb++) {
958                 spin_lock_bh(&tcp_metrics_lock);
959                 tm = deref_locked_genl(hb->chain);
960                 if (tm)
961                         hb->chain = NULL;
962                 spin_unlock_bh(&tcp_metrics_lock);
963                 while (tm) {
964                         struct tcp_metrics_block *next;
965 
966                         next = deref_genl(tm->tcpm_next);
967                         kfree_rcu(tm, rcu_head);
968                         tm = next;
969                 }
970         }
971         return 0;
972 }
973 
974 static int tcp_metrics_nl_cmd_del(struct sk_buff *skb, struct genl_info *info)
975 {
976         struct tcpm_hash_bucket *hb;
977         struct tcp_metrics_block *tm;
978         struct tcp_metrics_block __rcu **pp;
979         struct inetpeer_addr addr;
980         unsigned int hash;
981         struct net *net = genl_info_net(info);
982         int ret;
983 
984         ret = parse_nl_addr(info, &addr, &hash, 1);
985         if (ret < 0)
986                 return ret;
987         if (ret > 0)
988                 return tcp_metrics_flush_all(net);
989 
990         hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log);
991         hb = net->ipv4.tcp_metrics_hash + hash;
992         pp = &hb->chain;
993         spin_lock_bh(&tcp_metrics_lock);
994         for (tm = deref_locked_genl(*pp); tm;
995              pp = &tm->tcpm_next, tm = deref_locked_genl(*pp)) {
996                 if (addr_same(&tm->tcpm_addr, &addr)) {
997                         *pp = tm->tcpm_next;
998                         break;
999                 }
1000         }
1001         spin_unlock_bh(&tcp_metrics_lock);
1002         if (!tm)
1003                 return -ESRCH;
1004         kfree_rcu(tm, rcu_head);
1005         return 0;
1006 }
1007 
1008 static struct genl_ops tcp_metrics_nl_ops[] = {
1009         {
1010                 .cmd = TCP_METRICS_CMD_GET,
1011                 .doit = tcp_metrics_nl_cmd_get,
1012                 .dumpit = tcp_metrics_nl_dump,
1013                 .policy = tcp_metrics_nl_policy,
1014                 .flags = GENL_ADMIN_PERM,
1015         },
1016         {
1017                 .cmd = TCP_METRICS_CMD_DEL,
1018                 .doit = tcp_metrics_nl_cmd_del,
1019                 .policy = tcp_metrics_nl_policy,
1020                 .flags = GENL_ADMIN_PERM,
1021         },
1022 };
1023 
1024 static unsigned int tcpmhash_entries;
1025 static int __init set_tcpmhash_entries(char *str)
1026 {
1027         ssize_t ret;
1028 
1029         if (!str)
1030                 return 0;
1031 
1032         ret = kstrtouint(str, 0, &tcpmhash_entries);
1033         if (ret)
1034                 return 0;
1035 
1036         return 1;
1037 }
1038 __setup("tcpmhash_entries=", set_tcpmhash_entries);
1039 
1040 static int __net_init tcp_net_metrics_init(struct net *net)
1041 {
1042         size_t size;
1043         unsigned int slots;
1044 
1045         slots = tcpmhash_entries;
1046         if (!slots) {
1047                 if (totalram_pages >= 128 * 1024)
1048                         slots = 16 * 1024;
1049                 else
1050                         slots = 8 * 1024;
1051         }
1052 
1053         net->ipv4.tcp_metrics_hash_log = order_base_2(slots);
1054         size = sizeof(struct tcpm_hash_bucket) << net->ipv4.tcp_metrics_hash_log;
1055 
1056         net->ipv4.tcp_metrics_hash = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
1057         if (!net->ipv4.tcp_metrics_hash)
1058                 net->ipv4.tcp_metrics_hash = vzalloc(size);
1059 
1060         if (!net->ipv4.tcp_metrics_hash)
1061                 return -ENOMEM;
1062 
1063         return 0;
1064 }
1065 
1066 static void __net_exit tcp_net_metrics_exit(struct net *net)
1067 {
1068         unsigned int i;
1069 
1070         for (i = 0; i < (1U << net->ipv4.tcp_metrics_hash_log) ; i++) {
1071                 struct tcp_metrics_block *tm, *next;
1072 
1073                 tm = rcu_dereference_protected(net->ipv4.tcp_metrics_hash[i].chain, 1);
1074                 while (tm) {
1075                         next = rcu_dereference_protected(tm->tcpm_next, 1);
1076                         kfree(tm);
1077                         tm = next;
1078                 }
1079         }
1080         if (is_vmalloc_addr(net->ipv4.tcp_metrics_hash))
1081                 vfree(net->ipv4.tcp_metrics_hash);
1082         else
1083                 kfree(net->ipv4.tcp_metrics_hash);
1084 }
1085 
1086 static __net_initdata struct pernet_operations tcp_net_metrics_ops = {
1087         .init   =       tcp_net_metrics_init,
1088         .exit   =       tcp_net_metrics_exit,
1089 };
1090 
1091 void __init tcp_metrics_init(void)
1092 {
1093         int ret;
1094 
1095         ret = register_pernet_subsys(&tcp_net_metrics_ops);
1096         if (ret < 0)
1097                 goto cleanup;
1098         ret = genl_register_family_with_ops(&tcp_metrics_nl_family,
1099                                             tcp_metrics_nl_ops,
1100                                             ARRAY_SIZE(tcp_metrics_nl_ops));
1101         if (ret < 0)
1102                 goto cleanup_subsys;
1103         return;
1104 
1105 cleanup_subsys:
1106         unregister_pernet_subsys(&tcp_net_metrics_ops);
1107 
1108 cleanup:
1109         return;
1110 }
1111 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp