~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/netfilter/nf_conntrack_core.c

Version: ~ [ linux-6.0-rc6 ] ~ [ linux-5.19.10 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.69 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.144 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.214 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.259 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.294 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.329 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-only
  2 /* Connection state tracking for netfilter.  This is separated from,
  3    but required by, the NAT layer; it can also be used by an iptables
  4    extension. */
  5 
  6 /* (C) 1999-2001 Paul `Rusty' Russell
  7  * (C) 2002-2006 Netfilter Core Team <coreteam@netfilter.org>
  8  * (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org>
  9  * (C) 2005-2012 Patrick McHardy <kaber@trash.net>
 10  */
 11 
 12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 13 
 14 #include <linux/types.h>
 15 #include <linux/netfilter.h>
 16 #include <linux/module.h>
 17 #include <linux/sched.h>
 18 #include <linux/skbuff.h>
 19 #include <linux/proc_fs.h>
 20 #include <linux/vmalloc.h>
 21 #include <linux/stddef.h>
 22 #include <linux/slab.h>
 23 #include <linux/random.h>
 24 #include <linux/jhash.h>
 25 #include <linux/siphash.h>
 26 #include <linux/err.h>
 27 #include <linux/percpu.h>
 28 #include <linux/moduleparam.h>
 29 #include <linux/notifier.h>
 30 #include <linux/kernel.h>
 31 #include <linux/netdevice.h>
 32 #include <linux/socket.h>
 33 #include <linux/mm.h>
 34 #include <linux/nsproxy.h>
 35 #include <linux/rculist_nulls.h>
 36 
 37 #include <net/netfilter/nf_conntrack.h>
 38 #include <net/netfilter/nf_conntrack_l4proto.h>
 39 #include <net/netfilter/nf_conntrack_expect.h>
 40 #include <net/netfilter/nf_conntrack_helper.h>
 41 #include <net/netfilter/nf_conntrack_seqadj.h>
 42 #include <net/netfilter/nf_conntrack_core.h>
 43 #include <net/netfilter/nf_conntrack_extend.h>
 44 #include <net/netfilter/nf_conntrack_acct.h>
 45 #include <net/netfilter/nf_conntrack_ecache.h>
 46 #include <net/netfilter/nf_conntrack_zones.h>
 47 #include <net/netfilter/nf_conntrack_timestamp.h>
 48 #include <net/netfilter/nf_conntrack_timeout.h>
 49 #include <net/netfilter/nf_conntrack_labels.h>
 50 #include <net/netfilter/nf_conntrack_synproxy.h>
 51 #include <net/netfilter/nf_nat.h>
 52 #include <net/netfilter/nf_nat_helper.h>
 53 #include <net/netns/hash.h>
 54 #include <net/ip.h>
 55 
 56 #include "nf_internals.h"
 57 
 58 __cacheline_aligned_in_smp spinlock_t nf_conntrack_locks[CONNTRACK_LOCKS];
 59 EXPORT_SYMBOL_GPL(nf_conntrack_locks);
 60 
 61 __cacheline_aligned_in_smp DEFINE_SPINLOCK(nf_conntrack_expect_lock);
 62 EXPORT_SYMBOL_GPL(nf_conntrack_expect_lock);
 63 
 64 struct hlist_nulls_head *nf_conntrack_hash __read_mostly;
 65 EXPORT_SYMBOL_GPL(nf_conntrack_hash);
 66 
 67 struct conntrack_gc_work {
 68         struct delayed_work     dwork;
 69         u32                     next_bucket;
 70         bool                    exiting;
 71         bool                    early_drop;
 72 };
 73 
 74 static __read_mostly struct kmem_cache *nf_conntrack_cachep;
 75 static DEFINE_SPINLOCK(nf_conntrack_locks_all_lock);
 76 static __read_mostly bool nf_conntrack_locks_all;
 77 
 78 /* serialize hash resizes and nf_ct_iterate_cleanup */
 79 static DEFINE_MUTEX(nf_conntrack_mutex);
 80 
 81 #define GC_SCAN_INTERVAL        (120u * HZ)
 82 #define GC_SCAN_MAX_DURATION    msecs_to_jiffies(10)
 83 
 84 static struct conntrack_gc_work conntrack_gc_work;
 85 
 86 void nf_conntrack_lock(spinlock_t *lock) __acquires(lock)
 87 {
 88         /* 1) Acquire the lock */
 89         spin_lock(lock);
 90 
 91         /* 2) read nf_conntrack_locks_all, with ACQUIRE semantics
 92          * It pairs with the smp_store_release() in nf_conntrack_all_unlock()
 93          */
 94         if (likely(smp_load_acquire(&nf_conntrack_locks_all) == false))
 95                 return;
 96 
 97         /* fast path failed, unlock */
 98         spin_unlock(lock);
 99 
100         /* Slow path 1) get global lock */
101         spin_lock(&nf_conntrack_locks_all_lock);
102 
103         /* Slow path 2) get the lock we want */
104         spin_lock(lock);
105 
106         /* Slow path 3) release the global lock */
107         spin_unlock(&nf_conntrack_locks_all_lock);
108 }
109 EXPORT_SYMBOL_GPL(nf_conntrack_lock);
110 
111 static void nf_conntrack_double_unlock(unsigned int h1, unsigned int h2)
112 {
113         h1 %= CONNTRACK_LOCKS;
114         h2 %= CONNTRACK_LOCKS;
115         spin_unlock(&nf_conntrack_locks[h1]);
116         if (h1 != h2)
117                 spin_unlock(&nf_conntrack_locks[h2]);
118 }
119 
120 /* return true if we need to recompute hashes (in case hash table was resized) */
121 static bool nf_conntrack_double_lock(struct net *net, unsigned int h1,
122                                      unsigned int h2, unsigned int sequence)
123 {
124         h1 %= CONNTRACK_LOCKS;
125         h2 %= CONNTRACK_LOCKS;
126         if (h1 <= h2) {
127                 nf_conntrack_lock(&nf_conntrack_locks[h1]);
128                 if (h1 != h2)
129                         spin_lock_nested(&nf_conntrack_locks[h2],
130                                          SINGLE_DEPTH_NESTING);
131         } else {
132                 nf_conntrack_lock(&nf_conntrack_locks[h2]);
133                 spin_lock_nested(&nf_conntrack_locks[h1],
134                                  SINGLE_DEPTH_NESTING);
135         }
136         if (read_seqcount_retry(&nf_conntrack_generation, sequence)) {
137                 nf_conntrack_double_unlock(h1, h2);
138                 return true;
139         }
140         return false;
141 }
142 
143 static void nf_conntrack_all_lock(void)
144         __acquires(&nf_conntrack_locks_all_lock)
145 {
146         int i;
147 
148         spin_lock(&nf_conntrack_locks_all_lock);
149 
150         /* For nf_contrack_locks_all, only the latest time when another
151          * CPU will see an update is controlled, by the "release" of the
152          * spin_lock below.
153          * The earliest time is not controlled, an thus KCSAN could detect
154          * a race when nf_conntract_lock() reads the variable.
155          * WRITE_ONCE() is used to ensure the compiler will not
156          * optimize the write.
157          */
158         WRITE_ONCE(nf_conntrack_locks_all, true);
159 
160         for (i = 0; i < CONNTRACK_LOCKS; i++) {
161                 spin_lock(&nf_conntrack_locks[i]);
162 
163                 /* This spin_unlock provides the "release" to ensure that
164                  * nf_conntrack_locks_all==true is visible to everyone that
165                  * acquired spin_lock(&nf_conntrack_locks[]).
166                  */
167                 spin_unlock(&nf_conntrack_locks[i]);
168         }
169 }
170 
171 static void nf_conntrack_all_unlock(void)
172         __releases(&nf_conntrack_locks_all_lock)
173 {
174         /* All prior stores must be complete before we clear
175          * 'nf_conntrack_locks_all'. Otherwise nf_conntrack_lock()
176          * might observe the false value but not the entire
177          * critical section.
178          * It pairs with the smp_load_acquire() in nf_conntrack_lock()
179          */
180         smp_store_release(&nf_conntrack_locks_all, false);
181         spin_unlock(&nf_conntrack_locks_all_lock);
182 }
183 
184 unsigned int nf_conntrack_htable_size __read_mostly;
185 EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
186 
187 unsigned int nf_conntrack_max __read_mostly;
188 EXPORT_SYMBOL_GPL(nf_conntrack_max);
189 seqcount_spinlock_t nf_conntrack_generation __read_mostly;
190 static unsigned int nf_conntrack_hash_rnd __read_mostly;
191 
192 static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple,
193                               const struct net *net)
194 {
195         unsigned int n;
196         u32 seed;
197 
198         get_random_once(&nf_conntrack_hash_rnd, sizeof(nf_conntrack_hash_rnd));
199 
200         /* The direction must be ignored, so we hash everything up to the
201          * destination ports (which is a multiple of 4) and treat the last
202          * three bytes manually.
203          */
204         seed = nf_conntrack_hash_rnd ^ net_hash_mix(net);
205         n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32);
206         return jhash2((u32 *)tuple, n, seed ^
207                       (((__force __u16)tuple->dst.u.all << 16) |
208                       tuple->dst.protonum));
209 }
210 
211 static u32 scale_hash(u32 hash)
212 {
213         return reciprocal_scale(hash, nf_conntrack_htable_size);
214 }
215 
216 static u32 __hash_conntrack(const struct net *net,
217                             const struct nf_conntrack_tuple *tuple,
218                             unsigned int size)
219 {
220         return reciprocal_scale(hash_conntrack_raw(tuple, net), size);
221 }
222 
223 static u32 hash_conntrack(const struct net *net,
224                           const struct nf_conntrack_tuple *tuple)
225 {
226         return scale_hash(hash_conntrack_raw(tuple, net));
227 }
228 
229 static bool nf_ct_get_tuple_ports(const struct sk_buff *skb,
230                                   unsigned int dataoff,
231                                   struct nf_conntrack_tuple *tuple)
232 {       struct {
233                 __be16 sport;
234                 __be16 dport;
235         } _inet_hdr, *inet_hdr;
236 
237         /* Actually only need first 4 bytes to get ports. */
238         inet_hdr = skb_header_pointer(skb, dataoff, sizeof(_inet_hdr), &_inet_hdr);
239         if (!inet_hdr)
240                 return false;
241 
242         tuple->src.u.udp.port = inet_hdr->sport;
243         tuple->dst.u.udp.port = inet_hdr->dport;
244         return true;
245 }
246 
247 static bool
248 nf_ct_get_tuple(const struct sk_buff *skb,
249                 unsigned int nhoff,
250                 unsigned int dataoff,
251                 u_int16_t l3num,
252                 u_int8_t protonum,
253                 struct net *net,
254                 struct nf_conntrack_tuple *tuple)
255 {
256         unsigned int size;
257         const __be32 *ap;
258         __be32 _addrs[8];
259 
260         memset(tuple, 0, sizeof(*tuple));
261 
262         tuple->src.l3num = l3num;
263         switch (l3num) {
264         case NFPROTO_IPV4:
265                 nhoff += offsetof(struct iphdr, saddr);
266                 size = 2 * sizeof(__be32);
267                 break;
268         case NFPROTO_IPV6:
269                 nhoff += offsetof(struct ipv6hdr, saddr);
270                 size = sizeof(_addrs);
271                 break;
272         default:
273                 return true;
274         }
275 
276         ap = skb_header_pointer(skb, nhoff, size, _addrs);
277         if (!ap)
278                 return false;
279 
280         switch (l3num) {
281         case NFPROTO_IPV4:
282                 tuple->src.u3.ip = ap[0];
283                 tuple->dst.u3.ip = ap[1];
284                 break;
285         case NFPROTO_IPV6:
286                 memcpy(tuple->src.u3.ip6, ap, sizeof(tuple->src.u3.ip6));
287                 memcpy(tuple->dst.u3.ip6, ap + 4, sizeof(tuple->dst.u3.ip6));
288                 break;
289         }
290 
291         tuple->dst.protonum = protonum;
292         tuple->dst.dir = IP_CT_DIR_ORIGINAL;
293 
294         switch (protonum) {
295 #if IS_ENABLED(CONFIG_IPV6)
296         case IPPROTO_ICMPV6:
297                 return icmpv6_pkt_to_tuple(skb, dataoff, net, tuple);
298 #endif
299         case IPPROTO_ICMP:
300                 return icmp_pkt_to_tuple(skb, dataoff, net, tuple);
301 #ifdef CONFIG_NF_CT_PROTO_GRE
302         case IPPROTO_GRE:
303                 return gre_pkt_to_tuple(skb, dataoff, net, tuple);
304 #endif
305         case IPPROTO_TCP:
306         case IPPROTO_UDP: /* fallthrough */
307                 return nf_ct_get_tuple_ports(skb, dataoff, tuple);
308 #ifdef CONFIG_NF_CT_PROTO_UDPLITE
309         case IPPROTO_UDPLITE:
310                 return nf_ct_get_tuple_ports(skb, dataoff, tuple);
311 #endif
312 #ifdef CONFIG_NF_CT_PROTO_SCTP
313         case IPPROTO_SCTP:
314                 return nf_ct_get_tuple_ports(skb, dataoff, tuple);
315 #endif
316 #ifdef CONFIG_NF_CT_PROTO_DCCP
317         case IPPROTO_DCCP:
318                 return nf_ct_get_tuple_ports(skb, dataoff, tuple);
319 #endif
320         default:
321                 break;
322         }
323 
324         return true;
325 }
326 
327 static int ipv4_get_l4proto(const struct sk_buff *skb, unsigned int nhoff,
328                             u_int8_t *protonum)
329 {
330         int dataoff = -1;
331         const struct iphdr *iph;
332         struct iphdr _iph;
333 
334         iph = skb_header_pointer(skb, nhoff, sizeof(_iph), &_iph);
335         if (!iph)
336                 return -1;
337 
338         /* Conntrack defragments packets, we might still see fragments
339          * inside ICMP packets though.
340          */
341         if (iph->frag_off & htons(IP_OFFSET))
342                 return -1;
343 
344         dataoff = nhoff + (iph->ihl << 2);
345         *protonum = iph->protocol;
346 
347         /* Check bogus IP headers */
348         if (dataoff > skb->len) {
349                 pr_debug("bogus IPv4 packet: nhoff %u, ihl %u, skblen %u\n",
350                          nhoff, iph->ihl << 2, skb->len);
351                 return -1;
352         }
353         return dataoff;
354 }
355 
356 #if IS_ENABLED(CONFIG_IPV6)
357 static int ipv6_get_l4proto(const struct sk_buff *skb, unsigned int nhoff,
358                             u8 *protonum)
359 {
360         int protoff = -1;
361         unsigned int extoff = nhoff + sizeof(struct ipv6hdr);
362         __be16 frag_off;
363         u8 nexthdr;
364 
365         if (skb_copy_bits(skb, nhoff + offsetof(struct ipv6hdr, nexthdr),
366                           &nexthdr, sizeof(nexthdr)) != 0) {
367                 pr_debug("can't get nexthdr\n");
368                 return -1;
369         }
370         protoff = ipv6_skip_exthdr(skb, extoff, &nexthdr, &frag_off);
371         /*
372          * (protoff == skb->len) means the packet has not data, just
373          * IPv6 and possibly extensions headers, but it is tracked anyway
374          */
375         if (protoff < 0 || (frag_off & htons(~0x7)) != 0) {
376                 pr_debug("can't find proto in pkt\n");
377                 return -1;
378         }
379 
380         *protonum = nexthdr;
381         return protoff;
382 }
383 #endif
384 
385 static int get_l4proto(const struct sk_buff *skb,
386                        unsigned int nhoff, u8 pf, u8 *l4num)
387 {
388         switch (pf) {
389         case NFPROTO_IPV4:
390                 return ipv4_get_l4proto(skb, nhoff, l4num);
391 #if IS_ENABLED(CONFIG_IPV6)
392         case NFPROTO_IPV6:
393                 return ipv6_get_l4proto(skb, nhoff, l4num);
394 #endif
395         default:
396                 *l4num = 0;
397                 break;
398         }
399         return -1;
400 }
401 
402 bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,
403                        u_int16_t l3num,
404                        struct net *net, struct nf_conntrack_tuple *tuple)
405 {
406         u8 protonum;
407         int protoff;
408 
409         protoff = get_l4proto(skb, nhoff, l3num, &protonum);
410         if (protoff <= 0)
411                 return false;
412 
413         return nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, net, tuple);
414 }
415 EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr);
416 
417 bool
418 nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,
419                    const struct nf_conntrack_tuple *orig)
420 {
421         memset(inverse, 0, sizeof(*inverse));
422 
423         inverse->src.l3num = orig->src.l3num;
424 
425         switch (orig->src.l3num) {
426         case NFPROTO_IPV4:
427                 inverse->src.u3.ip = orig->dst.u3.ip;
428                 inverse->dst.u3.ip = orig->src.u3.ip;
429                 break;
430         case NFPROTO_IPV6:
431                 inverse->src.u3.in6 = orig->dst.u3.in6;
432                 inverse->dst.u3.in6 = orig->src.u3.in6;
433                 break;
434         default:
435                 break;
436         }
437 
438         inverse->dst.dir = !orig->dst.dir;
439 
440         inverse->dst.protonum = orig->dst.protonum;
441 
442         switch (orig->dst.protonum) {
443         case IPPROTO_ICMP:
444                 return nf_conntrack_invert_icmp_tuple(inverse, orig);
445 #if IS_ENABLED(CONFIG_IPV6)
446         case IPPROTO_ICMPV6:
447                 return nf_conntrack_invert_icmpv6_tuple(inverse, orig);
448 #endif
449         }
450 
451         inverse->src.u.all = orig->dst.u.all;
452         inverse->dst.u.all = orig->src.u.all;
453         return true;
454 }
455 EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);
456 
457 /* Generate a almost-unique pseudo-id for a given conntrack.
458  *
459  * intentionally doesn't re-use any of the seeds used for hash
460  * table location, we assume id gets exposed to userspace.
461  *
462  * Following nf_conn items do not change throughout lifetime
463  * of the nf_conn:
464  *
465  * 1. nf_conn address
466  * 2. nf_conn->master address (normally NULL)
467  * 3. the associated net namespace
468  * 4. the original direction tuple
469  */
470 u32 nf_ct_get_id(const struct nf_conn *ct)
471 {
472         static __read_mostly siphash_key_t ct_id_seed;
473         unsigned long a, b, c, d;
474 
475         net_get_random_once(&ct_id_seed, sizeof(ct_id_seed));
476 
477         a = (unsigned long)ct;
478         b = (unsigned long)ct->master;
479         c = (unsigned long)nf_ct_net(ct);
480         d = (unsigned long)siphash(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
481                                    sizeof(ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple),
482                                    &ct_id_seed);
483 #ifdef CONFIG_64BIT
484         return siphash_4u64((u64)a, (u64)b, (u64)c, (u64)d, &ct_id_seed);
485 #else
486         return siphash_4u32((u32)a, (u32)b, (u32)c, (u32)d, &ct_id_seed);
487 #endif
488 }
489 EXPORT_SYMBOL_GPL(nf_ct_get_id);
490 
491 static void
492 clean_from_lists(struct nf_conn *ct)
493 {
494         pr_debug("clean_from_lists(%p)\n", ct);
495         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
496         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);
497 
498         /* Destroy all pending expectations */
499         nf_ct_remove_expectations(ct);
500 }
501 
502 /* must be called with local_bh_disable */
503 static void nf_ct_add_to_dying_list(struct nf_conn *ct)
504 {
505         struct ct_pcpu *pcpu;
506 
507         /* add this conntrack to the (per cpu) dying list */
508         ct->cpu = smp_processor_id();
509         pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
510 
511         spin_lock(&pcpu->lock);
512         hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
513                              &pcpu->dying);
514         spin_unlock(&pcpu->lock);
515 }
516 
517 /* must be called with local_bh_disable */
518 static void nf_ct_add_to_unconfirmed_list(struct nf_conn *ct)
519 {
520         struct ct_pcpu *pcpu;
521 
522         /* add this conntrack to the (per cpu) unconfirmed list */
523         ct->cpu = smp_processor_id();
524         pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
525 
526         spin_lock(&pcpu->lock);
527         hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
528                              &pcpu->unconfirmed);
529         spin_unlock(&pcpu->lock);
530 }
531 
532 /* must be called with local_bh_disable */
533 static void nf_ct_del_from_dying_or_unconfirmed_list(struct nf_conn *ct)
534 {
535         struct ct_pcpu *pcpu;
536 
537         /* We overload first tuple to link into unconfirmed or dying list.*/
538         pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
539 
540         spin_lock(&pcpu->lock);
541         BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode));
542         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
543         spin_unlock(&pcpu->lock);
544 }
545 
546 #define NFCT_ALIGN(len) (((len) + NFCT_INFOMASK) & ~NFCT_INFOMASK)
547 
548 /* Released via destroy_conntrack() */
549 struct nf_conn *nf_ct_tmpl_alloc(struct net *net,
550                                  const struct nf_conntrack_zone *zone,
551                                  gfp_t flags)
552 {
553         struct nf_conn *tmpl, *p;
554 
555         if (ARCH_KMALLOC_MINALIGN <= NFCT_INFOMASK) {
556                 tmpl = kzalloc(sizeof(*tmpl) + NFCT_INFOMASK, flags);
557                 if (!tmpl)
558                         return NULL;
559 
560                 p = tmpl;
561                 tmpl = (struct nf_conn *)NFCT_ALIGN((unsigned long)p);
562                 if (tmpl != p) {
563                         tmpl = (struct nf_conn *)NFCT_ALIGN((unsigned long)p);
564                         tmpl->proto.tmpl_padto = (char *)tmpl - (char *)p;
565                 }
566         } else {
567                 tmpl = kzalloc(sizeof(*tmpl), flags);
568                 if (!tmpl)
569                         return NULL;
570         }
571 
572         tmpl->status = IPS_TEMPLATE;
573         write_pnet(&tmpl->ct_net, net);
574         nf_ct_zone_add(tmpl, zone);
575         atomic_set(&tmpl->ct_general.use, 0);
576 
577         return tmpl;
578 }
579 EXPORT_SYMBOL_GPL(nf_ct_tmpl_alloc);
580 
581 void nf_ct_tmpl_free(struct nf_conn *tmpl)
582 {
583         nf_ct_ext_destroy(tmpl);
584 
585         if (ARCH_KMALLOC_MINALIGN <= NFCT_INFOMASK)
586                 kfree((char *)tmpl - tmpl->proto.tmpl_padto);
587         else
588                 kfree(tmpl);
589 }
590 EXPORT_SYMBOL_GPL(nf_ct_tmpl_free);
591 
592 static void destroy_gre_conntrack(struct nf_conn *ct)
593 {
594 #ifdef CONFIG_NF_CT_PROTO_GRE
595         struct nf_conn *master = ct->master;
596 
597         if (master)
598                 nf_ct_gre_keymap_destroy(master);
599 #endif
600 }
601 
602 static void
603 destroy_conntrack(struct nf_conntrack *nfct)
604 {
605         struct nf_conn *ct = (struct nf_conn *)nfct;
606 
607         pr_debug("destroy_conntrack(%p)\n", ct);
608         WARN_ON(atomic_read(&nfct->use) != 0);
609 
610         if (unlikely(nf_ct_is_template(ct))) {
611                 nf_ct_tmpl_free(ct);
612                 return;
613         }
614 
615         if (unlikely(nf_ct_protonum(ct) == IPPROTO_GRE))
616                 destroy_gre_conntrack(ct);
617 
618         local_bh_disable();
619         /* Expectations will have been removed in clean_from_lists,
620          * except TFTP can create an expectation on the first packet,
621          * before connection is in the list, so we need to clean here,
622          * too.
623          */
624         nf_ct_remove_expectations(ct);
625 
626         nf_ct_del_from_dying_or_unconfirmed_list(ct);
627 
628         local_bh_enable();
629 
630         if (ct->master)
631                 nf_ct_put(ct->master);
632 
633         pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct);
634         nf_conntrack_free(ct);
635 }
636 
637 static void nf_ct_delete_from_lists(struct nf_conn *ct)
638 {
639         struct net *net = nf_ct_net(ct);
640         unsigned int hash, reply_hash;
641         unsigned int sequence;
642 
643         nf_ct_helper_destroy(ct);
644 
645         local_bh_disable();
646         do {
647                 sequence = read_seqcount_begin(&nf_conntrack_generation);
648                 hash = hash_conntrack(net,
649                                       &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
650                 reply_hash = hash_conntrack(net,
651                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
652         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
653 
654         clean_from_lists(ct);
655         nf_conntrack_double_unlock(hash, reply_hash);
656 
657         nf_ct_add_to_dying_list(ct);
658 
659         local_bh_enable();
660 }
661 
662 bool nf_ct_delete(struct nf_conn *ct, u32 portid, int report)
663 {
664         struct nf_conn_tstamp *tstamp;
665         struct net *net;
666 
667         if (test_and_set_bit(IPS_DYING_BIT, &ct->status))
668                 return false;
669 
670         tstamp = nf_conn_tstamp_find(ct);
671         if (tstamp) {
672                 s32 timeout = ct->timeout - nfct_time_stamp;
673 
674                 tstamp->stop = ktime_get_real_ns();
675                 if (timeout < 0)
676                         tstamp->stop -= jiffies_to_nsecs(-timeout);
677         }
678 
679         if (nf_conntrack_event_report(IPCT_DESTROY, ct,
680                                     portid, report) < 0) {
681                 /* destroy event was not delivered. nf_ct_put will
682                  * be done by event cache worker on redelivery.
683                  */
684                 nf_ct_delete_from_lists(ct);
685                 nf_conntrack_ecache_work(nf_ct_net(ct), NFCT_ECACHE_DESTROY_FAIL);
686                 return false;
687         }
688 
689         net = nf_ct_net(ct);
690         if (nf_conntrack_ecache_dwork_pending(net))
691                 nf_conntrack_ecache_work(net, NFCT_ECACHE_DESTROY_SENT);
692         nf_ct_delete_from_lists(ct);
693         nf_ct_put(ct);
694         return true;
695 }
696 EXPORT_SYMBOL_GPL(nf_ct_delete);
697 
698 static inline bool
699 nf_ct_key_equal(struct nf_conntrack_tuple_hash *h,
700                 const struct nf_conntrack_tuple *tuple,
701                 const struct nf_conntrack_zone *zone,
702                 const struct net *net)
703 {
704         struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
705 
706         /* A conntrack can be recreated with the equal tuple,
707          * so we need to check that the conntrack is confirmed
708          */
709         return nf_ct_tuple_equal(tuple, &h->tuple) &&
710                nf_ct_zone_equal(ct, zone, NF_CT_DIRECTION(h)) &&
711                nf_ct_is_confirmed(ct) &&
712                net_eq(net, nf_ct_net(ct));
713 }
714 
715 static inline bool
716 nf_ct_match(const struct nf_conn *ct1, const struct nf_conn *ct2)
717 {
718         return nf_ct_tuple_equal(&ct1->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
719                                  &ct2->tuplehash[IP_CT_DIR_ORIGINAL].tuple) &&
720                nf_ct_tuple_equal(&ct1->tuplehash[IP_CT_DIR_REPLY].tuple,
721                                  &ct2->tuplehash[IP_CT_DIR_REPLY].tuple) &&
722                nf_ct_zone_equal(ct1, nf_ct_zone(ct2), IP_CT_DIR_ORIGINAL) &&
723                nf_ct_zone_equal(ct1, nf_ct_zone(ct2), IP_CT_DIR_REPLY) &&
724                net_eq(nf_ct_net(ct1), nf_ct_net(ct2));
725 }
726 
727 /* caller must hold rcu readlock and none of the nf_conntrack_locks */
728 static void nf_ct_gc_expired(struct nf_conn *ct)
729 {
730         if (!atomic_inc_not_zero(&ct->ct_general.use))
731                 return;
732 
733         if (nf_ct_should_gc(ct))
734                 nf_ct_kill(ct);
735 
736         nf_ct_put(ct);
737 }
738 
739 /*
740  * Warning :
741  * - Caller must take a reference on returned object
742  *   and recheck nf_ct_tuple_equal(tuple, &h->tuple)
743  */
744 static struct nf_conntrack_tuple_hash *
745 ____nf_conntrack_find(struct net *net, const struct nf_conntrack_zone *zone,
746                       const struct nf_conntrack_tuple *tuple, u32 hash)
747 {
748         struct nf_conntrack_tuple_hash *h;
749         struct hlist_nulls_head *ct_hash;
750         struct hlist_nulls_node *n;
751         unsigned int bucket, hsize;
752 
753 begin:
754         nf_conntrack_get_ht(&ct_hash, &hsize);
755         bucket = reciprocal_scale(hash, hsize);
756 
757         hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[bucket], hnnode) {
758                 struct nf_conn *ct;
759 
760                 ct = nf_ct_tuplehash_to_ctrack(h);
761                 if (nf_ct_is_expired(ct)) {
762                         nf_ct_gc_expired(ct);
763                         continue;
764                 }
765 
766                 if (nf_ct_key_equal(h, tuple, zone, net))
767                         return h;
768         }
769         /*
770          * if the nulls value we got at the end of this lookup is
771          * not the expected one, we must restart lookup.
772          * We probably met an item that was moved to another chain.
773          */
774         if (get_nulls_value(n) != bucket) {
775                 NF_CT_STAT_INC_ATOMIC(net, search_restart);
776                 goto begin;
777         }
778 
779         return NULL;
780 }
781 
782 /* Find a connection corresponding to a tuple. */
783 static struct nf_conntrack_tuple_hash *
784 __nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone,
785                         const struct nf_conntrack_tuple *tuple, u32 hash)
786 {
787         struct nf_conntrack_tuple_hash *h;
788         struct nf_conn *ct;
789 
790         rcu_read_lock();
791 
792         h = ____nf_conntrack_find(net, zone, tuple, hash);
793         if (h) {
794                 /* We have a candidate that matches the tuple we're interested
795                  * in, try to obtain a reference and re-check tuple
796                  */
797                 ct = nf_ct_tuplehash_to_ctrack(h);
798                 if (likely(atomic_inc_not_zero(&ct->ct_general.use))) {
799                         if (likely(nf_ct_key_equal(h, tuple, zone, net)))
800                                 goto found;
801 
802                         /* TYPESAFE_BY_RCU recycled the candidate */
803                         nf_ct_put(ct);
804                 }
805 
806                 h = NULL;
807         }
808 found:
809         rcu_read_unlock();
810 
811         return h;
812 }
813 
814 struct nf_conntrack_tuple_hash *
815 nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone,
816                       const struct nf_conntrack_tuple *tuple)
817 {
818         return __nf_conntrack_find_get(net, zone, tuple,
819                                        hash_conntrack_raw(tuple, net));
820 }
821 EXPORT_SYMBOL_GPL(nf_conntrack_find_get);
822 
823 static void __nf_conntrack_hash_insert(struct nf_conn *ct,
824                                        unsigned int hash,
825                                        unsigned int reply_hash)
826 {
827         hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
828                            &nf_conntrack_hash[hash]);
829         hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
830                            &nf_conntrack_hash[reply_hash]);
831 }
832 
833 int
834 nf_conntrack_hash_check_insert(struct nf_conn *ct)
835 {
836         const struct nf_conntrack_zone *zone;
837         struct net *net = nf_ct_net(ct);
838         unsigned int hash, reply_hash;
839         struct nf_conntrack_tuple_hash *h;
840         struct hlist_nulls_node *n;
841         unsigned int sequence;
842 
843         zone = nf_ct_zone(ct);
844 
845         local_bh_disable();
846         do {
847                 sequence = read_seqcount_begin(&nf_conntrack_generation);
848                 hash = hash_conntrack(net,
849                                       &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
850                 reply_hash = hash_conntrack(net,
851                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
852         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
853 
854         /* See if there's one in the list already, including reverse */
855         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode)
856                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
857                                     zone, net))
858                         goto out;
859 
860         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode)
861                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple,
862                                     zone, net))
863                         goto out;
864 
865         smp_wmb();
866         /* The caller holds a reference to this object */
867         atomic_set(&ct->ct_general.use, 2);
868         __nf_conntrack_hash_insert(ct, hash, reply_hash);
869         nf_conntrack_double_unlock(hash, reply_hash);
870         NF_CT_STAT_INC(net, insert);
871         local_bh_enable();
872         return 0;
873 
874 out:
875         nf_conntrack_double_unlock(hash, reply_hash);
876         local_bh_enable();
877         return -EEXIST;
878 }
879 EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert);
880 
881 void nf_ct_acct_add(struct nf_conn *ct, u32 dir, unsigned int packets,
882                     unsigned int bytes)
883 {
884         struct nf_conn_acct *acct;
885 
886         acct = nf_conn_acct_find(ct);
887         if (acct) {
888                 struct nf_conn_counter *counter = acct->counter;
889 
890                 atomic64_add(packets, &counter[dir].packets);
891                 atomic64_add(bytes, &counter[dir].bytes);
892         }
893 }
894 EXPORT_SYMBOL_GPL(nf_ct_acct_add);
895 
896 static void nf_ct_acct_merge(struct nf_conn *ct, enum ip_conntrack_info ctinfo,
897                              const struct nf_conn *loser_ct)
898 {
899         struct nf_conn_acct *acct;
900 
901         acct = nf_conn_acct_find(loser_ct);
902         if (acct) {
903                 struct nf_conn_counter *counter = acct->counter;
904                 unsigned int bytes;
905 
906                 /* u32 should be fine since we must have seen one packet. */
907                 bytes = atomic64_read(&counter[CTINFO2DIR(ctinfo)].bytes);
908                 nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), bytes);
909         }
910 }
911 
912 static void __nf_conntrack_insert_prepare(struct nf_conn *ct)
913 {
914         struct nf_conn_tstamp *tstamp;
915 
916         atomic_inc(&ct->ct_general.use);
917         ct->status |= IPS_CONFIRMED;
918 
919         /* set conntrack timestamp, if enabled. */
920         tstamp = nf_conn_tstamp_find(ct);
921         if (tstamp)
922                 tstamp->start = ktime_get_real_ns();
923 }
924 
925 /* caller must hold locks to prevent concurrent changes */
926 static int __nf_ct_resolve_clash(struct sk_buff *skb,
927                                  struct nf_conntrack_tuple_hash *h)
928 {
929         /* This is the conntrack entry already in hashes that won race. */
930         struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
931         enum ip_conntrack_info ctinfo;
932         struct nf_conn *loser_ct;
933 
934         loser_ct = nf_ct_get(skb, &ctinfo);
935 
936         if (nf_ct_is_dying(ct))
937                 return NF_DROP;
938 
939         if (((ct->status & IPS_NAT_DONE_MASK) == 0) ||
940             nf_ct_match(ct, loser_ct)) {
941                 struct net *net = nf_ct_net(ct);
942 
943                 nf_conntrack_get(&ct->ct_general);
944 
945                 nf_ct_acct_merge(ct, ctinfo, loser_ct);
946                 nf_ct_add_to_dying_list(loser_ct);
947                 nf_conntrack_put(&loser_ct->ct_general);
948                 nf_ct_set(skb, ct, ctinfo);
949 
950                 NF_CT_STAT_INC(net, clash_resolve);
951                 return NF_ACCEPT;
952         }
953 
954         return NF_DROP;
955 }
956 
957 /**
958  * nf_ct_resolve_clash_harder - attempt to insert clashing conntrack entry
959  *
960  * @skb: skb that causes the collision
961  * @repl_idx: hash slot for reply direction
962  *
963  * Called when origin or reply direction had a clash.
964  * The skb can be handled without packet drop provided the reply direction
965  * is unique or there the existing entry has the identical tuple in both
966  * directions.
967  *
968  * Caller must hold conntrack table locks to prevent concurrent updates.
969  *
970  * Returns NF_DROP if the clash could not be handled.
971  */
972 static int nf_ct_resolve_clash_harder(struct sk_buff *skb, u32 repl_idx)
973 {
974         struct nf_conn *loser_ct = (struct nf_conn *)skb_nfct(skb);
975         const struct nf_conntrack_zone *zone;
976         struct nf_conntrack_tuple_hash *h;
977         struct hlist_nulls_node *n;
978         struct net *net;
979 
980         zone = nf_ct_zone(loser_ct);
981         net = nf_ct_net(loser_ct);
982 
983         /* Reply direction must never result in a clash, unless both origin
984          * and reply tuples are identical.
985          */
986         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[repl_idx], hnnode) {
987                 if (nf_ct_key_equal(h,
988                                     &loser_ct->tuplehash[IP_CT_DIR_REPLY].tuple,
989                                     zone, net))
990                         return __nf_ct_resolve_clash(skb, h);
991         }
992 
993         /* We want the clashing entry to go away real soon: 1 second timeout. */
994         loser_ct->timeout = nfct_time_stamp + HZ;
995 
996         /* IPS_NAT_CLASH removes the entry automatically on the first
997          * reply.  Also prevents UDP tracker from moving the entry to
998          * ASSURED state, i.e. the entry can always be evicted under
999          * pressure.
1000          */
1001         loser_ct->status |= IPS_FIXED_TIMEOUT | IPS_NAT_CLASH;
1002 
1003         __nf_conntrack_insert_prepare(loser_ct);
1004 
1005         /* fake add for ORIGINAL dir: we want lookups to only find the entry
1006          * already in the table.  This also hides the clashing entry from
1007          * ctnetlink iteration, i.e. conntrack -L won't show them.
1008          */
1009         hlist_nulls_add_fake(&loser_ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
1010 
1011         hlist_nulls_add_head_rcu(&loser_ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
1012                                  &nf_conntrack_hash[repl_idx]);
1013 
1014         NF_CT_STAT_INC(net, clash_resolve);
1015         return NF_ACCEPT;
1016 }
1017 
1018 /**
1019  * nf_ct_resolve_clash - attempt to handle clash without packet drop
1020  *
1021  * @skb: skb that causes the clash
1022  * @h: tuplehash of the clashing entry already in table
1023  * @reply_hash: hash slot for reply direction
1024  *
1025  * A conntrack entry can be inserted to the connection tracking table
1026  * if there is no existing entry with an identical tuple.
1027  *
1028  * If there is one, @skb (and the assocated, unconfirmed conntrack) has
1029  * to be dropped.  In case @skb is retransmitted, next conntrack lookup
1030  * will find the already-existing entry.
1031  *
1032  * The major problem with such packet drop is the extra delay added by
1033  * the packet loss -- it will take some time for a retransmit to occur
1034  * (or the sender to time out when waiting for a reply).
1035  *
1036  * This function attempts to handle the situation without packet drop.
1037  *
1038  * If @skb has no NAT transformation or if the colliding entries are
1039  * exactly the same, only the to-be-confirmed conntrack entry is discarded
1040  * and @skb is associated with the conntrack entry already in the table.
1041  *
1042  * Failing that, the new, unconfirmed conntrack is still added to the table
1043  * provided that the collision only occurs in the ORIGINAL direction.
1044  * The new entry will be added only in the non-clashing REPLY direction,
1045  * so packets in the ORIGINAL direction will continue to match the existing
1046  * entry.  The new entry will also have a fixed timeout so it expires --
1047  * due to the collision, it will only see reply traffic.
1048  *
1049  * Returns NF_DROP if the clash could not be resolved.
1050  */
1051 static __cold noinline int
1052 nf_ct_resolve_clash(struct sk_buff *skb, struct nf_conntrack_tuple_hash *h,
1053                     u32 reply_hash)
1054 {
1055         /* This is the conntrack entry already in hashes that won race. */
1056         struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
1057         const struct nf_conntrack_l4proto *l4proto;
1058         enum ip_conntrack_info ctinfo;
1059         struct nf_conn *loser_ct;
1060         struct net *net;
1061         int ret;
1062 
1063         loser_ct = nf_ct_get(skb, &ctinfo);
1064         net = nf_ct_net(loser_ct);
1065 
1066         l4proto = nf_ct_l4proto_find(nf_ct_protonum(ct));
1067         if (!l4proto->allow_clash)
1068                 goto drop;
1069 
1070         ret = __nf_ct_resolve_clash(skb, h);
1071         if (ret == NF_ACCEPT)
1072                 return ret;
1073 
1074         ret = nf_ct_resolve_clash_harder(skb, reply_hash);
1075         if (ret == NF_ACCEPT)
1076                 return ret;
1077 
1078 drop:
1079         nf_ct_add_to_dying_list(loser_ct);
1080         NF_CT_STAT_INC(net, drop);
1081         NF_CT_STAT_INC(net, insert_failed);
1082         return NF_DROP;
1083 }
1084 
1085 /* Confirm a connection given skb; places it in hash table */
1086 int
1087 __nf_conntrack_confirm(struct sk_buff *skb)
1088 {
1089         const struct nf_conntrack_zone *zone;
1090         unsigned int hash, reply_hash;
1091         struct nf_conntrack_tuple_hash *h;
1092         struct nf_conn *ct;
1093         struct nf_conn_help *help;
1094         struct hlist_nulls_node *n;
1095         enum ip_conntrack_info ctinfo;
1096         struct net *net;
1097         unsigned int sequence;
1098         int ret = NF_DROP;
1099 
1100         ct = nf_ct_get(skb, &ctinfo);
1101         net = nf_ct_net(ct);
1102 
1103         /* ipt_REJECT uses nf_conntrack_attach to attach related
1104            ICMP/TCP RST packets in other direction.  Actual packet
1105            which created connection will be IP_CT_NEW or for an
1106            expected connection, IP_CT_RELATED. */
1107         if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)
1108                 return NF_ACCEPT;
1109 
1110         zone = nf_ct_zone(ct);
1111         local_bh_disable();
1112 
1113         do {
1114                 sequence = read_seqcount_begin(&nf_conntrack_generation);
1115                 /* reuse the hash saved before */
1116                 hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;
1117                 hash = scale_hash(hash);
1118                 reply_hash = hash_conntrack(net,
1119                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
1120 
1121         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
1122 
1123         /* We're not in hash table, and we refuse to set up related
1124          * connections for unconfirmed conns.  But packet copies and
1125          * REJECT will give spurious warnings here.
1126          */
1127 
1128         /* Another skb with the same unconfirmed conntrack may
1129          * win the race. This may happen for bridge(br_flood)
1130          * or broadcast/multicast packets do skb_clone with
1131          * unconfirmed conntrack.
1132          */
1133         if (unlikely(nf_ct_is_confirmed(ct))) {
1134                 WARN_ON_ONCE(1);
1135                 nf_conntrack_double_unlock(hash, reply_hash);
1136                 local_bh_enable();
1137                 return NF_DROP;
1138         }
1139 
1140         pr_debug("Confirming conntrack %p\n", ct);
1141         /* We have to check the DYING flag after unlink to prevent
1142          * a race against nf_ct_get_next_corpse() possibly called from
1143          * user context, else we insert an already 'dead' hash, blocking
1144          * further use of that particular connection -JM.
1145          */
1146         nf_ct_del_from_dying_or_unconfirmed_list(ct);
1147 
1148         if (unlikely(nf_ct_is_dying(ct))) {
1149                 nf_ct_add_to_dying_list(ct);
1150                 NF_CT_STAT_INC(net, insert_failed);
1151                 goto dying;
1152         }
1153 
1154         /* See if there's one in the list already, including reverse:
1155            NAT could have grabbed it without realizing, since we're
1156            not in the hash.  If there is, we lost race. */
1157         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode)
1158                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
1159                                     zone, net))
1160                         goto out;
1161 
1162         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode)
1163                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple,
1164                                     zone, net))
1165                         goto out;
1166 
1167         /* Timer relative to confirmation time, not original
1168            setting time, otherwise we'd get timer wrap in
1169            weird delay cases. */
1170         ct->timeout += nfct_time_stamp;
1171 
1172         __nf_conntrack_insert_prepare(ct);
1173 
1174         /* Since the lookup is lockless, hash insertion must be done after
1175          * starting the timer and setting the CONFIRMED bit. The RCU barriers
1176          * guarantee that no other CPU can find the conntrack before the above
1177          * stores are visible.
1178          */
1179         __nf_conntrack_hash_insert(ct, hash, reply_hash);
1180         nf_conntrack_double_unlock(hash, reply_hash);
1181         local_bh_enable();
1182 
1183         help = nfct_help(ct);
1184         if (help && help->helper)
1185                 nf_conntrack_event_cache(IPCT_HELPER, ct);
1186 
1187         nf_conntrack_event_cache(master_ct(ct) ?
1188                                  IPCT_RELATED : IPCT_NEW, ct);
1189         return NF_ACCEPT;
1190 
1191 out:
1192         ret = nf_ct_resolve_clash(skb, h, reply_hash);
1193 dying:
1194         nf_conntrack_double_unlock(hash, reply_hash);
1195         local_bh_enable();
1196         return ret;
1197 }
1198 EXPORT_SYMBOL_GPL(__nf_conntrack_confirm);
1199 
1200 /* Returns true if a connection correspondings to the tuple (required
1201    for NAT). */
1202 int
1203 nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,
1204                          const struct nf_conn *ignored_conntrack)
1205 {
1206         struct net *net = nf_ct_net(ignored_conntrack);
1207         const struct nf_conntrack_zone *zone;
1208         struct nf_conntrack_tuple_hash *h;
1209         struct hlist_nulls_head *ct_hash;
1210         unsigned int hash, hsize;
1211         struct hlist_nulls_node *n;
1212         struct nf_conn *ct;
1213 
1214         zone = nf_ct_zone(ignored_conntrack);
1215 
1216         rcu_read_lock();
1217  begin:
1218         nf_conntrack_get_ht(&ct_hash, &hsize);
1219         hash = __hash_conntrack(net, tuple, hsize);
1220 
1221         hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[hash], hnnode) {
1222                 ct = nf_ct_tuplehash_to_ctrack(h);
1223 
1224                 if (ct == ignored_conntrack)
1225                         continue;
1226 
1227                 if (nf_ct_is_expired(ct)) {
1228                         nf_ct_gc_expired(ct);
1229                         continue;
1230                 }
1231 
1232                 if (nf_ct_key_equal(h, tuple, zone, net)) {
1233                         /* Tuple is taken already, so caller will need to find
1234                          * a new source port to use.
1235                          *
1236                          * Only exception:
1237                          * If the *original tuples* are identical, then both
1238                          * conntracks refer to the same flow.
1239                          * This is a rare situation, it can occur e.g. when
1240                          * more than one UDP packet is sent from same socket
1241                          * in different threads.
1242                          *
1243                          * Let nf_ct_resolve_clash() deal with this later.
1244                          */
1245                         if (nf_ct_tuple_equal(&ignored_conntrack->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
1246                                               &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple) &&
1247                                               nf_ct_zone_equal(ct, zone, IP_CT_DIR_ORIGINAL))
1248                                 continue;
1249 
1250                         NF_CT_STAT_INC_ATOMIC(net, found);
1251                         rcu_read_unlock();
1252                         return 1;
1253                 }
1254         }
1255 
1256         if (get_nulls_value(n) != hash) {
1257                 NF_CT_STAT_INC_ATOMIC(net, search_restart);
1258                 goto begin;
1259         }
1260 
1261         rcu_read_unlock();
1262 
1263         return 0;
1264 }
1265 EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);
1266 
1267 #define NF_CT_EVICTION_RANGE    8
1268 
1269 /* There's a small race here where we may free a just-assured
1270    connection.  Too bad: we're in trouble anyway. */
1271 static unsigned int early_drop_list(struct net *net,
1272                                     struct hlist_nulls_head *head)
1273 {
1274         struct nf_conntrack_tuple_hash *h;
1275         struct hlist_nulls_node *n;
1276         unsigned int drops = 0;
1277         struct nf_conn *tmp;
1278 
1279         hlist_nulls_for_each_entry_rcu(h, n, head, hnnode) {
1280                 tmp = nf_ct_tuplehash_to_ctrack(h);
1281 
1282                 if (test_bit(IPS_OFFLOAD_BIT, &tmp->status))
1283                         continue;
1284 
1285                 if (nf_ct_is_expired(tmp)) {
1286                         nf_ct_gc_expired(tmp);
1287                         continue;
1288                 }
1289 
1290                 if (test_bit(IPS_ASSURED_BIT, &tmp->status) ||
1291                     !net_eq(nf_ct_net(tmp), net) ||
1292                     nf_ct_is_dying(tmp))
1293                         continue;
1294 
1295                 if (!atomic_inc_not_zero(&tmp->ct_general.use))
1296                         continue;
1297 
1298                 /* kill only if still in same netns -- might have moved due to
1299                  * SLAB_TYPESAFE_BY_RCU rules.
1300                  *
1301                  * We steal the timer reference.  If that fails timer has
1302                  * already fired or someone else deleted it. Just drop ref
1303                  * and move to next entry.
1304                  */
1305                 if (net_eq(nf_ct_net(tmp), net) &&
1306                     nf_ct_is_confirmed(tmp) &&
1307                     nf_ct_delete(tmp, 0, 0))
1308                         drops++;
1309 
1310                 nf_ct_put(tmp);
1311         }
1312 
1313         return drops;
1314 }
1315 
1316 static noinline int early_drop(struct net *net, unsigned int hash)
1317 {
1318         unsigned int i, bucket;
1319 
1320         for (i = 0; i < NF_CT_EVICTION_RANGE; i++) {
1321                 struct hlist_nulls_head *ct_hash;
1322                 unsigned int hsize, drops;
1323 
1324                 rcu_read_lock();
1325                 nf_conntrack_get_ht(&ct_hash, &hsize);
1326                 if (!i)
1327                         bucket = reciprocal_scale(hash, hsize);
1328                 else
1329                         bucket = (bucket + 1) % hsize;
1330 
1331                 drops = early_drop_list(net, &ct_hash[bucket]);
1332                 rcu_read_unlock();
1333 
1334                 if (drops) {
1335                         NF_CT_STAT_ADD_ATOMIC(net, early_drop, drops);
1336                         return true;
1337                 }
1338         }
1339 
1340         return false;
1341 }
1342 
1343 static bool gc_worker_skip_ct(const struct nf_conn *ct)
1344 {
1345         return !nf_ct_is_confirmed(ct) || nf_ct_is_dying(ct);
1346 }
1347 
1348 static bool gc_worker_can_early_drop(const struct nf_conn *ct)
1349 {
1350         const struct nf_conntrack_l4proto *l4proto;
1351 
1352         if (!test_bit(IPS_ASSURED_BIT, &ct->status))
1353                 return true;
1354 
1355         l4proto = nf_ct_l4proto_find(nf_ct_protonum(ct));
1356         if (l4proto->can_early_drop && l4proto->can_early_drop(ct))
1357                 return true;
1358 
1359         return false;
1360 }
1361 
1362 static void gc_worker(struct work_struct *work)
1363 {
1364         unsigned long end_time = jiffies + GC_SCAN_MAX_DURATION;
1365         unsigned int i, hashsz, nf_conntrack_max95 = 0;
1366         unsigned long next_run = GC_SCAN_INTERVAL;
1367         struct conntrack_gc_work *gc_work;
1368         gc_work = container_of(work, struct conntrack_gc_work, dwork.work);
1369 
1370         i = gc_work->next_bucket;
1371         if (gc_work->early_drop)
1372                 nf_conntrack_max95 = nf_conntrack_max / 100u * 95u;
1373 
1374         do {
1375                 struct nf_conntrack_tuple_hash *h;
1376                 struct hlist_nulls_head *ct_hash;
1377                 struct hlist_nulls_node *n;
1378                 struct nf_conn *tmp;
1379 
1380                 rcu_read_lock();
1381 
1382                 nf_conntrack_get_ht(&ct_hash, &hashsz);
1383                 if (i >= hashsz) {
1384                         rcu_read_unlock();
1385                         break;
1386                 }
1387 
1388                 hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[i], hnnode) {
1389                         struct nf_conntrack_net *cnet;
1390                         struct net *net;
1391 
1392                         tmp = nf_ct_tuplehash_to_ctrack(h);
1393 
1394                         if (test_bit(IPS_OFFLOAD_BIT, &tmp->status)) {
1395                                 nf_ct_offload_timeout(tmp);
1396                                 continue;
1397                         }
1398 
1399                         if (nf_ct_is_expired(tmp)) {
1400                                 nf_ct_gc_expired(tmp);
1401                                 continue;
1402                         }
1403 
1404                         if (nf_conntrack_max95 == 0 || gc_worker_skip_ct(tmp))
1405                                 continue;
1406 
1407                         net = nf_ct_net(tmp);
1408                         cnet = nf_ct_pernet(net);
1409                         if (atomic_read(&cnet->count) < nf_conntrack_max95)
1410                                 continue;
1411 
1412                         /* need to take reference to avoid possible races */
1413                         if (!atomic_inc_not_zero(&tmp->ct_general.use))
1414                                 continue;
1415 
1416                         if (gc_worker_skip_ct(tmp)) {
1417                                 nf_ct_put(tmp);
1418                                 continue;
1419                         }
1420 
1421                         if (gc_worker_can_early_drop(tmp))
1422                                 nf_ct_kill(tmp);
1423 
1424                         nf_ct_put(tmp);
1425                 }
1426 
1427                 /* could check get_nulls_value() here and restart if ct
1428                  * was moved to another chain.  But given gc is best-effort
1429                  * we will just continue with next hash slot.
1430                  */
1431                 rcu_read_unlock();
1432                 cond_resched();
1433                 i++;
1434 
1435                 if (time_after(jiffies, end_time) && i < hashsz) {
1436                         gc_work->next_bucket = i;
1437                         next_run = 0;
1438                         break;
1439                 }
1440         } while (i < hashsz);
1441 
1442         if (gc_work->exiting)
1443                 return;
1444 
1445         /*
1446          * Eviction will normally happen from the packet path, and not
1447          * from this gc worker.
1448          *
1449          * This worker is only here to reap expired entries when system went
1450          * idle after a busy period.
1451          */
1452         if (next_run) {
1453                 gc_work->early_drop = false;
1454                 gc_work->next_bucket = 0;
1455         }
1456         queue_delayed_work(system_power_efficient_wq, &gc_work->dwork, next_run);
1457 }
1458 
1459 static void conntrack_gc_work_init(struct conntrack_gc_work *gc_work)
1460 {
1461         INIT_DEFERRABLE_WORK(&gc_work->dwork, gc_worker);
1462         gc_work->exiting = false;
1463 }
1464 
1465 static struct nf_conn *
1466 __nf_conntrack_alloc(struct net *net,
1467                      const struct nf_conntrack_zone *zone,
1468                      const struct nf_conntrack_tuple *orig,
1469                      const struct nf_conntrack_tuple *repl,
1470                      gfp_t gfp, u32 hash)
1471 {
1472         struct nf_conntrack_net *cnet = nf_ct_pernet(net);
1473         unsigned int ct_count;
1474         struct nf_conn *ct;
1475 
1476         /* We don't want any race condition at early drop stage */
1477         ct_count = atomic_inc_return(&cnet->count);
1478 
1479         if (nf_conntrack_max && unlikely(ct_count > nf_conntrack_max)) {
1480                 if (!early_drop(net, hash)) {
1481                         if (!conntrack_gc_work.early_drop)
1482                                 conntrack_gc_work.early_drop = true;
1483                         atomic_dec(&cnet->count);
1484                         net_warn_ratelimited("nf_conntrack: table full, dropping packet\n");
1485                         return ERR_PTR(-ENOMEM);
1486                 }
1487         }
1488 
1489         /*
1490          * Do not use kmem_cache_zalloc(), as this cache uses
1491          * SLAB_TYPESAFE_BY_RCU.
1492          */
1493         ct = kmem_cache_alloc(nf_conntrack_cachep, gfp);
1494         if (ct == NULL)
1495                 goto out;
1496 
1497         spin_lock_init(&ct->lock);
1498         ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;
1499         ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;
1500         ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;
1501         /* save hash for reusing when confirming */
1502         *(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;
1503         ct->status = 0;
1504         ct->timeout = 0;
1505         write_pnet(&ct->ct_net, net);
1506         memset(&ct->__nfct_init_offset, 0,
1507                offsetof(struct nf_conn, proto) -
1508                offsetof(struct nf_conn, __nfct_init_offset));
1509 
1510         nf_ct_zone_add(ct, zone);
1511 
1512         /* Because we use RCU lookups, we set ct_general.use to zero before
1513          * this is inserted in any list.
1514          */
1515         atomic_set(&ct->ct_general.use, 0);
1516         return ct;
1517 out:
1518         atomic_dec(&cnet->count);
1519         return ERR_PTR(-ENOMEM);
1520 }
1521 
1522 struct nf_conn *nf_conntrack_alloc(struct net *net,
1523                                    const struct nf_conntrack_zone *zone,
1524                                    const struct nf_conntrack_tuple *orig,
1525                                    const struct nf_conntrack_tuple *repl,
1526                                    gfp_t gfp)
1527 {
1528         return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);
1529 }
1530 EXPORT_SYMBOL_GPL(nf_conntrack_alloc);
1531 
1532 void nf_conntrack_free(struct nf_conn *ct)
1533 {
1534         struct net *net = nf_ct_net(ct);
1535         struct nf_conntrack_net *cnet;
1536 
1537         /* A freed object has refcnt == 0, that's
1538          * the golden rule for SLAB_TYPESAFE_BY_RCU
1539          */
1540         WARN_ON(atomic_read(&ct->ct_general.use) != 0);
1541 
1542         nf_ct_ext_destroy(ct);
1543         kmem_cache_free(nf_conntrack_cachep, ct);
1544         cnet = nf_ct_pernet(net);
1545 
1546         smp_mb__before_atomic();
1547         atomic_dec(&cnet->count);
1548 }
1549 EXPORT_SYMBOL_GPL(nf_conntrack_free);
1550 
1551 
1552 /* Allocate a new conntrack: we return -ENOMEM if classification
1553    failed due to stress.  Otherwise it really is unclassifiable. */
1554 static noinline struct nf_conntrack_tuple_hash *
1555 init_conntrack(struct net *net, struct nf_conn *tmpl,
1556                const struct nf_conntrack_tuple *tuple,
1557                struct sk_buff *skb,
1558                unsigned int dataoff, u32 hash)
1559 {
1560         struct nf_conn *ct;
1561         struct nf_conn_help *help;
1562         struct nf_conntrack_tuple repl_tuple;
1563         struct nf_conntrack_ecache *ecache;
1564         struct nf_conntrack_expect *exp = NULL;
1565         const struct nf_conntrack_zone *zone;
1566         struct nf_conn_timeout *timeout_ext;
1567         struct nf_conntrack_zone tmp;
1568         struct nf_conntrack_net *cnet;
1569 
1570         if (!nf_ct_invert_tuple(&repl_tuple, tuple)) {
1571                 pr_debug("Can't invert tuple.\n");
1572                 return NULL;
1573         }
1574 
1575         zone = nf_ct_zone_tmpl(tmpl, skb, &tmp);
1576         ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,
1577                                   hash);
1578         if (IS_ERR(ct))
1579                 return (struct nf_conntrack_tuple_hash *)ct;
1580 
1581         if (!nf_ct_add_synproxy(ct, tmpl)) {
1582                 nf_conntrack_free(ct);
1583                 return ERR_PTR(-ENOMEM);
1584         }
1585 
1586         timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL;
1587 
1588         if (timeout_ext)
1589                 nf_ct_timeout_ext_add(ct, rcu_dereference(timeout_ext->timeout),
1590                                       GFP_ATOMIC);
1591 
1592         nf_ct_acct_ext_add(ct, GFP_ATOMIC);
1593         nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);
1594         nf_ct_labels_ext_add(ct);
1595 
1596         ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;
1597         nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,
1598                                  ecache ? ecache->expmask : 0,
1599                              GFP_ATOMIC);
1600 
1601         local_bh_disable();
1602         cnet = nf_ct_pernet(net);
1603         if (cnet->expect_count) {
1604                 spin_lock(&nf_conntrack_expect_lock);
1605                 exp = nf_ct_find_expectation(net, zone, tuple);
1606                 if (exp) {
1607                         pr_debug("expectation arrives ct=%p exp=%p\n",
1608                                  ct, exp);
1609                         /* Welcome, Mr. Bond.  We've been expecting you... */
1610                         __set_bit(IPS_EXPECTED_BIT, &ct->status);
1611                         /* exp->master safe, refcnt bumped in nf_ct_find_expectation */
1612                         ct->master = exp->master;
1613                         if (exp->helper) {
1614                                 help = nf_ct_helper_ext_add(ct, GFP_ATOMIC);
1615                                 if (help)
1616                                         rcu_assign_pointer(help->helper, exp->helper);
1617                         }
1618 
1619 #ifdef CONFIG_NF_CONNTRACK_MARK
1620                         ct->mark = exp->master->mark;
1621 #endif
1622 #ifdef CONFIG_NF_CONNTRACK_SECMARK
1623                         ct->secmark = exp->master->secmark;
1624 #endif
1625                         NF_CT_STAT_INC(net, expect_new);
1626                 }
1627                 spin_unlock(&nf_conntrack_expect_lock);
1628         }
1629         if (!exp)
1630                 __nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);
1631 
1632         /* Now it is inserted into the unconfirmed list, bump refcount */
1633         nf_conntrack_get(&ct->ct_general);
1634         nf_ct_add_to_unconfirmed_list(ct);
1635 
1636         local_bh_enable();
1637 
1638         if (exp) {
1639                 if (exp->expectfn)
1640                         exp->expectfn(ct, exp);
1641                 nf_ct_expect_put(exp);
1642         }
1643 
1644         return &ct->tuplehash[IP_CT_DIR_ORIGINAL];
1645 }
1646 
1647 /* On success, returns 0, sets skb->_nfct | ctinfo */
1648 static int
1649 resolve_normal_ct(struct nf_conn *tmpl,
1650                   struct sk_buff *skb,
1651                   unsigned int dataoff,
1652                   u_int8_t protonum,
1653                   const struct nf_hook_state *state)
1654 {
1655         const struct nf_conntrack_zone *zone;
1656         struct nf_conntrack_tuple tuple;
1657         struct nf_conntrack_tuple_hash *h;
1658         enum ip_conntrack_info ctinfo;
1659         struct nf_conntrack_zone tmp;
1660         struct nf_conn *ct;
1661         u32 hash;
1662 
1663         if (!nf_ct_get_tuple(skb, skb_network_offset(skb),
1664                              dataoff, state->pf, protonum, state->net,
1665                              &tuple)) {
1666                 pr_debug("Can't get tuple\n");
1667                 return 0;
1668         }
1669 
1670         /* look for tuple match */
1671         zone = nf_ct_zone_tmpl(tmpl, skb, &tmp);
1672         hash = hash_conntrack_raw(&tuple, state->net);
1673         h = __nf_conntrack_find_get(state->net, zone, &tuple, hash);
1674         if (!h) {
1675                 h = init_conntrack(state->net, tmpl, &tuple,
1676                                    skb, dataoff, hash);
1677                 if (!h)
1678                         return 0;
1679                 if (IS_ERR(h))
1680                         return PTR_ERR(h);
1681         }
1682         ct = nf_ct_tuplehash_to_ctrack(h);
1683 
1684         /* It exists; we have (non-exclusive) reference. */
1685         if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {
1686                 ctinfo = IP_CT_ESTABLISHED_REPLY;
1687         } else {
1688                 /* Once we've had two way comms, always ESTABLISHED. */
1689                 if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {
1690                         pr_debug("normal packet for %p\n", ct);
1691                         ctinfo = IP_CT_ESTABLISHED;
1692                 } else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {
1693                         pr_debug("related packet for %p\n", ct);
1694                         ctinfo = IP_CT_RELATED;
1695                 } else {
1696                         pr_debug("new packet for %p\n", ct);
1697                         ctinfo = IP_CT_NEW;
1698                 }
1699         }
1700         nf_ct_set(skb, ct, ctinfo);
1701         return 0;
1702 }
1703 
1704 /*
1705  * icmp packets need special treatment to handle error messages that are
1706  * related to a connection.
1707  *
1708  * Callers need to check if skb has a conntrack assigned when this
1709  * helper returns; in such case skb belongs to an already known connection.
1710  */
1711 static unsigned int __cold
1712 nf_conntrack_handle_icmp(struct nf_conn *tmpl,
1713                          struct sk_buff *skb,
1714                          unsigned int dataoff,
1715                          u8 protonum,
1716                          const struct nf_hook_state *state)
1717 {
1718         int ret;
1719 
1720         if (state->pf == NFPROTO_IPV4 && protonum == IPPROTO_ICMP)
1721                 ret = nf_conntrack_icmpv4_error(tmpl, skb, dataoff, state);
1722 #if IS_ENABLED(CONFIG_IPV6)
1723         else if (state->pf == NFPROTO_IPV6 && protonum == IPPROTO_ICMPV6)
1724                 ret = nf_conntrack_icmpv6_error(tmpl, skb, dataoff, state);
1725 #endif
1726         else
1727                 return NF_ACCEPT;
1728 
1729         if (ret <= 0)
1730                 NF_CT_STAT_INC_ATOMIC(state->net, error);
1731 
1732         return ret;
1733 }
1734 
1735 static int generic_packet(struct nf_conn *ct, struct sk_buff *skb,
1736                           enum ip_conntrack_info ctinfo)
1737 {
1738         const unsigned int *timeout = nf_ct_timeout_lookup(ct);
1739 
1740         if (!timeout)
1741                 timeout = &nf_generic_pernet(nf_ct_net(ct))->timeout;
1742 
1743         nf_ct_refresh_acct(ct, ctinfo, skb, *timeout);
1744         return NF_ACCEPT;
1745 }
1746 
1747 /* Returns verdict for packet, or -1 for invalid. */
1748 static int nf_conntrack_handle_packet(struct nf_conn *ct,
1749                                       struct sk_buff *skb,
1750                                       unsigned int dataoff,
1751                                       enum ip_conntrack_info ctinfo,
1752                                       const struct nf_hook_state *state)
1753 {
1754         switch (nf_ct_protonum(ct)) {
1755         case IPPROTO_TCP:
1756                 return nf_conntrack_tcp_packet(ct, skb, dataoff,
1757                                                ctinfo, state);
1758         case IPPROTO_UDP:
1759                 return nf_conntrack_udp_packet(ct, skb, dataoff,
1760                                                ctinfo, state);
1761         case IPPROTO_ICMP:
1762                 return nf_conntrack_icmp_packet(ct, skb, ctinfo, state);
1763 #if IS_ENABLED(CONFIG_IPV6)
1764         case IPPROTO_ICMPV6:
1765                 return nf_conntrack_icmpv6_packet(ct, skb, ctinfo, state);
1766 #endif
1767 #ifdef CONFIG_NF_CT_PROTO_UDPLITE
1768         case IPPROTO_UDPLITE:
1769                 return nf_conntrack_udplite_packet(ct, skb, dataoff,
1770                                                    ctinfo, state);
1771 #endif
1772 #ifdef CONFIG_NF_CT_PROTO_SCTP
1773         case IPPROTO_SCTP:
1774                 return nf_conntrack_sctp_packet(ct, skb, dataoff,
1775                                                 ctinfo, state);
1776 #endif
1777 #ifdef CONFIG_NF_CT_PROTO_DCCP
1778         case IPPROTO_DCCP:
1779                 return nf_conntrack_dccp_packet(ct, skb, dataoff,
1780                                                 ctinfo, state);
1781 #endif
1782 #ifdef CONFIG_NF_CT_PROTO_GRE
1783         case IPPROTO_GRE:
1784                 return nf_conntrack_gre_packet(ct, skb, dataoff,
1785                                                ctinfo, state);
1786 #endif
1787         }
1788 
1789         return generic_packet(ct, skb, ctinfo);
1790 }
1791 
1792 unsigned int
1793 nf_conntrack_in(struct sk_buff *skb, const struct nf_hook_state *state)
1794 {
1795         enum ip_conntrack_info ctinfo;
1796         struct nf_conn *ct, *tmpl;
1797         u_int8_t protonum;
1798         int dataoff, ret;
1799 
1800         tmpl = nf_ct_get(skb, &ctinfo);
1801         if (tmpl || ctinfo == IP_CT_UNTRACKED) {
1802                 /* Previously seen (loopback or untracked)?  Ignore. */
1803                 if ((tmpl && !nf_ct_is_template(tmpl)) ||
1804                      ctinfo == IP_CT_UNTRACKED)
1805                         return NF_ACCEPT;
1806                 skb->_nfct = 0;
1807         }
1808 
1809         /* rcu_read_lock()ed by nf_hook_thresh */
1810         dataoff = get_l4proto(skb, skb_network_offset(skb), state->pf, &protonum);
1811         if (dataoff <= 0) {
1812                 pr_debug("not prepared to track yet or error occurred\n");
1813                 NF_CT_STAT_INC_ATOMIC(state->net, invalid);
1814                 ret = NF_ACCEPT;
1815                 goto out;
1816         }
1817 
1818         if (protonum == IPPROTO_ICMP || protonum == IPPROTO_ICMPV6) {
1819                 ret = nf_conntrack_handle_icmp(tmpl, skb, dataoff,
1820                                                protonum, state);
1821                 if (ret <= 0) {
1822                         ret = -ret;
1823                         goto out;
1824                 }
1825                 /* ICMP[v6] protocol trackers may assign one conntrack. */
1826                 if (skb->_nfct)
1827                         goto out;
1828         }
1829 repeat:
1830         ret = resolve_normal_ct(tmpl, skb, dataoff,
1831                                 protonum, state);
1832         if (ret < 0) {
1833                 /* Too stressed to deal. */
1834                 NF_CT_STAT_INC_ATOMIC(state->net, drop);
1835                 ret = NF_DROP;
1836                 goto out;
1837         }
1838 
1839         ct = nf_ct_get(skb, &ctinfo);
1840         if (!ct) {
1841                 /* Not valid part of a connection */
1842                 NF_CT_STAT_INC_ATOMIC(state->net, invalid);
1843                 ret = NF_ACCEPT;
1844                 goto out;
1845         }
1846 
1847         ret = nf_conntrack_handle_packet(ct, skb, dataoff, ctinfo, state);
1848         if (ret <= 0) {
1849                 /* Invalid: inverse of the return code tells
1850                  * the netfilter core what to do */
1851                 pr_debug("nf_conntrack_in: Can't track with proto module\n");
1852                 nf_conntrack_put(&ct->ct_general);
1853                 skb->_nfct = 0;
1854                 NF_CT_STAT_INC_ATOMIC(state->net, invalid);
1855                 if (ret == -NF_DROP)
1856                         NF_CT_STAT_INC_ATOMIC(state->net, drop);
1857                 /* Special case: TCP tracker reports an attempt to reopen a
1858                  * closed/aborted connection. We have to go back and create a
1859                  * fresh conntrack.
1860                  */
1861                 if (ret == -NF_REPEAT)
1862                         goto repeat;
1863                 ret = -ret;
1864                 goto out;
1865         }
1866 
1867         if (ctinfo == IP_CT_ESTABLISHED_REPLY &&
1868             !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))
1869                 nf_conntrack_event_cache(IPCT_REPLY, ct);
1870 out:
1871         if (tmpl)
1872                 nf_ct_put(tmpl);
1873 
1874         return ret;
1875 }
1876 EXPORT_SYMBOL_GPL(nf_conntrack_in);
1877 
1878 /* Alter reply tuple (maybe alter helper).  This is for NAT, and is
1879    implicitly racy: see __nf_conntrack_confirm */
1880 void nf_conntrack_alter_reply(struct nf_conn *ct,
1881                               const struct nf_conntrack_tuple *newreply)
1882 {
1883         struct nf_conn_help *help = nfct_help(ct);
1884 
1885         /* Should be unconfirmed, so not in hash table yet */
1886         WARN_ON(nf_ct_is_confirmed(ct));
1887 
1888         pr_debug("Altering reply tuple of %p to ", ct);
1889         nf_ct_dump_tuple(newreply);
1890 
1891         ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;
1892         if (ct->master || (help && !hlist_empty(&help->expectations)))
1893                 return;
1894 
1895         rcu_read_lock();
1896         __nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC);
1897         rcu_read_unlock();
1898 }
1899 EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply);
1900 
1901 /* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */
1902 void __nf_ct_refresh_acct(struct nf_conn *ct,
1903                           enum ip_conntrack_info ctinfo,
1904                           const struct sk_buff *skb,
1905                           u32 extra_jiffies,
1906                           bool do_acct)
1907 {
1908         /* Only update if this is not a fixed timeout */
1909         if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))
1910                 goto acct;
1911 
1912         /* If not in hash table, timer will not be active yet */
1913         if (nf_ct_is_confirmed(ct))
1914                 extra_jiffies += nfct_time_stamp;
1915 
1916         if (READ_ONCE(ct->timeout) != extra_jiffies)
1917                 WRITE_ONCE(ct->timeout, extra_jiffies);
1918 acct:
1919         if (do_acct)
1920                 nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), skb->len);
1921 }
1922 EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);
1923 
1924 bool nf_ct_kill_acct(struct nf_conn *ct,
1925                      enum ip_conntrack_info ctinfo,
1926                      const struct sk_buff *skb)
1927 {
1928         nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), skb->len);
1929 
1930         return nf_ct_delete(ct, 0, 0);
1931 }
1932 EXPORT_SYMBOL_GPL(nf_ct_kill_acct);
1933 
1934 #if IS_ENABLED(CONFIG_NF_CT_NETLINK)
1935 
1936 #include <linux/netfilter/nfnetlink.h>
1937 #include <linux/netfilter/nfnetlink_conntrack.h>
1938 #include <linux/mutex.h>
1939 
1940 /* Generic function for tcp/udp/sctp/dccp and alike. */
1941 int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,
1942                                const struct nf_conntrack_tuple *tuple)
1943 {
1944         if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) ||
1945             nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port))
1946                 goto nla_put_failure;
1947         return 0;
1948 
1949 nla_put_failure:
1950         return -1;
1951 }
1952 EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr);
1953 
1954 const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = {
1955         [CTA_PROTO_SRC_PORT]  = { .type = NLA_U16 },
1956         [CTA_PROTO_DST_PORT]  = { .type = NLA_U16 },
1957 };
1958 EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);
1959 
1960 int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],
1961                                struct nf_conntrack_tuple *t,
1962                                u_int32_t flags)
1963 {
1964         if (flags & CTA_FILTER_FLAG(CTA_PROTO_SRC_PORT)) {
1965                 if (!tb[CTA_PROTO_SRC_PORT])
1966                         return -EINVAL;
1967 
1968                 t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);
1969         }
1970 
1971         if (flags & CTA_FILTER_FLAG(CTA_PROTO_DST_PORT)) {
1972                 if (!tb[CTA_PROTO_DST_PORT])
1973                         return -EINVAL;
1974 
1975                 t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);
1976         }
1977 
1978         return 0;
1979 }
1980 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple);
1981 
1982 unsigned int nf_ct_port_nlattr_tuple_size(void)
1983 {
1984         static unsigned int size __read_mostly;
1985 
1986         if (!size)
1987                 size = nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);
1988 
1989         return size;
1990 }
1991 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size);
1992 #endif
1993 
1994 /* Used by ipt_REJECT and ip6t_REJECT. */
1995 static void nf_conntrack_attach(struct sk_buff *nskb, const struct sk_buff *skb)
1996 {
1997         struct nf_conn *ct;
1998         enum ip_conntrack_info ctinfo;
1999 
2000         /* This ICMP is in reverse direction to the packet which caused it */
2001         ct = nf_ct_get(skb, &ctinfo);
2002         if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)
2003                 ctinfo = IP_CT_RELATED_REPLY;
2004         else
2005                 ctinfo = IP_CT_RELATED;
2006 
2007         /* Attach to new skbuff, and increment count */
2008         nf_ct_set(nskb, ct, ctinfo);
2009         nf_conntrack_get(skb_nfct(nskb));
2010 }
2011 
2012 static int __nf_conntrack_update(struct net *net, struct sk_buff *skb,
2013                                  struct nf_conn *ct,
2014                                  enum ip_conntrack_info ctinfo)
2015 {
2016         struct nf_conntrack_tuple_hash *h;
2017         struct nf_conntrack_tuple tuple;
2018         struct nf_nat_hook *nat_hook;
2019         unsigned int status;
2020         int dataoff;
2021         u16 l3num;
2022         u8 l4num;
2023 
2024         l3num = nf_ct_l3num(ct);
2025 
2026         dataoff = get_l4proto(skb, skb_network_offset(skb), l3num, &l4num);
2027         if (dataoff <= 0)
2028                 return -1;
2029 
2030         if (!nf_ct_get_tuple(skb, skb_network_offset(skb), dataoff, l3num,
2031                              l4num, net, &tuple))
2032                 return -1;
2033 
2034         if (ct->status & IPS_SRC_NAT) {
2035                 memcpy(tuple.src.u3.all,
2036                        ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.src.u3.all,
2037                        sizeof(tuple.src.u3.all));
2038                 tuple.src.u.all =
2039                         ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.src.u.all;
2040         }
2041 
2042         if (ct->status & IPS_DST_NAT) {
2043                 memcpy(tuple.dst.u3.all,
2044                        ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.dst.u3.all,
2045                        sizeof(tuple.dst.u3.all));
2046                 tuple.dst.u.all =
2047                         ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.dst.u.all;
2048         }
2049 
2050         h = nf_conntrack_find_get(net, nf_ct_zone(ct), &tuple);
2051         if (!h)
2052                 return 0;
2053 
2054         /* Store status bits of the conntrack that is clashing to re-do NAT
2055          * mangling according to what it has been done already to this packet.
2056          */
2057         status = ct->status;
2058 
2059         nf_ct_put(ct);
2060         ct = nf_ct_tuplehash_to_ctrack(h);
2061         nf_ct_set(skb, ct, ctinfo);
2062 
2063         nat_hook = rcu_dereference(nf_nat_hook);
2064         if (!nat_hook)
2065                 return 0;
2066 
2067         if (status & IPS_SRC_NAT &&
2068             nat_hook->manip_pkt(skb, ct, NF_NAT_MANIP_SRC,
2069                                 IP_CT_DIR_ORIGINAL) == NF_DROP)
2070                 return -1;
2071 
2072         if (status & IPS_DST_NAT &&
2073             nat_hook->manip_pkt(skb, ct, NF_NAT_MANIP_DST,
2074                                 IP_CT_DIR_ORIGINAL) == NF_DROP)
2075                 return -1;
2076 
2077         return 0;
2078 }
2079 
2080 /* This packet is coming from userspace via nf_queue, complete the packet
2081  * processing after the helper invocation in nf_confirm().
2082  */
2083 static int nf_confirm_cthelper(struct sk_buff *skb, struct nf_conn *ct,
2084                                enum ip_conntrack_info ctinfo)
2085 {
2086         const struct nf_conntrack_helper *helper;
2087         const struct nf_conn_help *help;
2088         int protoff;
2089 
2090         help = nfct_help(ct);
2091         if (!help)
2092                 return 0;
2093 
2094         helper = rcu_dereference(help->helper);
2095         if (!(helper->flags & NF_CT_HELPER_F_USERSPACE))
2096                 return 0;
2097 
2098         switch (nf_ct_l3num(ct)) {
2099         case NFPROTO_IPV4:
2100                 protoff = skb_network_offset(skb) + ip_hdrlen(skb);
2101                 break;
2102 #if IS_ENABLED(CONFIG_IPV6)
2103         case NFPROTO_IPV6: {
2104                 __be16 frag_off;
2105                 u8 pnum;
2106 
2107                 pnum = ipv6_hdr(skb)->nexthdr;
2108                 protoff = ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &pnum,
2109                                            &frag_off);
2110                 if (protoff < 0 || (frag_off & htons(~0x7)) != 0)
2111                         return 0;
2112                 break;
2113         }
2114 #endif
2115         default:
2116                 return 0;
2117         }
2118 
2119         if (test_bit(IPS_SEQ_ADJUST_BIT, &ct->status) &&
2120             !nf_is_loopback_packet(skb)) {
2121                 if (!nf_ct_seq_adjust(skb, ct, ctinfo, protoff)) {
2122                         NF_CT_STAT_INC_ATOMIC(nf_ct_net(ct), drop);
2123                         return -1;
2124                 }
2125         }
2126 
2127         /* We've seen it coming out the other side: confirm it */
2128         return nf_conntrack_confirm(skb) == NF_DROP ? - 1 : 0;
2129 }
2130 
2131 static int nf_conntrack_update(struct net *net, struct sk_buff *skb)
2132 {
2133         enum ip_conntrack_info ctinfo;
2134         struct nf_conn *ct;
2135         int err;
2136 
2137         ct = nf_ct_get(skb, &ctinfo);
2138         if (!ct)
2139                 return 0;
2140 
2141         if (!nf_ct_is_confirmed(ct)) {
2142                 err = __nf_conntrack_update(net, skb, ct, ctinfo);
2143                 if (err < 0)
2144                         return err;
2145 
2146                 ct = nf_ct_get(skb, &ctinfo);
2147         }
2148 
2149         return nf_confirm_cthelper(skb, ct, ctinfo);
2150 }
2151 
2152 static bool nf_conntrack_get_tuple_skb(struct nf_conntrack_tuple *dst_tuple,
2153                                        const struct sk_buff *skb)
2154 {
2155         const struct nf_conntrack_tuple *src_tuple;
2156         const struct nf_conntrack_tuple_hash *hash;
2157         struct nf_conntrack_tuple srctuple;
2158         enum ip_conntrack_info ctinfo;
2159         struct nf_conn *ct;
2160 
2161         ct = nf_ct_get(skb, &ctinfo);
2162         if (ct) {
2163                 src_tuple = nf_ct_tuple(ct, CTINFO2DIR(ctinfo));
2164                 memcpy(dst_tuple, src_tuple, sizeof(*dst_tuple));
2165                 return true;
2166         }
2167 
2168         if (!nf_ct_get_tuplepr(skb, skb_network_offset(skb),
2169                                NFPROTO_IPV4, dev_net(skb->dev),
2170                                &srctuple))
2171                 return false;
2172 
2173         hash = nf_conntrack_find_get(dev_net(skb->dev),
2174                                      &nf_ct_zone_dflt,
2175                                      &srctuple);
2176         if (!hash)
2177                 return false;
2178 
2179         ct = nf_ct_tuplehash_to_ctrack(hash);
2180         src_tuple = nf_ct_tuple(ct, !hash->tuple.dst.dir);
2181         memcpy(dst_tuple, src_tuple, sizeof(*dst_tuple));
2182         nf_ct_put(ct);
2183 
2184         return true;
2185 }
2186 
2187 /* Bring out ya dead! */
2188 static struct nf_conn *
2189 get_next_corpse(int (*iter)(struct nf_conn *i, void *data),
2190                 void *data, unsigned int *bucket)
2191 {
2192         struct nf_conntrack_tuple_hash *h;
2193         struct nf_conn *ct;
2194         struct hlist_nulls_node *n;
2195         spinlock_t *lockp;
2196 
2197         for (; *bucket < nf_conntrack_htable_size; (*bucket)++) {
2198                 struct hlist_nulls_head *hslot = &nf_conntrack_hash[*bucket];
2199 
2200                 if (hlist_nulls_empty(hslot))
2201                         continue;
2202 
2203                 lockp = &nf_conntrack_locks[*bucket % CONNTRACK_LOCKS];
2204                 local_bh_disable();
2205                 nf_conntrack_lock(lockp);
2206                 hlist_nulls_for_each_entry(h, n, hslot, hnnode) {
2207                         if (NF_CT_DIRECTION(h) != IP_CT_DIR_REPLY)
2208                                 continue;
2209                         /* All nf_conn objects are added to hash table twice, one
2210                          * for original direction tuple, once for the reply tuple.
2211                          *
2212                          * Exception: In the IPS_NAT_CLASH case, only the reply
2213                          * tuple is added (the original tuple already existed for
2214                          * a different object).
2215                          *
2216                          * We only need to call the iterator once for each
2217                          * conntrack, so we just use the 'reply' direction
2218                          * tuple while iterating.
2219                          */
2220                         ct = nf_ct_tuplehash_to_ctrack(h);
2221                         if (iter(ct, data))
2222                                 goto found;
2223                 }
2224                 spin_unlock(lockp);
2225                 local_bh_enable();
2226                 cond_resched();
2227         }
2228 
2229         return NULL;
2230 found:
2231         atomic_inc(&ct->ct_general.use);
2232         spin_unlock(lockp);
2233         local_bh_enable();
2234         return ct;
2235 }
2236 
2237 static void nf_ct_iterate_cleanup(int (*iter)(struct nf_conn *i, void *data),
2238                                   void *data, u32 portid, int report)
2239 {
2240         unsigned int bucket = 0;
2241         struct nf_conn *ct;
2242 
2243         might_sleep();
2244 
2245         mutex_lock(&nf_conntrack_mutex);
2246         while ((ct = get_next_corpse(iter, data, &bucket)) != NULL) {
2247                 /* Time to push up daises... */
2248 
2249                 nf_ct_delete(ct, portid, report);
2250                 nf_ct_put(ct);
2251                 cond_resched();
2252         }
2253         mutex_unlock(&nf_conntrack_mutex);
2254 }
2255 
2256 struct iter_data {
2257         int (*iter)(struct nf_conn *i, void *data);
2258         void *data;
2259         struct net *net;
2260 };
2261 
2262 static int iter_net_only(struct nf_conn *i, void *data)
2263 {
2264         struct iter_data *d = data;
2265 
2266         if (!net_eq(d->net, nf_ct_net(i)))
2267                 return 0;
2268 
2269         return d->iter(i, d->data);
2270 }
2271 
2272 static void
2273 __nf_ct_unconfirmed_destroy(struct net *net)
2274 {
2275         int cpu;
2276 
2277         for_each_possible_cpu(cpu) {
2278                 struct nf_conntrack_tuple_hash *h;
2279                 struct hlist_nulls_node *n;
2280                 struct ct_pcpu *pcpu;
2281 
2282                 pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu);
2283 
2284                 spin_lock_bh(&pcpu->lock);
2285                 hlist_nulls_for_each_entry(h, n, &pcpu->unconfirmed, hnnode) {
2286                         struct nf_conn *ct;
2287 
2288                         ct = nf_ct_tuplehash_to_ctrack(h);
2289 
2290                         /* we cannot call iter() on unconfirmed list, the
2291                          * owning cpu can reallocate ct->ext at any time.
2292                          */
2293                         set_bit(IPS_DYING_BIT, &ct->status);
2294                 }
2295                 spin_unlock_bh(&pcpu->lock);
2296                 cond_resched();
2297         }
2298 }
2299 
2300 void nf_ct_unconfirmed_destroy(struct net *net)
2301 {
2302         struct nf_conntrack_net *cnet = nf_ct_pernet(net);
2303 
2304         might_sleep();
2305 
2306         if (atomic_read(&cnet->count) > 0) {
2307                 __nf_ct_unconfirmed_destroy(net);
2308                 nf_queue_nf_hook_drop(net);
2309                 synchronize_net();
2310         }
2311 }
2312 EXPORT_SYMBOL_GPL(nf_ct_unconfirmed_destroy);
2313 
2314 void nf_ct_iterate_cleanup_net(struct net *net,
2315                                int (*iter)(struct nf_conn *i, void *data),
2316                                void *data, u32 portid, int report)
2317 {
2318         struct nf_conntrack_net *cnet = nf_ct_pernet(net);
2319         struct iter_data d;
2320 
2321         might_sleep();
2322 
2323         if (atomic_read(&cnet->count) == 0)
2324                 return;
2325 
2326         d.iter = iter;
2327         d.data = data;
2328         d.net = net;
2329 
2330         nf_ct_iterate_cleanup(iter_net_only, &d, portid, report);
2331 }
2332 EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup_net);
2333 
2334 /**
2335  * nf_ct_iterate_destroy - destroy unconfirmed conntracks and iterate table
2336  * @iter: callback to invoke for each conntrack
2337  * @data: data to pass to @iter
2338  *
2339  * Like nf_ct_iterate_cleanup, but first marks conntracks on the
2340  * unconfirmed list as dying (so they will not be inserted into
2341  * main table).
2342  *
2343  * Can only be called in module exit path.
2344  */
2345 void
2346 nf_ct_iterate_destroy(int (*iter)(struct nf_conn *i, void *data), void *data)
2347 {
2348         struct net *net;
2349 
2350         down_read(&net_rwsem);
2351         for_each_net(net) {
2352                 struct nf_conntrack_net *cnet = nf_ct_pernet(net);
2353 
2354                 if (atomic_read(&cnet->count) == 0)
2355                         continue;
2356                 __nf_ct_unconfirmed_destroy(net);
2357                 nf_queue_nf_hook_drop(net);
2358         }
2359         up_read(&net_rwsem);
2360 
2361         /* Need to wait for netns cleanup worker to finish, if its
2362          * running -- it might have deleted a net namespace from
2363          * the global list, so our __nf_ct_unconfirmed_destroy() might
2364          * not have affected all namespaces.
2365          */
2366         net_ns_barrier();
2367 
2368         /* a conntrack could have been unlinked from unconfirmed list
2369          * before we grabbed pcpu lock in __nf_ct_unconfirmed_destroy().
2370          * This makes sure its inserted into conntrack table.
2371          */
2372         synchronize_net();
2373 
2374         nf_ct_iterate_cleanup(iter, data, 0, 0);
2375 }
2376 EXPORT_SYMBOL_GPL(nf_ct_iterate_destroy);
2377 
2378 static int kill_all(struct nf_conn *i, void *data)
2379 {
2380         return net_eq(nf_ct_net(i), data);
2381 }
2382 
2383 void nf_conntrack_cleanup_start(void)
2384 {
2385         conntrack_gc_work.exiting = true;
2386         RCU_INIT_POINTER(ip_ct_attach, NULL);
2387 }
2388 
2389 void nf_conntrack_cleanup_end(void)
2390 {
2391         RCU_INIT_POINTER(nf_ct_hook, NULL);
2392         cancel_delayed_work_sync(&conntrack_gc_work.dwork);
2393         kvfree(nf_conntrack_hash);
2394 
2395         nf_conntrack_proto_fini();
2396         nf_conntrack_seqadj_fini();
2397         nf_conntrack_labels_fini();
2398         nf_conntrack_helper_fini();
2399         nf_conntrack_timeout_fini();
2400         nf_conntrack_ecache_fini();
2401         nf_conntrack_tstamp_fini();
2402         nf_conntrack_acct_fini();
2403         nf_conntrack_expect_fini();
2404 
2405         kmem_cache_destroy(nf_conntrack_cachep);
2406 }
2407 
2408 /*
2409  * Mishearing the voices in his head, our hero wonders how he's
2410  * supposed to kill the mall.
2411  */
2412 void nf_conntrack_cleanup_net(struct net *net)
2413 {
2414         LIST_HEAD(single);
2415 
2416         list_add(&net->exit_list, &single);
2417         nf_conntrack_cleanup_net_list(&single);
2418 }
2419 
2420 void nf_conntrack_cleanup_net_list(struct list_head *net_exit_list)
2421 {
2422         int busy;
2423         struct net *net;
2424 
2425         /*
2426          * This makes sure all current packets have passed through
2427          *  netfilter framework.  Roll on, two-stage module
2428          *  delete...
2429          */
2430         synchronize_net();
2431 i_see_dead_people:
2432         busy = 0;
2433         list_for_each_entry(net, net_exit_list, exit_list) {
2434                 struct nf_conntrack_net *cnet = nf_ct_pernet(net);
2435 
2436                 nf_ct_iterate_cleanup(kill_all, net, 0, 0);
2437                 if (atomic_read(&cnet->count) != 0)
2438                         busy = 1;
2439         }
2440         if (busy) {
2441                 schedule();
2442                 goto i_see_dead_people;
2443         }
2444 
2445         list_for_each_entry(net, net_exit_list, exit_list) {
2446                 nf_conntrack_ecache_pernet_fini(net);
2447                 nf_conntrack_expect_pernet_fini(net);
2448                 free_percpu(net->ct.stat);
2449                 free_percpu(net->ct.pcpu_lists);
2450         }
2451 }
2452 
2453 void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)
2454 {
2455         struct hlist_nulls_head *hash;
2456         unsigned int nr_slots, i;
2457 
2458         if (*sizep > (UINT_MAX / sizeof(struct hlist_nulls_head)))
2459                 return NULL;
2460 
2461         BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));
2462         nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));
2463 
2464         hash = kvcalloc(nr_slots, sizeof(struct hlist_nulls_head), GFP_KERNEL);
2465 
2466         if (hash && nulls)
2467                 for (i = 0; i < nr_slots; i++)
2468                         INIT_HLIST_NULLS_HEAD(&hash[i], i);
2469 
2470         return hash;
2471 }
2472 EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable);
2473 
2474 int nf_conntrack_hash_resize(unsigned int hashsize)
2475 {
2476         int i, bucket;
2477         unsigned int old_size;
2478         struct hlist_nulls_head *hash, *old_hash;
2479         struct nf_conntrack_tuple_hash *h;
2480         struct nf_conn *ct;
2481 
2482         if (!hashsize)
2483                 return -EINVAL;
2484 
2485         hash = nf_ct_alloc_hashtable(&hashsize, 1);
2486         if (!hash)
2487                 return -ENOMEM;
2488 
2489         mutex_lock(&nf_conntrack_mutex);
2490         old_size = nf_conntrack_htable_size;
2491         if (old_size == hashsize) {
2492                 mutex_unlock(&nf_conntrack_mutex);
2493                 kvfree(hash);
2494                 return 0;
2495         }
2496 
2497         local_bh_disable();
2498         nf_conntrack_all_lock();
2499         write_seqcount_begin(&nf_conntrack_generation);
2500 
2501         /* Lookups in the old hash might happen in parallel, which means we
2502          * might get false negatives during connection lookup. New connections
2503          * created because of a false negative won't make it into the hash
2504          * though since that required taking the locks.
2505          */
2506 
2507         for (i = 0; i < nf_conntrack_htable_size; i++) {
2508                 while (!hlist_nulls_empty(&nf_conntrack_hash[i])) {
2509                         h = hlist_nulls_entry(nf_conntrack_hash[i].first,
2510                                               struct nf_conntrack_tuple_hash, hnnode);
2511                         ct = nf_ct_tuplehash_to_ctrack(h);
2512                         hlist_nulls_del_rcu(&h->hnnode);
2513                         bucket = __hash_conntrack(nf_ct_net(ct),
2514                                                   &h->tuple, hashsize);
2515                         hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);
2516                 }
2517         }
2518         old_size = nf_conntrack_htable_size;
2519         old_hash = nf_conntrack_hash;
2520 
2521         nf_conntrack_hash = hash;
2522         nf_conntrack_htable_size = hashsize;
2523 
2524         write_seqcount_end(&nf_conntrack_generation);
2525         nf_conntrack_all_unlock();
2526         local_bh_enable();
2527 
2528         mutex_unlock(&nf_conntrack_mutex);
2529 
2530         synchronize_net();
2531         kvfree(old_hash);
2532         return 0;
2533 }
2534 
2535 int nf_conntrack_set_hashsize(const char *val, const struct kernel_param *kp)
2536 {
2537         unsigned int hashsize;
2538         int rc;
2539 
2540         if (current->nsproxy->net_ns != &init_net)
2541                 return -EOPNOTSUPP;
2542 
2543         /* On boot, we can set this without any fancy locking. */
2544         if (!nf_conntrack_hash)
2545                 return param_set_uint(val, kp);
2546 
2547         rc = kstrtouint(val, 0, &hashsize);
2548         if (rc)
2549                 return rc;
2550 
2551         return nf_conntrack_hash_resize(hashsize);
2552 }
2553 
2554 static __always_inline unsigned int total_extension_size(void)
2555 {
2556         /* remember to add new extensions below */
2557         BUILD_BUG_ON(NF_CT_EXT_NUM > 9);
2558 
2559         return sizeof(struct nf_ct_ext) +
2560                sizeof(struct nf_conn_help)
2561 #if IS_ENABLED(CONFIG_NF_NAT)
2562                 + sizeof(struct nf_conn_nat)
2563 #endif
2564                 + sizeof(struct nf_conn_seqadj)
2565                 + sizeof(struct nf_conn_acct)
2566 #ifdef CONFIG_NF_CONNTRACK_EVENTS
2567                 + sizeof(struct nf_conntrack_ecache)
2568 #endif
2569 #ifdef CONFIG_NF_CONNTRACK_TIMESTAMP
2570                 + sizeof(struct nf_conn_tstamp)
2571 #endif
2572 #ifdef CONFIG_NF_CONNTRACK_TIMEOUT
2573                 + sizeof(struct nf_conn_timeout)
2574 #endif
2575 #ifdef CONFIG_NF_CONNTRACK_LABELS
2576                 + sizeof(struct nf_conn_labels)
2577 #endif
2578 #if IS_ENABLED(CONFIG_NETFILTER_SYNPROXY)
2579                 + sizeof(struct nf_conn_synproxy)
2580 #endif
2581         ;
2582 };
2583 
2584 int nf_conntrack_init_start(void)
2585 {
2586         unsigned long nr_pages = totalram_pages();
2587         int max_factor = 8;
2588         int ret = -ENOMEM;
2589         int i;
2590 
2591         /* struct nf_ct_ext uses u8 to store offsets/size */
2592         BUILD_BUG_ON(total_extension_size() > 255u);
2593 
2594         seqcount_spinlock_init(&nf_conntrack_generation,
2595                                &nf_conntrack_locks_all_lock);
2596 
2597         for (i = 0; i < CONNTRACK_LOCKS; i++)
2598                 spin_lock_init(&nf_conntrack_locks[i]);
2599 
2600         if (!nf_conntrack_htable_size) {
2601                 /* Idea from tcp.c: use 1/16384 of memory.
2602                  * On i386: 32MB machine has 512 buckets.
2603                  * >= 1GB machines have 16384 buckets.
2604                  * >= 4GB machines have 65536 buckets.
2605                  */
2606                 nf_conntrack_htable_size
2607                         = (((nr_pages << PAGE_SHIFT) / 16384)
2608                            / sizeof(struct hlist_head));
2609                 if (nr_pages > (4 * (1024 * 1024 * 1024 / PAGE_SIZE)))
2610                         nf_conntrack_htable_size = 65536;
2611                 else if (nr_pages > (1024 * 1024 * 1024 / PAGE_SIZE))
2612                         nf_conntrack_htable_size = 16384;
2613                 if (nf_conntrack_htable_size < 32)
2614                         nf_conntrack_htable_size = 32;
2615 
2616                 /* Use a max. factor of four by default to get the same max as
2617                  * with the old struct list_heads. When a table size is given
2618                  * we use the old value of 8 to avoid reducing the max.
2619                  * entries. */
2620                 max_factor = 4;
2621         }
2622 
2623         nf_conntrack_hash = nf_ct_alloc_hashtable(&nf_conntrack_htable_size, 1);
2624         if (!nf_conntrack_hash)
2625                 return -ENOMEM;
2626 
2627         nf_conntrack_max = max_factor * nf_conntrack_htable_size;
2628 
2629         nf_conntrack_cachep = kmem_cache_create("nf_conntrack",
2630                                                 sizeof(struct nf_conn),
2631                                                 NFCT_INFOMASK + 1,
2632                                                 SLAB_TYPESAFE_BY_RCU | SLAB_HWCACHE_ALIGN, NULL);
2633         if (!nf_conntrack_cachep)
2634                 goto err_cachep;
2635 
2636         ret = nf_conntrack_expect_init();
2637         if (ret < 0)
2638                 goto err_expect;
2639 
2640         ret = nf_conntrack_acct_init();
2641         if (ret < 0)
2642                 goto err_acct;
2643 
2644         ret = nf_conntrack_tstamp_init();
2645         if (ret < 0)
2646                 goto err_tstamp;
2647 
2648         ret = nf_conntrack_ecache_init();
2649         if (ret < 0)
2650                 goto err_ecache;
2651 
2652         ret = nf_conntrack_timeout_init();
2653         if (ret < 0)
2654                 goto err_timeout;
2655 
2656         ret = nf_conntrack_helper_init();
2657         if (ret < 0)
2658                 goto err_helper;
2659 
2660         ret = nf_conntrack_labels_init();
2661         if (ret < 0)
2662                 goto err_labels;
2663 
2664         ret = nf_conntrack_seqadj_init();
2665         if (ret < 0)
2666                 goto err_seqadj;
2667 
2668         ret = nf_conntrack_proto_init();
2669         if (ret < 0)
2670                 goto err_proto;
2671 
2672         conntrack_gc_work_init(&conntrack_gc_work);
2673         queue_delayed_work(system_power_efficient_wq, &conntrack_gc_work.dwork, HZ);
2674 
2675         return 0;
2676 
2677 err_proto:
2678         nf_conntrack_seqadj_fini();
2679 err_seqadj:
2680         nf_conntrack_labels_fini();
2681 err_labels:
2682         nf_conntrack_helper_fini();
2683 err_helper:
2684         nf_conntrack_timeout_fini();
2685 err_timeout:
2686         nf_conntrack_ecache_fini();
2687 err_ecache:
2688         nf_conntrack_tstamp_fini();
2689 err_tstamp:
2690         nf_conntrack_acct_fini();
2691 err_acct:
2692         nf_conntrack_expect_fini();
2693 err_expect:
2694         kmem_cache_destroy(nf_conntrack_cachep);
2695 err_cachep:
2696         kvfree(nf_conntrack_hash);
2697         return ret;
2698 }
2699 
2700 static struct nf_ct_hook nf_conntrack_hook = {
2701         .update         = nf_conntrack_update,
2702         .destroy        = destroy_conntrack,
2703         .get_tuple_skb  = nf_conntrack_get_tuple_skb,
2704 };
2705 
2706 void nf_conntrack_init_end(void)
2707 {
2708         /* For use by REJECT target */
2709         RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach);
2710         RCU_INIT_POINTER(nf_ct_hook, &nf_conntrack_hook);
2711 }
2712 
2713 /*
2714  * We need to use special "null" values, not used in hash table
2715  */
2716 #define UNCONFIRMED_NULLS_VAL   ((1<<30)+0)
2717 #define DYING_NULLS_VAL         ((1<<30)+1)
2718 
2719 int nf_conntrack_init_net(struct net *net)
2720 {
2721         struct nf_conntrack_net *cnet = nf_ct_pernet(net);
2722         int ret = -ENOMEM;
2723         int cpu;
2724 
2725         BUILD_BUG_ON(IP_CT_UNTRACKED == IP_CT_NUMBER);
2726         BUILD_BUG_ON_NOT_POWER_OF_2(CONNTRACK_LOCKS);
2727         atomic_set(&cnet->count, 0);
2728 
2729         net->ct.pcpu_lists = alloc_percpu(struct ct_pcpu);
2730         if (!net->ct.pcpu_lists)
2731                 goto err_stat;
2732 
2733         for_each_possible_cpu(cpu) {
2734                 struct ct_pcpu *pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu);
2735 
2736                 spin_lock_init(&pcpu->lock);
2737                 INIT_HLIST_NULLS_HEAD(&pcpu->unconfirmed, UNCONFIRMED_NULLS_VAL);
2738                 INIT_HLIST_NULLS_HEAD(&pcpu->dying, DYING_NULLS_VAL);
2739         }
2740 
2741         net->ct.stat = alloc_percpu(struct ip_conntrack_stat);
2742         if (!net->ct.stat)
2743                 goto err_pcpu_lists;
2744 
2745         ret = nf_conntrack_expect_pernet_init(net);
2746         if (ret < 0)
2747                 goto err_expect;
2748 
2749         nf_conntrack_acct_pernet_init(net);
2750         nf_conntrack_tstamp_pernet_init(net);
2751         nf_conntrack_ecache_pernet_init(net);
2752         nf_conntrack_helper_pernet_init(net);
2753         nf_conntrack_proto_pernet_init(net);
2754 
2755         return 0;
2756 
2757 err_expect:
2758         free_percpu(net->ct.stat);
2759 err_pcpu_lists:
2760         free_percpu(net->ct.pcpu_lists);
2761 err_stat:
2762         return ret;
2763 }
2764 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp