~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/netfilter/nf_conntrack_core.c

Version: ~ [ linux-5.16-rc3 ] ~ [ linux-5.15.5 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.82 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.162 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.218 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.256 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.291 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.293 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* Connection state tracking for netfilter.  This is separated from,
  2    but required by, the NAT layer; it can also be used by an iptables
  3    extension. */
  4 
  5 /* (C) 1999-2001 Paul `Rusty' Russell
  6  * (C) 2002-2006 Netfilter Core Team <coreteam@netfilter.org>
  7  * (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org>
  8  * (C) 2005-2012 Patrick McHardy <kaber@trash.net>
  9  *
 10  * This program is free software; you can redistribute it and/or modify
 11  * it under the terms of the GNU General Public License version 2 as
 12  * published by the Free Software Foundation.
 13  */
 14 
 15 #include <linux/types.h>
 16 #include <linux/netfilter.h>
 17 #include <linux/module.h>
 18 #include <linux/sched.h>
 19 #include <linux/skbuff.h>
 20 #include <linux/proc_fs.h>
 21 #include <linux/vmalloc.h>
 22 #include <linux/stddef.h>
 23 #include <linux/slab.h>
 24 #include <linux/random.h>
 25 #include <linux/jhash.h>
 26 #include <linux/err.h>
 27 #include <linux/percpu.h>
 28 #include <linux/moduleparam.h>
 29 #include <linux/notifier.h>
 30 #include <linux/kernel.h>
 31 #include <linux/netdevice.h>
 32 #include <linux/socket.h>
 33 #include <linux/mm.h>
 34 #include <linux/nsproxy.h>
 35 #include <linux/rculist_nulls.h>
 36 
 37 #include <net/netfilter/nf_conntrack.h>
 38 #include <net/netfilter/nf_conntrack_l3proto.h>
 39 #include <net/netfilter/nf_conntrack_l4proto.h>
 40 #include <net/netfilter/nf_conntrack_expect.h>
 41 #include <net/netfilter/nf_conntrack_helper.h>
 42 #include <net/netfilter/nf_conntrack_seqadj.h>
 43 #include <net/netfilter/nf_conntrack_core.h>
 44 #include <net/netfilter/nf_conntrack_extend.h>
 45 #include <net/netfilter/nf_conntrack_acct.h>
 46 #include <net/netfilter/nf_conntrack_ecache.h>
 47 #include <net/netfilter/nf_conntrack_zones.h>
 48 #include <net/netfilter/nf_conntrack_timestamp.h>
 49 #include <net/netfilter/nf_conntrack_timeout.h>
 50 #include <net/netfilter/nf_conntrack_labels.h>
 51 #include <net/netfilter/nf_conntrack_synproxy.h>
 52 #include <net/netfilter/nf_nat.h>
 53 #include <net/netfilter/nf_nat_core.h>
 54 #include <net/netfilter/nf_nat_helper.h>
 55 
 56 #define NF_CONNTRACK_VERSION    "0.5.0"
 57 
 58 int (*nfnetlink_parse_nat_setup_hook)(struct nf_conn *ct,
 59                                       enum nf_nat_manip_type manip,
 60                                       const struct nlattr *attr) __read_mostly;
 61 EXPORT_SYMBOL_GPL(nfnetlink_parse_nat_setup_hook);
 62 
 63 __cacheline_aligned_in_smp spinlock_t nf_conntrack_locks[CONNTRACK_LOCKS];
 64 EXPORT_SYMBOL_GPL(nf_conntrack_locks);
 65 
 66 __cacheline_aligned_in_smp DEFINE_SPINLOCK(nf_conntrack_expect_lock);
 67 EXPORT_SYMBOL_GPL(nf_conntrack_expect_lock);
 68 
 69 static void nf_conntrack_double_unlock(unsigned int h1, unsigned int h2)
 70 {
 71         h1 %= CONNTRACK_LOCKS;
 72         h2 %= CONNTRACK_LOCKS;
 73         spin_unlock(&nf_conntrack_locks[h1]);
 74         if (h1 != h2)
 75                 spin_unlock(&nf_conntrack_locks[h2]);
 76 }
 77 
 78 /* return true if we need to recompute hashes (in case hash table was resized) */
 79 static bool nf_conntrack_double_lock(struct net *net, unsigned int h1,
 80                                      unsigned int h2, unsigned int sequence)
 81 {
 82         h1 %= CONNTRACK_LOCKS;
 83         h2 %= CONNTRACK_LOCKS;
 84         if (h1 <= h2) {
 85                 spin_lock(&nf_conntrack_locks[h1]);
 86                 if (h1 != h2)
 87                         spin_lock_nested(&nf_conntrack_locks[h2],
 88                                          SINGLE_DEPTH_NESTING);
 89         } else {
 90                 spin_lock(&nf_conntrack_locks[h2]);
 91                 spin_lock_nested(&nf_conntrack_locks[h1],
 92                                  SINGLE_DEPTH_NESTING);
 93         }
 94         if (read_seqcount_retry(&net->ct.generation, sequence)) {
 95                 nf_conntrack_double_unlock(h1, h2);
 96                 return true;
 97         }
 98         return false;
 99 }
100 
101 static void nf_conntrack_all_lock(void)
102 {
103         int i;
104 
105         for (i = 0; i < CONNTRACK_LOCKS; i++)
106                 spin_lock_nested(&nf_conntrack_locks[i], i);
107 }
108 
109 static void nf_conntrack_all_unlock(void)
110 {
111         int i;
112 
113         for (i = 0; i < CONNTRACK_LOCKS; i++)
114                 spin_unlock(&nf_conntrack_locks[i]);
115 }
116 
117 unsigned int nf_conntrack_htable_size __read_mostly;
118 EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
119 
120 unsigned int nf_conntrack_max __read_mostly;
121 EXPORT_SYMBOL_GPL(nf_conntrack_max);
122 
123 DEFINE_PER_CPU(struct nf_conn, nf_conntrack_untracked);
124 EXPORT_PER_CPU_SYMBOL(nf_conntrack_untracked);
125 
126 unsigned int nf_conntrack_hash_rnd __read_mostly;
127 EXPORT_SYMBOL_GPL(nf_conntrack_hash_rnd);
128 
129 static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple, u16 zone)
130 {
131         unsigned int n;
132 
133         /* The direction must be ignored, so we hash everything up to the
134          * destination ports (which is a multiple of 4) and treat the last
135          * three bytes manually.
136          */
137         n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32);
138         return jhash2((u32 *)tuple, n, zone ^ nf_conntrack_hash_rnd ^
139                       (((__force __u16)tuple->dst.u.all << 16) |
140                       tuple->dst.protonum));
141 }
142 
143 static u32 __hash_bucket(u32 hash, unsigned int size)
144 {
145         return reciprocal_scale(hash, size);
146 }
147 
148 static u32 hash_bucket(u32 hash, const struct net *net)
149 {
150         return __hash_bucket(hash, net->ct.htable_size);
151 }
152 
153 static u_int32_t __hash_conntrack(const struct nf_conntrack_tuple *tuple,
154                                   u16 zone, unsigned int size)
155 {
156         return __hash_bucket(hash_conntrack_raw(tuple, zone), size);
157 }
158 
159 static inline u_int32_t hash_conntrack(const struct net *net, u16 zone,
160                                        const struct nf_conntrack_tuple *tuple)
161 {
162         return __hash_conntrack(tuple, zone, net->ct.htable_size);
163 }
164 
165 bool
166 nf_ct_get_tuple(const struct sk_buff *skb,
167                 unsigned int nhoff,
168                 unsigned int dataoff,
169                 u_int16_t l3num,
170                 u_int8_t protonum,
171                 struct nf_conntrack_tuple *tuple,
172                 const struct nf_conntrack_l3proto *l3proto,
173                 const struct nf_conntrack_l4proto *l4proto)
174 {
175         memset(tuple, 0, sizeof(*tuple));
176 
177         tuple->src.l3num = l3num;
178         if (l3proto->pkt_to_tuple(skb, nhoff, tuple) == 0)
179                 return false;
180 
181         tuple->dst.protonum = protonum;
182         tuple->dst.dir = IP_CT_DIR_ORIGINAL;
183 
184         return l4proto->pkt_to_tuple(skb, dataoff, tuple);
185 }
186 EXPORT_SYMBOL_GPL(nf_ct_get_tuple);
187 
188 bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,
189                        u_int16_t l3num, struct nf_conntrack_tuple *tuple)
190 {
191         struct nf_conntrack_l3proto *l3proto;
192         struct nf_conntrack_l4proto *l4proto;
193         unsigned int protoff;
194         u_int8_t protonum;
195         int ret;
196 
197         rcu_read_lock();
198 
199         l3proto = __nf_ct_l3proto_find(l3num);
200         ret = l3proto->get_l4proto(skb, nhoff, &protoff, &protonum);
201         if (ret != NF_ACCEPT) {
202                 rcu_read_unlock();
203                 return false;
204         }
205 
206         l4proto = __nf_ct_l4proto_find(l3num, protonum);
207 
208         ret = nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, tuple,
209                               l3proto, l4proto);
210 
211         rcu_read_unlock();
212         return ret;
213 }
214 EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr);
215 
216 bool
217 nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,
218                    const struct nf_conntrack_tuple *orig,
219                    const struct nf_conntrack_l3proto *l3proto,
220                    const struct nf_conntrack_l4proto *l4proto)
221 {
222         memset(inverse, 0, sizeof(*inverse));
223 
224         inverse->src.l3num = orig->src.l3num;
225         if (l3proto->invert_tuple(inverse, orig) == 0)
226                 return false;
227 
228         inverse->dst.dir = !orig->dst.dir;
229 
230         inverse->dst.protonum = orig->dst.protonum;
231         return l4proto->invert_tuple(inverse, orig);
232 }
233 EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);
234 
235 static void
236 clean_from_lists(struct nf_conn *ct)
237 {
238         pr_debug("clean_from_lists(%p)\n", ct);
239         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
240         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);
241 
242         /* Destroy all pending expectations */
243         nf_ct_remove_expectations(ct);
244 }
245 
246 /* must be called with local_bh_disable */
247 static void nf_ct_add_to_dying_list(struct nf_conn *ct)
248 {
249         struct ct_pcpu *pcpu;
250 
251         /* add this conntrack to the (per cpu) dying list */
252         ct->cpu = smp_processor_id();
253         pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
254 
255         spin_lock(&pcpu->lock);
256         hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
257                              &pcpu->dying);
258         spin_unlock(&pcpu->lock);
259 }
260 
261 /* must be called with local_bh_disable */
262 static void nf_ct_add_to_unconfirmed_list(struct nf_conn *ct)
263 {
264         struct ct_pcpu *pcpu;
265 
266         /* add this conntrack to the (per cpu) unconfirmed list */
267         ct->cpu = smp_processor_id();
268         pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
269 
270         spin_lock(&pcpu->lock);
271         hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
272                              &pcpu->unconfirmed);
273         spin_unlock(&pcpu->lock);
274 }
275 
276 /* must be called with local_bh_disable */
277 static void nf_ct_del_from_dying_or_unconfirmed_list(struct nf_conn *ct)
278 {
279         struct ct_pcpu *pcpu;
280 
281         /* We overload first tuple to link into unconfirmed or dying list.*/
282         pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
283 
284         spin_lock(&pcpu->lock);
285         BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode));
286         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
287         spin_unlock(&pcpu->lock);
288 }
289 
290 /* Released via destroy_conntrack() */
291 struct nf_conn *nf_ct_tmpl_alloc(struct net *net, u16 zone, gfp_t flags)
292 {
293         struct nf_conn *tmpl;
294 
295         tmpl = kzalloc(sizeof(*tmpl), flags);
296         if (tmpl == NULL)
297                 return NULL;
298 
299         tmpl->status = IPS_TEMPLATE;
300         write_pnet(&tmpl->ct_net, net);
301 
302 #ifdef CONFIG_NF_CONNTRACK_ZONES
303         if (zone) {
304                 struct nf_conntrack_zone *nf_ct_zone;
305 
306                 nf_ct_zone = nf_ct_ext_add(tmpl, NF_CT_EXT_ZONE, flags);
307                 if (!nf_ct_zone)
308                         goto out_free;
309                 nf_ct_zone->id = zone;
310         }
311 #endif
312         atomic_set(&tmpl->ct_general.use, 0);
313 
314         return tmpl;
315 #ifdef CONFIG_NF_CONNTRACK_ZONES
316 out_free:
317         kfree(tmpl);
318         return NULL;
319 #endif
320 }
321 EXPORT_SYMBOL_GPL(nf_ct_tmpl_alloc);
322 
323 void nf_ct_tmpl_free(struct nf_conn *tmpl)
324 {
325         nf_ct_ext_destroy(tmpl);
326         nf_ct_ext_free(tmpl);
327         kfree(tmpl);
328 }
329 EXPORT_SYMBOL_GPL(nf_ct_tmpl_free);
330 
331 static void
332 destroy_conntrack(struct nf_conntrack *nfct)
333 {
334         struct nf_conn *ct = (struct nf_conn *)nfct;
335         struct net *net = nf_ct_net(ct);
336         struct nf_conntrack_l4proto *l4proto;
337 
338         pr_debug("destroy_conntrack(%p)\n", ct);
339         NF_CT_ASSERT(atomic_read(&nfct->use) == 0);
340         NF_CT_ASSERT(!timer_pending(&ct->timeout));
341 
342         if (unlikely(nf_ct_is_template(ct))) {
343                 nf_ct_tmpl_free(ct);
344                 return;
345         }
346         rcu_read_lock();
347         l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));
348         if (l4proto && l4proto->destroy)
349                 l4proto->destroy(ct);
350 
351         rcu_read_unlock();
352 
353         local_bh_disable();
354         /* Expectations will have been removed in clean_from_lists,
355          * except TFTP can create an expectation on the first packet,
356          * before connection is in the list, so we need to clean here,
357          * too.
358          */
359         nf_ct_remove_expectations(ct);
360 
361         nf_ct_del_from_dying_or_unconfirmed_list(ct);
362 
363         NF_CT_STAT_INC(net, delete);
364         local_bh_enable();
365 
366         if (ct->master)
367                 nf_ct_put(ct->master);
368 
369         pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct);
370         nf_conntrack_free(ct);
371 }
372 
373 static void nf_ct_delete_from_lists(struct nf_conn *ct)
374 {
375         struct net *net = nf_ct_net(ct);
376         unsigned int hash, reply_hash;
377         u16 zone = nf_ct_zone(ct);
378         unsigned int sequence;
379 
380         nf_ct_helper_destroy(ct);
381 
382         local_bh_disable();
383         do {
384                 sequence = read_seqcount_begin(&net->ct.generation);
385                 hash = hash_conntrack(net, zone,
386                                       &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
387                 reply_hash = hash_conntrack(net, zone,
388                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
389         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
390 
391         clean_from_lists(ct);
392         nf_conntrack_double_unlock(hash, reply_hash);
393 
394         nf_ct_add_to_dying_list(ct);
395 
396         NF_CT_STAT_INC(net, delete_list);
397         local_bh_enable();
398 }
399 
400 bool nf_ct_delete(struct nf_conn *ct, u32 portid, int report)
401 {
402         struct nf_conn_tstamp *tstamp;
403 
404         tstamp = nf_conn_tstamp_find(ct);
405         if (tstamp && tstamp->stop == 0)
406                 tstamp->stop = ktime_get_real_ns();
407 
408         if (nf_ct_is_dying(ct))
409                 goto delete;
410 
411         if (nf_conntrack_event_report(IPCT_DESTROY, ct,
412                                     portid, report) < 0) {
413                 /* destroy event was not delivered */
414                 nf_ct_delete_from_lists(ct);
415                 nf_conntrack_ecache_delayed_work(nf_ct_net(ct));
416                 return false;
417         }
418 
419         nf_conntrack_ecache_work(nf_ct_net(ct));
420         set_bit(IPS_DYING_BIT, &ct->status);
421  delete:
422         nf_ct_delete_from_lists(ct);
423         nf_ct_put(ct);
424         return true;
425 }
426 EXPORT_SYMBOL_GPL(nf_ct_delete);
427 
428 static void death_by_timeout(unsigned long ul_conntrack)
429 {
430         nf_ct_delete((struct nf_conn *)ul_conntrack, 0, 0);
431 }
432 
433 static inline bool
434 nf_ct_key_equal(struct nf_conntrack_tuple_hash *h,
435                         const struct nf_conntrack_tuple *tuple,
436                         u16 zone)
437 {
438         struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
439 
440         /* A conntrack can be recreated with the equal tuple,
441          * so we need to check that the conntrack is confirmed
442          */
443         return nf_ct_tuple_equal(tuple, &h->tuple) &&
444                 nf_ct_zone(ct) == zone &&
445                 nf_ct_is_confirmed(ct);
446 }
447 
448 /*
449  * Warning :
450  * - Caller must take a reference on returned object
451  *   and recheck nf_ct_tuple_equal(tuple, &h->tuple)
452  */
453 static struct nf_conntrack_tuple_hash *
454 ____nf_conntrack_find(struct net *net, u16 zone,
455                       const struct nf_conntrack_tuple *tuple, u32 hash)
456 {
457         struct nf_conntrack_tuple_hash *h;
458         struct hlist_nulls_node *n;
459         unsigned int bucket = hash_bucket(hash, net);
460 
461         /* Disable BHs the entire time since we normally need to disable them
462          * at least once for the stats anyway.
463          */
464         local_bh_disable();
465 begin:
466         hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[bucket], hnnode) {
467                 if (nf_ct_key_equal(h, tuple, zone)) {
468                         NF_CT_STAT_INC(net, found);
469                         local_bh_enable();
470                         return h;
471                 }
472                 NF_CT_STAT_INC(net, searched);
473         }
474         /*
475          * if the nulls value we got at the end of this lookup is
476          * not the expected one, we must restart lookup.
477          * We probably met an item that was moved to another chain.
478          */
479         if (get_nulls_value(n) != bucket) {
480                 NF_CT_STAT_INC(net, search_restart);
481                 goto begin;
482         }
483         local_bh_enable();
484 
485         return NULL;
486 }
487 
488 /* Find a connection corresponding to a tuple. */
489 static struct nf_conntrack_tuple_hash *
490 __nf_conntrack_find_get(struct net *net, u16 zone,
491                         const struct nf_conntrack_tuple *tuple, u32 hash)
492 {
493         struct nf_conntrack_tuple_hash *h;
494         struct nf_conn *ct;
495 
496         rcu_read_lock();
497 begin:
498         h = ____nf_conntrack_find(net, zone, tuple, hash);
499         if (h) {
500                 ct = nf_ct_tuplehash_to_ctrack(h);
501                 if (unlikely(nf_ct_is_dying(ct) ||
502                              !atomic_inc_not_zero(&ct->ct_general.use)))
503                         h = NULL;
504                 else {
505                         if (unlikely(!nf_ct_key_equal(h, tuple, zone))) {
506                                 nf_ct_put(ct);
507                                 goto begin;
508                         }
509                 }
510         }
511         rcu_read_unlock();
512 
513         return h;
514 }
515 
516 struct nf_conntrack_tuple_hash *
517 nf_conntrack_find_get(struct net *net, u16 zone,
518                       const struct nf_conntrack_tuple *tuple)
519 {
520         return __nf_conntrack_find_get(net, zone, tuple,
521                                        hash_conntrack_raw(tuple, zone));
522 }
523 EXPORT_SYMBOL_GPL(nf_conntrack_find_get);
524 
525 static void __nf_conntrack_hash_insert(struct nf_conn *ct,
526                                        unsigned int hash,
527                                        unsigned int reply_hash)
528 {
529         struct net *net = nf_ct_net(ct);
530 
531         hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
532                            &net->ct.hash[hash]);
533         hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
534                            &net->ct.hash[reply_hash]);
535 }
536 
537 int
538 nf_conntrack_hash_check_insert(struct nf_conn *ct)
539 {
540         struct net *net = nf_ct_net(ct);
541         unsigned int hash, reply_hash;
542         struct nf_conntrack_tuple_hash *h;
543         struct hlist_nulls_node *n;
544         u16 zone;
545         unsigned int sequence;
546 
547         zone = nf_ct_zone(ct);
548 
549         local_bh_disable();
550         do {
551                 sequence = read_seqcount_begin(&net->ct.generation);
552                 hash = hash_conntrack(net, zone,
553                                       &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
554                 reply_hash = hash_conntrack(net, zone,
555                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
556         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
557 
558         /* See if there's one in the list already, including reverse */
559         hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)
560                 if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
561                                       &h->tuple) &&
562                     zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
563                         goto out;
564         hlist_nulls_for_each_entry(h, n, &net->ct.hash[reply_hash], hnnode)
565                 if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,
566                                       &h->tuple) &&
567                     zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
568                         goto out;
569 
570         add_timer(&ct->timeout);
571         smp_wmb();
572         /* The caller holds a reference to this object */
573         atomic_set(&ct->ct_general.use, 2);
574         __nf_conntrack_hash_insert(ct, hash, reply_hash);
575         nf_conntrack_double_unlock(hash, reply_hash);
576         NF_CT_STAT_INC(net, insert);
577         local_bh_enable();
578         return 0;
579 
580 out:
581         nf_conntrack_double_unlock(hash, reply_hash);
582         NF_CT_STAT_INC(net, insert_failed);
583         local_bh_enable();
584         return -EEXIST;
585 }
586 EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert);
587 
588 /* Confirm a connection given skb; places it in hash table */
589 int
590 __nf_conntrack_confirm(struct sk_buff *skb)
591 {
592         unsigned int hash, reply_hash;
593         struct nf_conntrack_tuple_hash *h;
594         struct nf_conn *ct;
595         struct nf_conn_help *help;
596         struct nf_conn_tstamp *tstamp;
597         struct hlist_nulls_node *n;
598         enum ip_conntrack_info ctinfo;
599         struct net *net;
600         u16 zone;
601         unsigned int sequence;
602 
603         ct = nf_ct_get(skb, &ctinfo);
604         net = nf_ct_net(ct);
605 
606         /* ipt_REJECT uses nf_conntrack_attach to attach related
607            ICMP/TCP RST packets in other direction.  Actual packet
608            which created connection will be IP_CT_NEW or for an
609            expected connection, IP_CT_RELATED. */
610         if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)
611                 return NF_ACCEPT;
612 
613         zone = nf_ct_zone(ct);
614         local_bh_disable();
615 
616         do {
617                 sequence = read_seqcount_begin(&net->ct.generation);
618                 /* reuse the hash saved before */
619                 hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;
620                 hash = hash_bucket(hash, net);
621                 reply_hash = hash_conntrack(net, zone,
622                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
623 
624         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
625 
626         /* We're not in hash table, and we refuse to set up related
627          * connections for unconfirmed conns.  But packet copies and
628          * REJECT will give spurious warnings here.
629          */
630         /* NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 1); */
631 
632         /* No external references means no one else could have
633          * confirmed us.
634          */
635         NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
636         pr_debug("Confirming conntrack %p\n", ct);
637         /* We have to check the DYING flag after unlink to prevent
638          * a race against nf_ct_get_next_corpse() possibly called from
639          * user context, else we insert an already 'dead' hash, blocking
640          * further use of that particular connection -JM.
641          */
642         nf_ct_del_from_dying_or_unconfirmed_list(ct);
643 
644         if (unlikely(nf_ct_is_dying(ct)))
645                 goto out;
646 
647         /* See if there's one in the list already, including reverse:
648            NAT could have grabbed it without realizing, since we're
649            not in the hash.  If there is, we lost race. */
650         hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)
651                 if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
652                                       &h->tuple) &&
653                     zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
654                         goto out;
655         hlist_nulls_for_each_entry(h, n, &net->ct.hash[reply_hash], hnnode)
656                 if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,
657                                       &h->tuple) &&
658                     zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
659                         goto out;
660 
661         /* Timer relative to confirmation time, not original
662            setting time, otherwise we'd get timer wrap in
663            weird delay cases. */
664         ct->timeout.expires += jiffies;
665         add_timer(&ct->timeout);
666         atomic_inc(&ct->ct_general.use);
667         ct->status |= IPS_CONFIRMED;
668 
669         /* set conntrack timestamp, if enabled. */
670         tstamp = nf_conn_tstamp_find(ct);
671         if (tstamp) {
672                 if (skb->tstamp.tv64 == 0)
673                         __net_timestamp(skb);
674 
675                 tstamp->start = ktime_to_ns(skb->tstamp);
676         }
677         /* Since the lookup is lockless, hash insertion must be done after
678          * starting the timer and setting the CONFIRMED bit. The RCU barriers
679          * guarantee that no other CPU can find the conntrack before the above
680          * stores are visible.
681          */
682         __nf_conntrack_hash_insert(ct, hash, reply_hash);
683         nf_conntrack_double_unlock(hash, reply_hash);
684         NF_CT_STAT_INC(net, insert);
685         local_bh_enable();
686 
687         help = nfct_help(ct);
688         if (help && help->helper)
689                 nf_conntrack_event_cache(IPCT_HELPER, ct);
690 
691         nf_conntrack_event_cache(master_ct(ct) ?
692                                  IPCT_RELATED : IPCT_NEW, ct);
693         return NF_ACCEPT;
694 
695 out:
696         nf_ct_add_to_dying_list(ct);
697         nf_conntrack_double_unlock(hash, reply_hash);
698         NF_CT_STAT_INC(net, insert_failed);
699         local_bh_enable();
700         return NF_DROP;
701 }
702 EXPORT_SYMBOL_GPL(__nf_conntrack_confirm);
703 
704 /* Returns true if a connection correspondings to the tuple (required
705    for NAT). */
706 int
707 nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,
708                          const struct nf_conn *ignored_conntrack)
709 {
710         struct net *net = nf_ct_net(ignored_conntrack);
711         struct nf_conntrack_tuple_hash *h;
712         struct hlist_nulls_node *n;
713         struct nf_conn *ct;
714         u16 zone = nf_ct_zone(ignored_conntrack);
715         unsigned int hash = hash_conntrack(net, zone, tuple);
716 
717         /* Disable BHs the entire time since we need to disable them at
718          * least once for the stats anyway.
719          */
720         rcu_read_lock_bh();
721         hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash], hnnode) {
722                 ct = nf_ct_tuplehash_to_ctrack(h);
723                 if (ct != ignored_conntrack &&
724                     nf_ct_tuple_equal(tuple, &h->tuple) &&
725                     nf_ct_zone(ct) == zone) {
726                         NF_CT_STAT_INC(net, found);
727                         rcu_read_unlock_bh();
728                         return 1;
729                 }
730                 NF_CT_STAT_INC(net, searched);
731         }
732         rcu_read_unlock_bh();
733 
734         return 0;
735 }
736 EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);
737 
738 #define NF_CT_EVICTION_RANGE    8
739 
740 /* There's a small race here where we may free a just-assured
741    connection.  Too bad: we're in trouble anyway. */
742 static noinline int early_drop(struct net *net, unsigned int _hash)
743 {
744         /* Use oldest entry, which is roughly LRU */
745         struct nf_conntrack_tuple_hash *h;
746         struct nf_conn *ct = NULL, *tmp;
747         struct hlist_nulls_node *n;
748         unsigned int i = 0, cnt = 0;
749         int dropped = 0;
750         unsigned int hash, sequence;
751         spinlock_t *lockp;
752 
753         local_bh_disable();
754 restart:
755         sequence = read_seqcount_begin(&net->ct.generation);
756         hash = hash_bucket(_hash, net);
757         for (; i < net->ct.htable_size; i++) {
758                 lockp = &nf_conntrack_locks[hash % CONNTRACK_LOCKS];
759                 spin_lock(lockp);
760                 if (read_seqcount_retry(&net->ct.generation, sequence)) {
761                         spin_unlock(lockp);
762                         goto restart;
763                 }
764                 hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash],
765                                          hnnode) {
766                         tmp = nf_ct_tuplehash_to_ctrack(h);
767                         if (!test_bit(IPS_ASSURED_BIT, &tmp->status) &&
768                             !nf_ct_is_dying(tmp) &&
769                             atomic_inc_not_zero(&tmp->ct_general.use)) {
770                                 ct = tmp;
771                                 break;
772                         }
773                         cnt++;
774                 }
775 
776                 hash = (hash + 1) % net->ct.htable_size;
777                 spin_unlock(lockp);
778 
779                 if (ct || cnt >= NF_CT_EVICTION_RANGE)
780                         break;
781 
782         }
783         local_bh_enable();
784 
785         if (!ct)
786                 return dropped;
787 
788         if (del_timer(&ct->timeout)) {
789                 if (nf_ct_delete(ct, 0, 0)) {
790                         dropped = 1;
791                         NF_CT_STAT_INC_ATOMIC(net, early_drop);
792                 }
793         }
794         nf_ct_put(ct);
795         return dropped;
796 }
797 
798 void init_nf_conntrack_hash_rnd(void)
799 {
800         unsigned int rand;
801 
802         /*
803          * Why not initialize nf_conntrack_rnd in a "init()" function ?
804          * Because there isn't enough entropy when system initializing,
805          * and we initialize it as late as possible.
806          */
807         do {
808                 get_random_bytes(&rand, sizeof(rand));
809         } while (!rand);
810         cmpxchg(&nf_conntrack_hash_rnd, 0, rand);
811 }
812 
813 static struct nf_conn *
814 __nf_conntrack_alloc(struct net *net, u16 zone,
815                      const struct nf_conntrack_tuple *orig,
816                      const struct nf_conntrack_tuple *repl,
817                      gfp_t gfp, u32 hash)
818 {
819         struct nf_conn *ct;
820 
821         if (unlikely(!nf_conntrack_hash_rnd)) {
822                 init_nf_conntrack_hash_rnd();
823                 /* recompute the hash as nf_conntrack_hash_rnd is initialized */
824                 hash = hash_conntrack_raw(orig, zone);
825         }
826 
827         /* We don't want any race condition at early drop stage */
828         atomic_inc(&net->ct.count);
829 
830         if (nf_conntrack_max &&
831             unlikely(atomic_read(&net->ct.count) > nf_conntrack_max)) {
832                 if (!early_drop(net, hash)) {
833                         atomic_dec(&net->ct.count);
834                         net_warn_ratelimited("nf_conntrack: table full, dropping packet\n");
835                         return ERR_PTR(-ENOMEM);
836                 }
837         }
838 
839         /*
840          * Do not use kmem_cache_zalloc(), as this cache uses
841          * SLAB_DESTROY_BY_RCU.
842          */
843         ct = kmem_cache_alloc(net->ct.nf_conntrack_cachep, gfp);
844         if (ct == NULL) {
845                 atomic_dec(&net->ct.count);
846                 return ERR_PTR(-ENOMEM);
847         }
848         spin_lock_init(&ct->lock);
849         ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;
850         ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;
851         ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;
852         /* save hash for reusing when confirming */
853         *(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;
854         ct->status = 0;
855         /* Don't set timer yet: wait for confirmation */
856         setup_timer(&ct->timeout, death_by_timeout, (unsigned long)ct);
857         write_pnet(&ct->ct_net, net);
858         memset(&ct->__nfct_init_offset[0], 0,
859                offsetof(struct nf_conn, proto) -
860                offsetof(struct nf_conn, __nfct_init_offset[0]));
861 #ifdef CONFIG_NF_CONNTRACK_ZONES
862         if (zone) {
863                 struct nf_conntrack_zone *nf_ct_zone;
864 
865                 nf_ct_zone = nf_ct_ext_add(ct, NF_CT_EXT_ZONE, GFP_ATOMIC);
866                 if (!nf_ct_zone)
867                         goto out_free;
868                 nf_ct_zone->id = zone;
869         }
870 #endif
871         /* Because we use RCU lookups, we set ct_general.use to zero before
872          * this is inserted in any list.
873          */
874         atomic_set(&ct->ct_general.use, 0);
875         return ct;
876 
877 #ifdef CONFIG_NF_CONNTRACK_ZONES
878 out_free:
879         atomic_dec(&net->ct.count);
880         kmem_cache_free(net->ct.nf_conntrack_cachep, ct);
881         return ERR_PTR(-ENOMEM);
882 #endif
883 }
884 
885 struct nf_conn *nf_conntrack_alloc(struct net *net, u16 zone,
886                                    const struct nf_conntrack_tuple *orig,
887                                    const struct nf_conntrack_tuple *repl,
888                                    gfp_t gfp)
889 {
890         return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);
891 }
892 EXPORT_SYMBOL_GPL(nf_conntrack_alloc);
893 
894 void nf_conntrack_free(struct nf_conn *ct)
895 {
896         struct net *net = nf_ct_net(ct);
897 
898         /* A freed object has refcnt == 0, that's
899          * the golden rule for SLAB_DESTROY_BY_RCU
900          */
901         NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 0);
902 
903         nf_ct_ext_destroy(ct);
904         nf_ct_ext_free(ct);
905         kmem_cache_free(net->ct.nf_conntrack_cachep, ct);
906         smp_mb__before_atomic();
907         atomic_dec(&net->ct.count);
908 }
909 EXPORT_SYMBOL_GPL(nf_conntrack_free);
910 
911 
912 /* Allocate a new conntrack: we return -ENOMEM if classification
913    failed due to stress.  Otherwise it really is unclassifiable. */
914 static struct nf_conntrack_tuple_hash *
915 init_conntrack(struct net *net, struct nf_conn *tmpl,
916                const struct nf_conntrack_tuple *tuple,
917                struct nf_conntrack_l3proto *l3proto,
918                struct nf_conntrack_l4proto *l4proto,
919                struct sk_buff *skb,
920                unsigned int dataoff, u32 hash)
921 {
922         struct nf_conn *ct;
923         struct nf_conn_help *help;
924         struct nf_conntrack_tuple repl_tuple;
925         struct nf_conntrack_ecache *ecache;
926         struct nf_conntrack_expect *exp = NULL;
927         u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;
928         struct nf_conn_timeout *timeout_ext;
929         unsigned int *timeouts;
930 
931         if (!nf_ct_invert_tuple(&repl_tuple, tuple, l3proto, l4proto)) {
932                 pr_debug("Can't invert tuple.\n");
933                 return NULL;
934         }
935 
936         ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,
937                                   hash);
938         if (IS_ERR(ct))
939                 return (struct nf_conntrack_tuple_hash *)ct;
940 
941         if (tmpl && nfct_synproxy(tmpl)) {
942                 nfct_seqadj_ext_add(ct);
943                 nfct_synproxy_ext_add(ct);
944         }
945 
946         timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL;
947         if (timeout_ext)
948                 timeouts = NF_CT_TIMEOUT_EXT_DATA(timeout_ext);
949         else
950                 timeouts = l4proto->get_timeouts(net);
951 
952         if (!l4proto->new(ct, skb, dataoff, timeouts)) {
953                 nf_conntrack_free(ct);
954                 pr_debug("init conntrack: can't track with proto module\n");
955                 return NULL;
956         }
957 
958         if (timeout_ext)
959                 nf_ct_timeout_ext_add(ct, timeout_ext->timeout, GFP_ATOMIC);
960 
961         nf_ct_acct_ext_add(ct, GFP_ATOMIC);
962         nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);
963         nf_ct_labels_ext_add(ct);
964 
965         ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;
966         nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,
967                                  ecache ? ecache->expmask : 0,
968                              GFP_ATOMIC);
969 
970         local_bh_disable();
971         if (net->ct.expect_count) {
972                 spin_lock(&nf_conntrack_expect_lock);
973                 exp = nf_ct_find_expectation(net, zone, tuple);
974                 if (exp) {
975                         pr_debug("conntrack: expectation arrives ct=%p exp=%p\n",
976                                  ct, exp);
977                         /* Welcome, Mr. Bond.  We've been expecting you... */
978                         __set_bit(IPS_EXPECTED_BIT, &ct->status);
979                         /* exp->master safe, refcnt bumped in nf_ct_find_expectation */
980                         ct->master = exp->master;
981                         if (exp->helper) {
982                                 help = nf_ct_helper_ext_add(ct, exp->helper,
983                                                             GFP_ATOMIC);
984                                 if (help)
985                                         rcu_assign_pointer(help->helper, exp->helper);
986                         }
987 
988 #ifdef CONFIG_NF_CONNTRACK_MARK
989                         ct->mark = exp->master->mark;
990 #endif
991 #ifdef CONFIG_NF_CONNTRACK_SECMARK
992                         ct->secmark = exp->master->secmark;
993 #endif
994                         NF_CT_STAT_INC(net, expect_new);
995                 }
996                 spin_unlock(&nf_conntrack_expect_lock);
997         }
998         if (!exp) {
999                 __nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);
1000                 NF_CT_STAT_INC(net, new);
1001         }
1002 
1003         /* Now it is inserted into the unconfirmed list, bump refcount */
1004         nf_conntrack_get(&ct->ct_general);
1005         nf_ct_add_to_unconfirmed_list(ct);
1006 
1007         local_bh_enable();
1008 
1009         if (exp) {
1010                 if (exp->expectfn)
1011                         exp->expectfn(ct, exp);
1012                 nf_ct_expect_put(exp);
1013         }
1014 
1015         return &ct->tuplehash[IP_CT_DIR_ORIGINAL];
1016 }
1017 
1018 /* On success, returns conntrack ptr, sets skb->nfct and ctinfo */
1019 static inline struct nf_conn *
1020 resolve_normal_ct(struct net *net, struct nf_conn *tmpl,
1021                   struct sk_buff *skb,
1022                   unsigned int dataoff,
1023                   u_int16_t l3num,
1024                   u_int8_t protonum,
1025                   struct nf_conntrack_l3proto *l3proto,
1026                   struct nf_conntrack_l4proto *l4proto,
1027                   int *set_reply,
1028                   enum ip_conntrack_info *ctinfo)
1029 {
1030         struct nf_conntrack_tuple tuple;
1031         struct nf_conntrack_tuple_hash *h;
1032         struct nf_conn *ct;
1033         u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;
1034         u32 hash;
1035 
1036         if (!nf_ct_get_tuple(skb, skb_network_offset(skb),
1037                              dataoff, l3num, protonum, &tuple, l3proto,
1038                              l4proto)) {
1039                 pr_debug("resolve_normal_ct: Can't get tuple\n");
1040                 return NULL;
1041         }
1042 
1043         /* look for tuple match */
1044         hash = hash_conntrack_raw(&tuple, zone);
1045         h = __nf_conntrack_find_get(net, zone, &tuple, hash);
1046         if (!h) {
1047                 h = init_conntrack(net, tmpl, &tuple, l3proto, l4proto,
1048                                    skb, dataoff, hash);
1049                 if (!h)
1050                         return NULL;
1051                 if (IS_ERR(h))
1052                         return (void *)h;
1053         }
1054         ct = nf_ct_tuplehash_to_ctrack(h);
1055 
1056         /* It exists; we have (non-exclusive) reference. */
1057         if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {
1058                 *ctinfo = IP_CT_ESTABLISHED_REPLY;
1059                 /* Please set reply bit if this packet OK */
1060                 *set_reply = 1;
1061         } else {
1062                 /* Once we've had two way comms, always ESTABLISHED. */
1063                 if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {
1064                         pr_debug("nf_conntrack_in: normal packet for %p\n", ct);
1065                         *ctinfo = IP_CT_ESTABLISHED;
1066                 } else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {
1067                         pr_debug("nf_conntrack_in: related packet for %p\n",
1068                                  ct);
1069                         *ctinfo = IP_CT_RELATED;
1070                 } else {
1071                         pr_debug("nf_conntrack_in: new packet for %p\n", ct);
1072                         *ctinfo = IP_CT_NEW;
1073                 }
1074                 *set_reply = 0;
1075         }
1076         skb->nfct = &ct->ct_general;
1077         skb->nfctinfo = *ctinfo;
1078         return ct;
1079 }
1080 
1081 unsigned int
1082 nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,
1083                 struct sk_buff *skb)
1084 {
1085         struct nf_conn *ct, *tmpl = NULL;
1086         enum ip_conntrack_info ctinfo;
1087         struct nf_conntrack_l3proto *l3proto;
1088         struct nf_conntrack_l4proto *l4proto;
1089         unsigned int *timeouts;
1090         unsigned int dataoff;
1091         u_int8_t protonum;
1092         int set_reply = 0;
1093         int ret;
1094 
1095         if (skb->nfct) {
1096                 /* Previously seen (loopback or untracked)?  Ignore. */
1097                 tmpl = (struct nf_conn *)skb->nfct;
1098                 if (!nf_ct_is_template(tmpl)) {
1099                         NF_CT_STAT_INC_ATOMIC(net, ignore);
1100                         return NF_ACCEPT;
1101                 }
1102                 skb->nfct = NULL;
1103         }
1104 
1105         /* rcu_read_lock()ed by nf_hook_slow */
1106         l3proto = __nf_ct_l3proto_find(pf);
1107         ret = l3proto->get_l4proto(skb, skb_network_offset(skb),
1108                                    &dataoff, &protonum);
1109         if (ret <= 0) {
1110                 pr_debug("not prepared to track yet or error occurred\n");
1111                 NF_CT_STAT_INC_ATOMIC(net, error);
1112                 NF_CT_STAT_INC_ATOMIC(net, invalid);
1113                 ret = -ret;
1114                 goto out;
1115         }
1116 
1117         l4proto = __nf_ct_l4proto_find(pf, protonum);
1118 
1119         /* It may be an special packet, error, unclean...
1120          * inverse of the return code tells to the netfilter
1121          * core what to do with the packet. */
1122         if (l4proto->error != NULL) {
1123                 ret = l4proto->error(net, tmpl, skb, dataoff, &ctinfo,
1124                                      pf, hooknum);
1125                 if (ret <= 0) {
1126                         NF_CT_STAT_INC_ATOMIC(net, error);
1127                         NF_CT_STAT_INC_ATOMIC(net, invalid);
1128                         ret = -ret;
1129                         goto out;
1130                 }
1131                 /* ICMP[v6] protocol trackers may assign one conntrack. */
1132                 if (skb->nfct)
1133                         goto out;
1134         }
1135 
1136         ct = resolve_normal_ct(net, tmpl, skb, dataoff, pf, protonum,
1137                                l3proto, l4proto, &set_reply, &ctinfo);
1138         if (!ct) {
1139                 /* Not valid part of a connection */
1140                 NF_CT_STAT_INC_ATOMIC(net, invalid);
1141                 ret = NF_ACCEPT;
1142                 goto out;
1143         }
1144 
1145         if (IS_ERR(ct)) {
1146                 /* Too stressed to deal. */
1147                 NF_CT_STAT_INC_ATOMIC(net, drop);
1148                 ret = NF_DROP;
1149                 goto out;
1150         }
1151 
1152         NF_CT_ASSERT(skb->nfct);
1153 
1154         /* Decide what timeout policy we want to apply to this flow. */
1155         timeouts = nf_ct_timeout_lookup(net, ct, l4proto);
1156 
1157         ret = l4proto->packet(ct, skb, dataoff, ctinfo, pf, hooknum, timeouts);
1158         if (ret <= 0) {
1159                 /* Invalid: inverse of the return code tells
1160                  * the netfilter core what to do */
1161                 pr_debug("nf_conntrack_in: Can't track with proto module\n");
1162                 nf_conntrack_put(skb->nfct);
1163                 skb->nfct = NULL;
1164                 NF_CT_STAT_INC_ATOMIC(net, invalid);
1165                 if (ret == -NF_DROP)
1166                         NF_CT_STAT_INC_ATOMIC(net, drop);
1167                 ret = -ret;
1168                 goto out;
1169         }
1170 
1171         if (set_reply && !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))
1172                 nf_conntrack_event_cache(IPCT_REPLY, ct);
1173 out:
1174         if (tmpl) {
1175                 /* Special case: we have to repeat this hook, assign the
1176                  * template again to this packet. We assume that this packet
1177                  * has no conntrack assigned. This is used by nf_ct_tcp. */
1178                 if (ret == NF_REPEAT)
1179                         skb->nfct = (struct nf_conntrack *)tmpl;
1180                 else
1181                         nf_ct_put(tmpl);
1182         }
1183 
1184         return ret;
1185 }
1186 EXPORT_SYMBOL_GPL(nf_conntrack_in);
1187 
1188 bool nf_ct_invert_tuplepr(struct nf_conntrack_tuple *inverse,
1189                           const struct nf_conntrack_tuple *orig)
1190 {
1191         bool ret;
1192 
1193         rcu_read_lock();
1194         ret = nf_ct_invert_tuple(inverse, orig,
1195                                  __nf_ct_l3proto_find(orig->src.l3num),
1196                                  __nf_ct_l4proto_find(orig->src.l3num,
1197                                                       orig->dst.protonum));
1198         rcu_read_unlock();
1199         return ret;
1200 }
1201 EXPORT_SYMBOL_GPL(nf_ct_invert_tuplepr);
1202 
1203 /* Alter reply tuple (maybe alter helper).  This is for NAT, and is
1204    implicitly racy: see __nf_conntrack_confirm */
1205 void nf_conntrack_alter_reply(struct nf_conn *ct,
1206                               const struct nf_conntrack_tuple *newreply)
1207 {
1208         struct nf_conn_help *help = nfct_help(ct);
1209 
1210         /* Should be unconfirmed, so not in hash table yet */
1211         NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
1212 
1213         pr_debug("Altering reply tuple of %p to ", ct);
1214         nf_ct_dump_tuple(newreply);
1215 
1216         ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;
1217         if (ct->master || (help && !hlist_empty(&help->expectations)))
1218                 return;
1219 
1220         rcu_read_lock();
1221         __nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC);
1222         rcu_read_unlock();
1223 }
1224 EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply);
1225 
1226 /* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */
1227 void __nf_ct_refresh_acct(struct nf_conn *ct,
1228                           enum ip_conntrack_info ctinfo,
1229                           const struct sk_buff *skb,
1230                           unsigned long extra_jiffies,
1231                           int do_acct)
1232 {
1233         NF_CT_ASSERT(ct->timeout.data == (unsigned long)ct);
1234         NF_CT_ASSERT(skb);
1235 
1236         /* Only update if this is not a fixed timeout */
1237         if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))
1238                 goto acct;
1239 
1240         /* If not in hash table, timer will not be active yet */
1241         if (!nf_ct_is_confirmed(ct)) {
1242                 ct->timeout.expires = extra_jiffies;
1243         } else {
1244                 unsigned long newtime = jiffies + extra_jiffies;
1245 
1246                 /* Only update the timeout if the new timeout is at least
1247                    HZ jiffies from the old timeout. Need del_timer for race
1248                    avoidance (may already be dying). */
1249                 if (newtime - ct->timeout.expires >= HZ)
1250                         mod_timer_pending(&ct->timeout, newtime);
1251         }
1252 
1253 acct:
1254         if (do_acct) {
1255                 struct nf_conn_acct *acct;
1256 
1257                 acct = nf_conn_acct_find(ct);
1258                 if (acct) {
1259                         struct nf_conn_counter *counter = acct->counter;
1260 
1261                         atomic64_inc(&counter[CTINFO2DIR(ctinfo)].packets);
1262                         atomic64_add(skb->len, &counter[CTINFO2DIR(ctinfo)].bytes);
1263                 }
1264         }
1265 }
1266 EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);
1267 
1268 bool __nf_ct_kill_acct(struct nf_conn *ct,
1269                        enum ip_conntrack_info ctinfo,
1270                        const struct sk_buff *skb,
1271                        int do_acct)
1272 {
1273         if (do_acct) {
1274                 struct nf_conn_acct *acct;
1275 
1276                 acct = nf_conn_acct_find(ct);
1277                 if (acct) {
1278                         struct nf_conn_counter *counter = acct->counter;
1279 
1280                         atomic64_inc(&counter[CTINFO2DIR(ctinfo)].packets);
1281                         atomic64_add(skb->len - skb_network_offset(skb),
1282                                      &counter[CTINFO2DIR(ctinfo)].bytes);
1283                 }
1284         }
1285 
1286         if (del_timer(&ct->timeout)) {
1287                 ct->timeout.function((unsigned long)ct);
1288                 return true;
1289         }
1290         return false;
1291 }
1292 EXPORT_SYMBOL_GPL(__nf_ct_kill_acct);
1293 
1294 #ifdef CONFIG_NF_CONNTRACK_ZONES
1295 static struct nf_ct_ext_type nf_ct_zone_extend __read_mostly = {
1296         .len    = sizeof(struct nf_conntrack_zone),
1297         .align  = __alignof__(struct nf_conntrack_zone),
1298         .id     = NF_CT_EXT_ZONE,
1299 };
1300 #endif
1301 
1302 #if IS_ENABLED(CONFIG_NF_CT_NETLINK)
1303 
1304 #include <linux/netfilter/nfnetlink.h>
1305 #include <linux/netfilter/nfnetlink_conntrack.h>
1306 #include <linux/mutex.h>
1307 
1308 /* Generic function for tcp/udp/sctp/dccp and alike. This needs to be
1309  * in ip_conntrack_core, since we don't want the protocols to autoload
1310  * or depend on ctnetlink */
1311 int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,
1312                                const struct nf_conntrack_tuple *tuple)
1313 {
1314         if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) ||
1315             nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port))
1316                 goto nla_put_failure;
1317         return 0;
1318 
1319 nla_put_failure:
1320         return -1;
1321 }
1322 EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr);
1323 
1324 const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = {
1325         [CTA_PROTO_SRC_PORT]  = { .type = NLA_U16 },
1326         [CTA_PROTO_DST_PORT]  = { .type = NLA_U16 },
1327 };
1328 EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);
1329 
1330 int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],
1331                                struct nf_conntrack_tuple *t)
1332 {
1333         if (!tb[CTA_PROTO_SRC_PORT] || !tb[CTA_PROTO_DST_PORT])
1334                 return -EINVAL;
1335 
1336         t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);
1337         t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);
1338 
1339         return 0;
1340 }
1341 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple);
1342 
1343 int nf_ct_port_nlattr_tuple_size(void)
1344 {
1345         return nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);
1346 }
1347 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size);
1348 #endif
1349 
1350 /* Used by ipt_REJECT and ip6t_REJECT. */
1351 static void nf_conntrack_attach(struct sk_buff *nskb, const struct sk_buff *skb)
1352 {
1353         struct nf_conn *ct;
1354         enum ip_conntrack_info ctinfo;
1355 
1356         /* This ICMP is in reverse direction to the packet which caused it */
1357         ct = nf_ct_get(skb, &ctinfo);
1358         if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)
1359                 ctinfo = IP_CT_RELATED_REPLY;
1360         else
1361                 ctinfo = IP_CT_RELATED;
1362 
1363         /* Attach to new skbuff, and increment count */
1364         nskb->nfct = &ct->ct_general;
1365         nskb->nfctinfo = ctinfo;
1366         nf_conntrack_get(nskb->nfct);
1367 }
1368 
1369 /* Bring out ya dead! */
1370 static struct nf_conn *
1371 get_next_corpse(struct net *net, int (*iter)(struct nf_conn *i, void *data),
1372                 void *data, unsigned int *bucket)
1373 {
1374         struct nf_conntrack_tuple_hash *h;
1375         struct nf_conn *ct;
1376         struct hlist_nulls_node *n;
1377         int cpu;
1378         spinlock_t *lockp;
1379 
1380         for (; *bucket < net->ct.htable_size; (*bucket)++) {
1381                 lockp = &nf_conntrack_locks[*bucket % CONNTRACK_LOCKS];
1382                 local_bh_disable();
1383                 spin_lock(lockp);
1384                 if (*bucket < net->ct.htable_size) {
1385                         hlist_nulls_for_each_entry(h, n, &net->ct.hash[*bucket], hnnode) {
1386                                 if (NF_CT_DIRECTION(h) != IP_CT_DIR_ORIGINAL)
1387                                         continue;
1388                                 ct = nf_ct_tuplehash_to_ctrack(h);
1389                                 if (iter(ct, data))
1390                                         goto found;
1391                         }
1392                 }
1393                 spin_unlock(lockp);
1394                 local_bh_enable();
1395         }
1396 
1397         for_each_possible_cpu(cpu) {
1398                 struct ct_pcpu *pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu);
1399 
1400                 spin_lock_bh(&pcpu->lock);
1401                 hlist_nulls_for_each_entry(h, n, &pcpu->unconfirmed, hnnode) {
1402                         ct = nf_ct_tuplehash_to_ctrack(h);
1403                         if (iter(ct, data))
1404                                 set_bit(IPS_DYING_BIT, &ct->status);
1405                 }
1406                 spin_unlock_bh(&pcpu->lock);
1407         }
1408         return NULL;
1409 found:
1410         atomic_inc(&ct->ct_general.use);
1411         spin_unlock(lockp);
1412         local_bh_enable();
1413         return ct;
1414 }
1415 
1416 void nf_ct_iterate_cleanup(struct net *net,
1417                            int (*iter)(struct nf_conn *i, void *data),
1418                            void *data, u32 portid, int report)
1419 {
1420         struct nf_conn *ct;
1421         unsigned int bucket = 0;
1422 
1423         while ((ct = get_next_corpse(net, iter, data, &bucket)) != NULL) {
1424                 /* Time to push up daises... */
1425                 if (del_timer(&ct->timeout))
1426                         nf_ct_delete(ct, portid, report);
1427 
1428                 /* ... else the timer will get him soon. */
1429 
1430                 nf_ct_put(ct);
1431         }
1432 }
1433 EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup);
1434 
1435 static int kill_all(struct nf_conn *i, void *data)
1436 {
1437         return 1;
1438 }
1439 
1440 void nf_ct_free_hashtable(void *hash, unsigned int size)
1441 {
1442         if (is_vmalloc_addr(hash))
1443                 vfree(hash);
1444         else
1445                 free_pages((unsigned long)hash,
1446                            get_order(sizeof(struct hlist_head) * size));
1447 }
1448 EXPORT_SYMBOL_GPL(nf_ct_free_hashtable);
1449 
1450 static int untrack_refs(void)
1451 {
1452         int cnt = 0, cpu;
1453 
1454         for_each_possible_cpu(cpu) {
1455                 struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
1456 
1457                 cnt += atomic_read(&ct->ct_general.use) - 1;
1458         }
1459         return cnt;
1460 }
1461 
1462 void nf_conntrack_cleanup_start(void)
1463 {
1464         RCU_INIT_POINTER(ip_ct_attach, NULL);
1465 }
1466 
1467 void nf_conntrack_cleanup_end(void)
1468 {
1469         RCU_INIT_POINTER(nf_ct_destroy, NULL);
1470         while (untrack_refs() > 0)
1471                 schedule();
1472 
1473 #ifdef CONFIG_NF_CONNTRACK_ZONES
1474         nf_ct_extend_unregister(&nf_ct_zone_extend);
1475 #endif
1476         nf_conntrack_proto_fini();
1477         nf_conntrack_seqadj_fini();
1478         nf_conntrack_labels_fini();
1479         nf_conntrack_helper_fini();
1480         nf_conntrack_timeout_fini();
1481         nf_conntrack_ecache_fini();
1482         nf_conntrack_tstamp_fini();
1483         nf_conntrack_acct_fini();
1484         nf_conntrack_expect_fini();
1485 }
1486 
1487 /*
1488  * Mishearing the voices in his head, our hero wonders how he's
1489  * supposed to kill the mall.
1490  */
1491 void nf_conntrack_cleanup_net(struct net *net)
1492 {
1493         LIST_HEAD(single);
1494 
1495         list_add(&net->exit_list, &single);
1496         nf_conntrack_cleanup_net_list(&single);
1497 }
1498 
1499 void nf_conntrack_cleanup_net_list(struct list_head *net_exit_list)
1500 {
1501         int busy;
1502         struct net *net;
1503 
1504         /*
1505          * This makes sure all current packets have passed through
1506          *  netfilter framework.  Roll on, two-stage module
1507          *  delete...
1508          */
1509         synchronize_net();
1510 i_see_dead_people:
1511         busy = 0;
1512         list_for_each_entry(net, net_exit_list, exit_list) {
1513                 nf_ct_iterate_cleanup(net, kill_all, NULL, 0, 0);
1514                 if (atomic_read(&net->ct.count) != 0)
1515                         busy = 1;
1516         }
1517         if (busy) {
1518                 schedule();
1519                 goto i_see_dead_people;
1520         }
1521 
1522         list_for_each_entry(net, net_exit_list, exit_list) {
1523                 nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
1524                 nf_conntrack_proto_pernet_fini(net);
1525                 nf_conntrack_helper_pernet_fini(net);
1526                 nf_conntrack_ecache_pernet_fini(net);
1527                 nf_conntrack_tstamp_pernet_fini(net);
1528                 nf_conntrack_acct_pernet_fini(net);
1529                 nf_conntrack_expect_pernet_fini(net);
1530                 kmem_cache_destroy(net->ct.nf_conntrack_cachep);
1531                 kfree(net->ct.slabname);
1532                 free_percpu(net->ct.stat);
1533                 free_percpu(net->ct.pcpu_lists);
1534         }
1535 }
1536 
1537 void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)
1538 {
1539         struct hlist_nulls_head *hash;
1540         unsigned int nr_slots, i;
1541         size_t sz;
1542 
1543         BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));
1544         nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));
1545         sz = nr_slots * sizeof(struct hlist_nulls_head);
1546         hash = (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
1547                                         get_order(sz));
1548         if (!hash)
1549                 hash = vzalloc(sz);
1550 
1551         if (hash && nulls)
1552                 for (i = 0; i < nr_slots; i++)
1553                         INIT_HLIST_NULLS_HEAD(&hash[i], i);
1554 
1555         return hash;
1556 }
1557 EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable);
1558 
1559 int nf_conntrack_set_hashsize(const char *val, struct kernel_param *kp)
1560 {
1561         int i, bucket, rc;
1562         unsigned int hashsize, old_size;
1563         struct hlist_nulls_head *hash, *old_hash;
1564         struct nf_conntrack_tuple_hash *h;
1565         struct nf_conn *ct;
1566 
1567         if (current->nsproxy->net_ns != &init_net)
1568                 return -EOPNOTSUPP;
1569 
1570         /* On boot, we can set this without any fancy locking. */
1571         if (!nf_conntrack_htable_size)
1572                 return param_set_uint(val, kp);
1573 
1574         rc = kstrtouint(val, 0, &hashsize);
1575         if (rc)
1576                 return rc;
1577         if (!hashsize)
1578                 return -EINVAL;
1579 
1580         hash = nf_ct_alloc_hashtable(&hashsize, 1);
1581         if (!hash)
1582                 return -ENOMEM;
1583 
1584         local_bh_disable();
1585         nf_conntrack_all_lock();
1586         write_seqcount_begin(&init_net.ct.generation);
1587 
1588         /* Lookups in the old hash might happen in parallel, which means we
1589          * might get false negatives during connection lookup. New connections
1590          * created because of a false negative won't make it into the hash
1591          * though since that required taking the locks.
1592          */
1593 
1594         for (i = 0; i < init_net.ct.htable_size; i++) {
1595                 while (!hlist_nulls_empty(&init_net.ct.hash[i])) {
1596                         h = hlist_nulls_entry(init_net.ct.hash[i].first,
1597                                         struct nf_conntrack_tuple_hash, hnnode);
1598                         ct = nf_ct_tuplehash_to_ctrack(h);
1599                         hlist_nulls_del_rcu(&h->hnnode);
1600                         bucket = __hash_conntrack(&h->tuple, nf_ct_zone(ct),
1601                                                   hashsize);
1602                         hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);
1603                 }
1604         }
1605         old_size = init_net.ct.htable_size;
1606         old_hash = init_net.ct.hash;
1607 
1608         init_net.ct.htable_size = nf_conntrack_htable_size = hashsize;
1609         init_net.ct.hash = hash;
1610 
1611         write_seqcount_end(&init_net.ct.generation);
1612         nf_conntrack_all_unlock();
1613         local_bh_enable();
1614 
1615         nf_ct_free_hashtable(old_hash, old_size);
1616         return 0;
1617 }
1618 EXPORT_SYMBOL_GPL(nf_conntrack_set_hashsize);
1619 
1620 module_param_call(hashsize, nf_conntrack_set_hashsize, param_get_uint,
1621                   &nf_conntrack_htable_size, 0600);
1622 
1623 void nf_ct_untracked_status_or(unsigned long bits)
1624 {
1625         int cpu;
1626 
1627         for_each_possible_cpu(cpu)
1628                 per_cpu(nf_conntrack_untracked, cpu).status |= bits;
1629 }
1630 EXPORT_SYMBOL_GPL(nf_ct_untracked_status_or);
1631 
1632 int nf_conntrack_init_start(void)
1633 {
1634         int max_factor = 8;
1635         int i, ret, cpu;
1636 
1637         for (i = 0; i < CONNTRACK_LOCKS; i++)
1638                 spin_lock_init(&nf_conntrack_locks[i]);
1639 
1640         if (!nf_conntrack_htable_size) {
1641                 /* Idea from tcp.c: use 1/16384 of memory.
1642                  * On i386: 32MB machine has 512 buckets.
1643                  * >= 1GB machines have 16384 buckets.
1644                  * >= 4GB machines have 65536 buckets.
1645                  */
1646                 nf_conntrack_htable_size
1647                         = (((totalram_pages << PAGE_SHIFT) / 16384)
1648                            / sizeof(struct hlist_head));
1649                 if (totalram_pages > (4 * (1024 * 1024 * 1024 / PAGE_SIZE)))
1650                         nf_conntrack_htable_size = 65536;
1651                 else if (totalram_pages > (1024 * 1024 * 1024 / PAGE_SIZE))
1652                         nf_conntrack_htable_size = 16384;
1653                 if (nf_conntrack_htable_size < 32)
1654                         nf_conntrack_htable_size = 32;
1655 
1656                 /* Use a max. factor of four by default to get the same max as
1657                  * with the old struct list_heads. When a table size is given
1658                  * we use the old value of 8 to avoid reducing the max.
1659                  * entries. */
1660                 max_factor = 4;
1661         }
1662         nf_conntrack_max = max_factor * nf_conntrack_htable_size;
1663 
1664         printk(KERN_INFO "nf_conntrack version %s (%u buckets, %d max)\n",
1665                NF_CONNTRACK_VERSION, nf_conntrack_htable_size,
1666                nf_conntrack_max);
1667 
1668         ret = nf_conntrack_expect_init();
1669         if (ret < 0)
1670                 goto err_expect;
1671 
1672         ret = nf_conntrack_acct_init();
1673         if (ret < 0)
1674                 goto err_acct;
1675 
1676         ret = nf_conntrack_tstamp_init();
1677         if (ret < 0)
1678                 goto err_tstamp;
1679 
1680         ret = nf_conntrack_ecache_init();
1681         if (ret < 0)
1682                 goto err_ecache;
1683 
1684         ret = nf_conntrack_timeout_init();
1685         if (ret < 0)
1686                 goto err_timeout;
1687 
1688         ret = nf_conntrack_helper_init();
1689         if (ret < 0)
1690                 goto err_helper;
1691 
1692         ret = nf_conntrack_labels_init();
1693         if (ret < 0)
1694                 goto err_labels;
1695 
1696         ret = nf_conntrack_seqadj_init();
1697         if (ret < 0)
1698                 goto err_seqadj;
1699 
1700 #ifdef CONFIG_NF_CONNTRACK_ZONES
1701         ret = nf_ct_extend_register(&nf_ct_zone_extend);
1702         if (ret < 0)
1703                 goto err_extend;
1704 #endif
1705         ret = nf_conntrack_proto_init();
1706         if (ret < 0)
1707                 goto err_proto;
1708 
1709         /* Set up fake conntrack: to never be deleted, not in any hashes */
1710         for_each_possible_cpu(cpu) {
1711                 struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
1712                 write_pnet(&ct->ct_net, &init_net);
1713                 atomic_set(&ct->ct_general.use, 1);
1714         }
1715         /*  - and look it like as a confirmed connection */
1716         nf_ct_untracked_status_or(IPS_CONFIRMED | IPS_UNTRACKED);
1717         return 0;
1718 
1719 err_proto:
1720 #ifdef CONFIG_NF_CONNTRACK_ZONES
1721         nf_ct_extend_unregister(&nf_ct_zone_extend);
1722 err_extend:
1723 #endif
1724         nf_conntrack_seqadj_fini();
1725 err_seqadj:
1726         nf_conntrack_labels_fini();
1727 err_labels:
1728         nf_conntrack_helper_fini();
1729 err_helper:
1730         nf_conntrack_timeout_fini();
1731 err_timeout:
1732         nf_conntrack_ecache_fini();
1733 err_ecache:
1734         nf_conntrack_tstamp_fini();
1735 err_tstamp:
1736         nf_conntrack_acct_fini();
1737 err_acct:
1738         nf_conntrack_expect_fini();
1739 err_expect:
1740         return ret;
1741 }
1742 
1743 void nf_conntrack_init_end(void)
1744 {
1745         /* For use by REJECT target */
1746         RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach);
1747         RCU_INIT_POINTER(nf_ct_destroy, destroy_conntrack);
1748 }
1749 
1750 /*
1751  * We need to use special "null" values, not used in hash table
1752  */
1753 #define UNCONFIRMED_NULLS_VAL   ((1<<30)+0)
1754 #define DYING_NULLS_VAL         ((1<<30)+1)
1755 #define TEMPLATE_NULLS_VAL      ((1<<30)+2)
1756 
1757 int nf_conntrack_init_net(struct net *net)
1758 {
1759         int ret = -ENOMEM;
1760         int cpu;
1761 
1762         atomic_set(&net->ct.count, 0);
1763         seqcount_init(&net->ct.generation);
1764 
1765         net->ct.pcpu_lists = alloc_percpu(struct ct_pcpu);
1766         if (!net->ct.pcpu_lists)
1767                 goto err_stat;
1768 
1769         for_each_possible_cpu(cpu) {
1770                 struct ct_pcpu *pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu);
1771 
1772                 spin_lock_init(&pcpu->lock);
1773                 INIT_HLIST_NULLS_HEAD(&pcpu->unconfirmed, UNCONFIRMED_NULLS_VAL);
1774                 INIT_HLIST_NULLS_HEAD(&pcpu->dying, DYING_NULLS_VAL);
1775         }
1776 
1777         net->ct.stat = alloc_percpu(struct ip_conntrack_stat);
1778         if (!net->ct.stat)
1779                 goto err_pcpu_lists;
1780 
1781         net->ct.slabname = kasprintf(GFP_KERNEL, "nf_conntrack_%p", net);
1782         if (!net->ct.slabname)
1783                 goto err_slabname;
1784 
1785         net->ct.nf_conntrack_cachep = kmem_cache_create(net->ct.slabname,
1786                                                         sizeof(struct nf_conn), 0,
1787                                                         SLAB_DESTROY_BY_RCU, NULL);
1788         if (!net->ct.nf_conntrack_cachep) {
1789                 printk(KERN_ERR "Unable to create nf_conn slab cache\n");
1790                 goto err_cache;
1791         }
1792 
1793         net->ct.htable_size = nf_conntrack_htable_size;
1794         net->ct.hash = nf_ct_alloc_hashtable(&net->ct.htable_size, 1);
1795         if (!net->ct.hash) {
1796                 printk(KERN_ERR "Unable to create nf_conntrack_hash\n");
1797                 goto err_hash;
1798         }
1799         ret = nf_conntrack_expect_pernet_init(net);
1800         if (ret < 0)
1801                 goto err_expect;
1802         ret = nf_conntrack_acct_pernet_init(net);
1803         if (ret < 0)
1804                 goto err_acct;
1805         ret = nf_conntrack_tstamp_pernet_init(net);
1806         if (ret < 0)
1807                 goto err_tstamp;
1808         ret = nf_conntrack_ecache_pernet_init(net);
1809         if (ret < 0)
1810                 goto err_ecache;
1811         ret = nf_conntrack_helper_pernet_init(net);
1812         if (ret < 0)
1813                 goto err_helper;
1814         ret = nf_conntrack_proto_pernet_init(net);
1815         if (ret < 0)
1816                 goto err_proto;
1817         return 0;
1818 
1819 err_proto:
1820         nf_conntrack_helper_pernet_fini(net);
1821 err_helper:
1822         nf_conntrack_ecache_pernet_fini(net);
1823 err_ecache:
1824         nf_conntrack_tstamp_pernet_fini(net);
1825 err_tstamp:
1826         nf_conntrack_acct_pernet_fini(net);
1827 err_acct:
1828         nf_conntrack_expect_pernet_fini(net);
1829 err_expect:
1830         nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
1831 err_hash:
1832         kmem_cache_destroy(net->ct.nf_conntrack_cachep);
1833 err_cache:
1834         kfree(net->ct.slabname);
1835 err_slabname:
1836         free_percpu(net->ct.stat);
1837 err_pcpu_lists:
1838         free_percpu(net->ct.pcpu_lists);
1839 err_stat:
1840         return ret;
1841 }
1842 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp