~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/xfrm/xfrm_policy.c

Version: ~ [ linux-6.1-rc7 ] ~ [ linux-6.0.10 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.80 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.156 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.225 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.267 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.300 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.334 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-only
  2 /*
  3  * xfrm_policy.c
  4  *
  5  * Changes:
  6  *      Mitsuru KANDA @USAGI
  7  *      Kazunori MIYAZAWA @USAGI
  8  *      Kunihiro Ishiguro <kunihiro@ipinfusion.com>
  9  *              IPv6 support
 10  *      Kazunori MIYAZAWA @USAGI
 11  *      YOSHIFUJI Hideaki
 12  *              Split up af-specific portion
 13  *      Derek Atkins <derek@ihtfp.com>          Add the post_input processor
 14  *
 15  */
 16 
 17 #include <linux/err.h>
 18 #include <linux/slab.h>
 19 #include <linux/kmod.h>
 20 #include <linux/list.h>
 21 #include <linux/spinlock.h>
 22 #include <linux/workqueue.h>
 23 #include <linux/notifier.h>
 24 #include <linux/netdevice.h>
 25 #include <linux/netfilter.h>
 26 #include <linux/module.h>
 27 #include <linux/cache.h>
 28 #include <linux/cpu.h>
 29 #include <linux/audit.h>
 30 #include <linux/rhashtable.h>
 31 #include <linux/if_tunnel.h>
 32 #include <net/dst.h>
 33 #include <net/flow.h>
 34 #include <net/xfrm.h>
 35 #include <net/ip.h>
 36 #if IS_ENABLED(CONFIG_IPV6_MIP6)
 37 #include <net/mip6.h>
 38 #endif
 39 #ifdef CONFIG_XFRM_STATISTICS
 40 #include <net/snmp.h>
 41 #endif
 42 
 43 #include "xfrm_hash.h"
 44 
 45 #define XFRM_QUEUE_TMO_MIN ((unsigned)(HZ/10))
 46 #define XFRM_QUEUE_TMO_MAX ((unsigned)(60*HZ))
 47 #define XFRM_MAX_QUEUE_LEN      100
 48 
 49 struct xfrm_flo {
 50         struct dst_entry *dst_orig;
 51         u8 flags;
 52 };
 53 
 54 /* prefixes smaller than this are stored in lists, not trees. */
 55 #define INEXACT_PREFIXLEN_IPV4  16
 56 #define INEXACT_PREFIXLEN_IPV6  48
 57 
 58 struct xfrm_pol_inexact_node {
 59         struct rb_node node;
 60         union {
 61                 xfrm_address_t addr;
 62                 struct rcu_head rcu;
 63         };
 64         u8 prefixlen;
 65 
 66         struct rb_root root;
 67 
 68         /* the policies matching this node, can be empty list */
 69         struct hlist_head hhead;
 70 };
 71 
 72 /* xfrm inexact policy search tree:
 73  * xfrm_pol_inexact_bin = hash(dir,type,family,if_id);
 74  *  |
 75  * +---- root_d: sorted by daddr:prefix
 76  * |                 |
 77  * |        xfrm_pol_inexact_node
 78  * |                 |
 79  * |                 +- root: sorted by saddr/prefix
 80  * |                 |              |
 81  * |                 |         xfrm_pol_inexact_node
 82  * |                 |              |
 83  * |                 |              + root: unused
 84  * |                 |              |
 85  * |                 |              + hhead: saddr:daddr policies
 86  * |                 |
 87  * |                 +- coarse policies and all any:daddr policies
 88  * |
 89  * +---- root_s: sorted by saddr:prefix
 90  * |                 |
 91  * |        xfrm_pol_inexact_node
 92  * |                 |
 93  * |                 + root: unused
 94  * |                 |
 95  * |                 + hhead: saddr:any policies
 96  * |
 97  * +---- coarse policies and all any:any policies
 98  *
 99  * Lookups return four candidate lists:
100  * 1. any:any list from top-level xfrm_pol_inexact_bin
101  * 2. any:daddr list from daddr tree
102  * 3. saddr:daddr list from 2nd level daddr tree
103  * 4. saddr:any list from saddr tree
104  *
105  * This result set then needs to be searched for the policy with
106  * the lowest priority.  If two results have same prio, youngest one wins.
107  */
108 
109 struct xfrm_pol_inexact_key {
110         possible_net_t net;
111         u32 if_id;
112         u16 family;
113         u8 dir, type;
114 };
115 
116 struct xfrm_pol_inexact_bin {
117         struct xfrm_pol_inexact_key k;
118         struct rhash_head head;
119         /* list containing '*:*' policies */
120         struct hlist_head hhead;
121 
122         seqcount_t count;
123         /* tree sorted by daddr/prefix */
124         struct rb_root root_d;
125 
126         /* tree sorted by saddr/prefix */
127         struct rb_root root_s;
128 
129         /* slow path below */
130         struct list_head inexact_bins;
131         struct rcu_head rcu;
132 };
133 
134 enum xfrm_pol_inexact_candidate_type {
135         XFRM_POL_CAND_BOTH,
136         XFRM_POL_CAND_SADDR,
137         XFRM_POL_CAND_DADDR,
138         XFRM_POL_CAND_ANY,
139 
140         XFRM_POL_CAND_MAX,
141 };
142 
143 struct xfrm_pol_inexact_candidates {
144         struct hlist_head *res[XFRM_POL_CAND_MAX];
145 };
146 
147 static DEFINE_SPINLOCK(xfrm_if_cb_lock);
148 static struct xfrm_if_cb const __rcu *xfrm_if_cb __read_mostly;
149 
150 static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock);
151 static struct xfrm_policy_afinfo const __rcu *xfrm_policy_afinfo[AF_INET6 + 1]
152                                                 __read_mostly;
153 
154 static struct kmem_cache *xfrm_dst_cache __ro_after_init;
155 static __read_mostly seqcount_t xfrm_policy_hash_generation;
156 
157 static struct rhashtable xfrm_policy_inexact_table;
158 static const struct rhashtable_params xfrm_pol_inexact_params;
159 
160 static void xfrm_init_pmtu(struct xfrm_dst **bundle, int nr);
161 static int stale_bundle(struct dst_entry *dst);
162 static int xfrm_bundle_ok(struct xfrm_dst *xdst);
163 static void xfrm_policy_queue_process(struct timer_list *t);
164 
165 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir);
166 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
167                                                 int dir);
168 
169 static struct xfrm_pol_inexact_bin *
170 xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family, u8 dir,
171                            u32 if_id);
172 
173 static struct xfrm_pol_inexact_bin *
174 xfrm_policy_inexact_lookup_rcu(struct net *net,
175                                u8 type, u16 family, u8 dir, u32 if_id);
176 static struct xfrm_policy *
177 xfrm_policy_insert_list(struct hlist_head *chain, struct xfrm_policy *policy,
178                         bool excl);
179 static void xfrm_policy_insert_inexact_list(struct hlist_head *chain,
180                                             struct xfrm_policy *policy);
181 
182 static bool
183 xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates *cand,
184                                     struct xfrm_pol_inexact_bin *b,
185                                     const xfrm_address_t *saddr,
186                                     const xfrm_address_t *daddr);
187 
188 static inline bool xfrm_pol_hold_rcu(struct xfrm_policy *policy)
189 {
190         return refcount_inc_not_zero(&policy->refcnt);
191 }
192 
193 static inline bool
194 __xfrm4_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
195 {
196         const struct flowi4 *fl4 = &fl->u.ip4;
197 
198         return  addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) &&
199                 addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) &&
200                 !((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) &&
201                 !((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) &&
202                 (fl4->flowi4_proto == sel->proto || !sel->proto) &&
203                 (fl4->flowi4_oif == sel->ifindex || !sel->ifindex);
204 }
205 
206 static inline bool
207 __xfrm6_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
208 {
209         const struct flowi6 *fl6 = &fl->u.ip6;
210 
211         return  addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) &&
212                 addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) &&
213                 !((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) &&
214                 !((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) &&
215                 (fl6->flowi6_proto == sel->proto || !sel->proto) &&
216                 (fl6->flowi6_oif == sel->ifindex || !sel->ifindex);
217 }
218 
219 bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl,
220                          unsigned short family)
221 {
222         switch (family) {
223         case AF_INET:
224                 return __xfrm4_selector_match(sel, fl);
225         case AF_INET6:
226                 return __xfrm6_selector_match(sel, fl);
227         }
228         return false;
229 }
230 
231 static const struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family)
232 {
233         const struct xfrm_policy_afinfo *afinfo;
234 
235         if (unlikely(family >= ARRAY_SIZE(xfrm_policy_afinfo)))
236                 return NULL;
237         rcu_read_lock();
238         afinfo = rcu_dereference(xfrm_policy_afinfo[family]);
239         if (unlikely(!afinfo))
240                 rcu_read_unlock();
241         return afinfo;
242 }
243 
244 /* Called with rcu_read_lock(). */
245 static const struct xfrm_if_cb *xfrm_if_get_cb(void)
246 {
247         return rcu_dereference(xfrm_if_cb);
248 }
249 
250 struct dst_entry *__xfrm_dst_lookup(struct net *net, int tos, int oif,
251                                     const xfrm_address_t *saddr,
252                                     const xfrm_address_t *daddr,
253                                     int family, u32 mark)
254 {
255         const struct xfrm_policy_afinfo *afinfo;
256         struct dst_entry *dst;
257 
258         afinfo = xfrm_policy_get_afinfo(family);
259         if (unlikely(afinfo == NULL))
260                 return ERR_PTR(-EAFNOSUPPORT);
261 
262         dst = afinfo->dst_lookup(net, tos, oif, saddr, daddr, mark);
263 
264         rcu_read_unlock();
265 
266         return dst;
267 }
268 EXPORT_SYMBOL(__xfrm_dst_lookup);
269 
270 static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x,
271                                                 int tos, int oif,
272                                                 xfrm_address_t *prev_saddr,
273                                                 xfrm_address_t *prev_daddr,
274                                                 int family, u32 mark)
275 {
276         struct net *net = xs_net(x);
277         xfrm_address_t *saddr = &x->props.saddr;
278         xfrm_address_t *daddr = &x->id.daddr;
279         struct dst_entry *dst;
280 
281         if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) {
282                 saddr = x->coaddr;
283                 daddr = prev_daddr;
284         }
285         if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) {
286                 saddr = prev_saddr;
287                 daddr = x->coaddr;
288         }
289 
290         dst = __xfrm_dst_lookup(net, tos, oif, saddr, daddr, family, mark);
291 
292         if (!IS_ERR(dst)) {
293                 if (prev_saddr != saddr)
294                         memcpy(prev_saddr, saddr,  sizeof(*prev_saddr));
295                 if (prev_daddr != daddr)
296                         memcpy(prev_daddr, daddr,  sizeof(*prev_daddr));
297         }
298 
299         return dst;
300 }
301 
302 static inline unsigned long make_jiffies(long secs)
303 {
304         if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ)
305                 return MAX_SCHEDULE_TIMEOUT-1;
306         else
307                 return secs*HZ;
308 }
309 
310 static void xfrm_policy_timer(struct timer_list *t)
311 {
312         struct xfrm_policy *xp = from_timer(xp, t, timer);
313         time64_t now = ktime_get_real_seconds();
314         time64_t next = TIME64_MAX;
315         int warn = 0;
316         int dir;
317 
318         read_lock(&xp->lock);
319 
320         if (unlikely(xp->walk.dead))
321                 goto out;
322 
323         dir = xfrm_policy_id2dir(xp->index);
324 
325         if (xp->lft.hard_add_expires_seconds) {
326                 time64_t tmo = xp->lft.hard_add_expires_seconds +
327                         xp->curlft.add_time - now;
328                 if (tmo <= 0)
329                         goto expired;
330                 if (tmo < next)
331                         next = tmo;
332         }
333         if (xp->lft.hard_use_expires_seconds) {
334                 time64_t tmo = xp->lft.hard_use_expires_seconds +
335                         (xp->curlft.use_time ? : xp->curlft.add_time) - now;
336                 if (tmo <= 0)
337                         goto expired;
338                 if (tmo < next)
339                         next = tmo;
340         }
341         if (xp->lft.soft_add_expires_seconds) {
342                 time64_t tmo = xp->lft.soft_add_expires_seconds +
343                         xp->curlft.add_time - now;
344                 if (tmo <= 0) {
345                         warn = 1;
346                         tmo = XFRM_KM_TIMEOUT;
347                 }
348                 if (tmo < next)
349                         next = tmo;
350         }
351         if (xp->lft.soft_use_expires_seconds) {
352                 time64_t tmo = xp->lft.soft_use_expires_seconds +
353                         (xp->curlft.use_time ? : xp->curlft.add_time) - now;
354                 if (tmo <= 0) {
355                         warn = 1;
356                         tmo = XFRM_KM_TIMEOUT;
357                 }
358                 if (tmo < next)
359                         next = tmo;
360         }
361 
362         if (warn)
363                 km_policy_expired(xp, dir, 0, 0);
364         if (next != TIME64_MAX &&
365             !mod_timer(&xp->timer, jiffies + make_jiffies(next)))
366                 xfrm_pol_hold(xp);
367 
368 out:
369         read_unlock(&xp->lock);
370         xfrm_pol_put(xp);
371         return;
372 
373 expired:
374         read_unlock(&xp->lock);
375         if (!xfrm_policy_delete(xp, dir))
376                 km_policy_expired(xp, dir, 1, 0);
377         xfrm_pol_put(xp);
378 }
379 
380 /* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2
381  * SPD calls.
382  */
383 
384 struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp)
385 {
386         struct xfrm_policy *policy;
387 
388         policy = kzalloc(sizeof(struct xfrm_policy), gfp);
389 
390         if (policy) {
391                 write_pnet(&policy->xp_net, net);
392                 INIT_LIST_HEAD(&policy->walk.all);
393                 INIT_HLIST_NODE(&policy->bydst_inexact_list);
394                 INIT_HLIST_NODE(&policy->bydst);
395                 INIT_HLIST_NODE(&policy->byidx);
396                 rwlock_init(&policy->lock);
397                 refcount_set(&policy->refcnt, 1);
398                 skb_queue_head_init(&policy->polq.hold_queue);
399                 timer_setup(&policy->timer, xfrm_policy_timer, 0);
400                 timer_setup(&policy->polq.hold_timer,
401                             xfrm_policy_queue_process, 0);
402         }
403         return policy;
404 }
405 EXPORT_SYMBOL(xfrm_policy_alloc);
406 
407 static void xfrm_policy_destroy_rcu(struct rcu_head *head)
408 {
409         struct xfrm_policy *policy = container_of(head, struct xfrm_policy, rcu);
410 
411         security_xfrm_policy_free(policy->security);
412         kfree(policy);
413 }
414 
415 /* Destroy xfrm_policy: descendant resources must be released to this moment. */
416 
417 void xfrm_policy_destroy(struct xfrm_policy *policy)
418 {
419         BUG_ON(!policy->walk.dead);
420 
421         if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer))
422                 BUG();
423 
424         call_rcu(&policy->rcu, xfrm_policy_destroy_rcu);
425 }
426 EXPORT_SYMBOL(xfrm_policy_destroy);
427 
428 /* Rule must be locked. Release descendant resources, announce
429  * entry dead. The rule must be unlinked from lists to the moment.
430  */
431 
432 static void xfrm_policy_kill(struct xfrm_policy *policy)
433 {
434         write_lock_bh(&policy->lock);
435         policy->walk.dead = 1;
436         write_unlock_bh(&policy->lock);
437 
438         atomic_inc(&policy->genid);
439 
440         if (del_timer(&policy->polq.hold_timer))
441                 xfrm_pol_put(policy);
442         skb_queue_purge(&policy->polq.hold_queue);
443 
444         if (del_timer(&policy->timer))
445                 xfrm_pol_put(policy);
446 
447         xfrm_pol_put(policy);
448 }
449 
450 static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024;
451 
452 static inline unsigned int idx_hash(struct net *net, u32 index)
453 {
454         return __idx_hash(index, net->xfrm.policy_idx_hmask);
455 }
456 
457 /* calculate policy hash thresholds */
458 static void __get_hash_thresh(struct net *net,
459                               unsigned short family, int dir,
460                               u8 *dbits, u8 *sbits)
461 {
462         switch (family) {
463         case AF_INET:
464                 *dbits = net->xfrm.policy_bydst[dir].dbits4;
465                 *sbits = net->xfrm.policy_bydst[dir].sbits4;
466                 break;
467 
468         case AF_INET6:
469                 *dbits = net->xfrm.policy_bydst[dir].dbits6;
470                 *sbits = net->xfrm.policy_bydst[dir].sbits6;
471                 break;
472 
473         default:
474                 *dbits = 0;
475                 *sbits = 0;
476         }
477 }
478 
479 static struct hlist_head *policy_hash_bysel(struct net *net,
480                                             const struct xfrm_selector *sel,
481                                             unsigned short family, int dir)
482 {
483         unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
484         unsigned int hash;
485         u8 dbits;
486         u8 sbits;
487 
488         __get_hash_thresh(net, family, dir, &dbits, &sbits);
489         hash = __sel_hash(sel, family, hmask, dbits, sbits);
490 
491         if (hash == hmask + 1)
492                 return NULL;
493 
494         return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
495                      lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
496 }
497 
498 static struct hlist_head *policy_hash_direct(struct net *net,
499                                              const xfrm_address_t *daddr,
500                                              const xfrm_address_t *saddr,
501                                              unsigned short family, int dir)
502 {
503         unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
504         unsigned int hash;
505         u8 dbits;
506         u8 sbits;
507 
508         __get_hash_thresh(net, family, dir, &dbits, &sbits);
509         hash = __addr_hash(daddr, saddr, family, hmask, dbits, sbits);
510 
511         return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
512                      lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
513 }
514 
515 static void xfrm_dst_hash_transfer(struct net *net,
516                                    struct hlist_head *list,
517                                    struct hlist_head *ndsttable,
518                                    unsigned int nhashmask,
519                                    int dir)
520 {
521         struct hlist_node *tmp, *entry0 = NULL;
522         struct xfrm_policy *pol;
523         unsigned int h0 = 0;
524         u8 dbits;
525         u8 sbits;
526 
527 redo:
528         hlist_for_each_entry_safe(pol, tmp, list, bydst) {
529                 unsigned int h;
530 
531                 __get_hash_thresh(net, pol->family, dir, &dbits, &sbits);
532                 h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,
533                                 pol->family, nhashmask, dbits, sbits);
534                 if (!entry0) {
535                         hlist_del_rcu(&pol->bydst);
536                         hlist_add_head_rcu(&pol->bydst, ndsttable + h);
537                         h0 = h;
538                 } else {
539                         if (h != h0)
540                                 continue;
541                         hlist_del_rcu(&pol->bydst);
542                         hlist_add_behind_rcu(&pol->bydst, entry0);
543                 }
544                 entry0 = &pol->bydst;
545         }
546         if (!hlist_empty(list)) {
547                 entry0 = NULL;
548                 goto redo;
549         }
550 }
551 
552 static void xfrm_idx_hash_transfer(struct hlist_head *list,
553                                    struct hlist_head *nidxtable,
554                                    unsigned int nhashmask)
555 {
556         struct hlist_node *tmp;
557         struct xfrm_policy *pol;
558 
559         hlist_for_each_entry_safe(pol, tmp, list, byidx) {
560                 unsigned int h;
561 
562                 h = __idx_hash(pol->index, nhashmask);
563                 hlist_add_head(&pol->byidx, nidxtable+h);
564         }
565 }
566 
567 static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
568 {
569         return ((old_hmask + 1) << 1) - 1;
570 }
571 
572 static void xfrm_bydst_resize(struct net *net, int dir)
573 {
574         unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
575         unsigned int nhashmask = xfrm_new_hash_mask(hmask);
576         unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
577         struct hlist_head *ndst = xfrm_hash_alloc(nsize);
578         struct hlist_head *odst;
579         int i;
580 
581         if (!ndst)
582                 return;
583 
584         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
585         write_seqcount_begin(&xfrm_policy_hash_generation);
586 
587         odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table,
588                                 lockdep_is_held(&net->xfrm.xfrm_policy_lock));
589 
590         for (i = hmask; i >= 0; i--)
591                 xfrm_dst_hash_transfer(net, odst + i, ndst, nhashmask, dir);
592 
593         rcu_assign_pointer(net->xfrm.policy_bydst[dir].table, ndst);
594         net->xfrm.policy_bydst[dir].hmask = nhashmask;
595 
596         write_seqcount_end(&xfrm_policy_hash_generation);
597         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
598 
599         synchronize_rcu();
600 
601         xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
602 }
603 
604 static void xfrm_byidx_resize(struct net *net, int total)
605 {
606         unsigned int hmask = net->xfrm.policy_idx_hmask;
607         unsigned int nhashmask = xfrm_new_hash_mask(hmask);
608         unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
609         struct hlist_head *oidx = net->xfrm.policy_byidx;
610         struct hlist_head *nidx = xfrm_hash_alloc(nsize);
611         int i;
612 
613         if (!nidx)
614                 return;
615 
616         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
617 
618         for (i = hmask; i >= 0; i--)
619                 xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);
620 
621         net->xfrm.policy_byidx = nidx;
622         net->xfrm.policy_idx_hmask = nhashmask;
623 
624         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
625 
626         xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
627 }
628 
629 static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total)
630 {
631         unsigned int cnt = net->xfrm.policy_count[dir];
632         unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
633 
634         if (total)
635                 *total += cnt;
636 
637         if ((hmask + 1) < xfrm_policy_hashmax &&
638             cnt > hmask)
639                 return 1;
640 
641         return 0;
642 }
643 
644 static inline int xfrm_byidx_should_resize(struct net *net, int total)
645 {
646         unsigned int hmask = net->xfrm.policy_idx_hmask;
647 
648         if ((hmask + 1) < xfrm_policy_hashmax &&
649             total > hmask)
650                 return 1;
651 
652         return 0;
653 }
654 
655 void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si)
656 {
657         si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN];
658         si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT];
659         si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD];
660         si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX];
661         si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX];
662         si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX];
663         si->spdhcnt = net->xfrm.policy_idx_hmask;
664         si->spdhmcnt = xfrm_policy_hashmax;
665 }
666 EXPORT_SYMBOL(xfrm_spd_getinfo);
667 
668 static DEFINE_MUTEX(hash_resize_mutex);
669 static void xfrm_hash_resize(struct work_struct *work)
670 {
671         struct net *net = container_of(work, struct net, xfrm.policy_hash_work);
672         int dir, total;
673 
674         mutex_lock(&hash_resize_mutex);
675 
676         total = 0;
677         for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
678                 if (xfrm_bydst_should_resize(net, dir, &total))
679                         xfrm_bydst_resize(net, dir);
680         }
681         if (xfrm_byidx_should_resize(net, total))
682                 xfrm_byidx_resize(net, total);
683 
684         mutex_unlock(&hash_resize_mutex);
685 }
686 
687 /* Make sure *pol can be inserted into fastbin.
688  * Useful to check that later insert requests will be sucessful
689  * (provided xfrm_policy_lock is held throughout).
690  */
691 static struct xfrm_pol_inexact_bin *
692 xfrm_policy_inexact_alloc_bin(const struct xfrm_policy *pol, u8 dir)
693 {
694         struct xfrm_pol_inexact_bin *bin, *prev;
695         struct xfrm_pol_inexact_key k = {
696                 .family = pol->family,
697                 .type = pol->type,
698                 .dir = dir,
699                 .if_id = pol->if_id,
700         };
701         struct net *net = xp_net(pol);
702 
703         lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
704 
705         write_pnet(&k.net, net);
706         bin = rhashtable_lookup_fast(&xfrm_policy_inexact_table, &k,
707                                      xfrm_pol_inexact_params);
708         if (bin)
709                 return bin;
710 
711         bin = kzalloc(sizeof(*bin), GFP_ATOMIC);
712         if (!bin)
713                 return NULL;
714 
715         bin->k = k;
716         INIT_HLIST_HEAD(&bin->hhead);
717         bin->root_d = RB_ROOT;
718         bin->root_s = RB_ROOT;
719         seqcount_init(&bin->count);
720 
721         prev = rhashtable_lookup_get_insert_key(&xfrm_policy_inexact_table,
722                                                 &bin->k, &bin->head,
723                                                 xfrm_pol_inexact_params);
724         if (!prev) {
725                 list_add(&bin->inexact_bins, &net->xfrm.inexact_bins);
726                 return bin;
727         }
728 
729         kfree(bin);
730 
731         return IS_ERR(prev) ? NULL : prev;
732 }
733 
734 static bool xfrm_pol_inexact_addr_use_any_list(const xfrm_address_t *addr,
735                                                int family, u8 prefixlen)
736 {
737         if (xfrm_addr_any(addr, family))
738                 return true;
739 
740         if (family == AF_INET6 && prefixlen < INEXACT_PREFIXLEN_IPV6)
741                 return true;
742 
743         if (family == AF_INET && prefixlen < INEXACT_PREFIXLEN_IPV4)
744                 return true;
745 
746         return false;
747 }
748 
749 static bool
750 xfrm_policy_inexact_insert_use_any_list(const struct xfrm_policy *policy)
751 {
752         const xfrm_address_t *addr;
753         bool saddr_any, daddr_any;
754         u8 prefixlen;
755 
756         addr = &policy->selector.saddr;
757         prefixlen = policy->selector.prefixlen_s;
758 
759         saddr_any = xfrm_pol_inexact_addr_use_any_list(addr,
760                                                        policy->family,
761                                                        prefixlen);
762         addr = &policy->selector.daddr;
763         prefixlen = policy->selector.prefixlen_d;
764         daddr_any = xfrm_pol_inexact_addr_use_any_list(addr,
765                                                        policy->family,
766                                                        prefixlen);
767         return saddr_any && daddr_any;
768 }
769 
770 static void xfrm_pol_inexact_node_init(struct xfrm_pol_inexact_node *node,
771                                        const xfrm_address_t *addr, u8 prefixlen)
772 {
773         node->addr = *addr;
774         node->prefixlen = prefixlen;
775 }
776 
777 static struct xfrm_pol_inexact_node *
778 xfrm_pol_inexact_node_alloc(const xfrm_address_t *addr, u8 prefixlen)
779 {
780         struct xfrm_pol_inexact_node *node;
781 
782         node = kzalloc(sizeof(*node), GFP_ATOMIC);
783         if (node)
784                 xfrm_pol_inexact_node_init(node, addr, prefixlen);
785 
786         return node;
787 }
788 
789 static int xfrm_policy_addr_delta(const xfrm_address_t *a,
790                                   const xfrm_address_t *b,
791                                   u8 prefixlen, u16 family)
792 {
793         unsigned int pdw, pbi;
794         int delta = 0;
795 
796         switch (family) {
797         case AF_INET:
798                 if (sizeof(long) == 4 && prefixlen == 0)
799                         return ntohl(a->a4) - ntohl(b->a4);
800                 return (ntohl(a->a4) & ((~0UL << (32 - prefixlen)))) -
801                        (ntohl(b->a4) & ((~0UL << (32 - prefixlen))));
802         case AF_INET6:
803                 pdw = prefixlen >> 5;
804                 pbi = prefixlen & 0x1f;
805 
806                 if (pdw) {
807                         delta = memcmp(a->a6, b->a6, pdw << 2);
808                         if (delta)
809                                 return delta;
810                 }
811                 if (pbi) {
812                         u32 mask = ~0u << (32 - pbi);
813 
814                         delta = (ntohl(a->a6[pdw]) & mask) -
815                                 (ntohl(b->a6[pdw]) & mask);
816                 }
817                 break;
818         default:
819                 break;
820         }
821 
822         return delta;
823 }
824 
825 static void xfrm_policy_inexact_list_reinsert(struct net *net,
826                                               struct xfrm_pol_inexact_node *n,
827                                               u16 family)
828 {
829         unsigned int matched_s, matched_d;
830         struct xfrm_policy *policy, *p;
831 
832         matched_s = 0;
833         matched_d = 0;
834 
835         list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
836                 struct hlist_node *newpos = NULL;
837                 bool matches_s, matches_d;
838 
839                 if (!policy->bydst_reinsert)
840                         continue;
841 
842                 WARN_ON_ONCE(policy->family != family);
843 
844                 policy->bydst_reinsert = false;
845                 hlist_for_each_entry(p, &n->hhead, bydst) {
846                         if (policy->priority > p->priority)
847                                 newpos = &p->bydst;
848                         else if (policy->priority == p->priority &&
849                                  policy->pos > p->pos)
850                                 newpos = &p->bydst;
851                         else
852                                 break;
853                 }
854 
855                 if (newpos)
856                         hlist_add_behind_rcu(&policy->bydst, newpos);
857                 else
858                         hlist_add_head_rcu(&policy->bydst, &n->hhead);
859 
860                 /* paranoia checks follow.
861                  * Check that the reinserted policy matches at least
862                  * saddr or daddr for current node prefix.
863                  *
864                  * Matching both is fine, matching saddr in one policy
865                  * (but not daddr) and then matching only daddr in another
866                  * is a bug.
867                  */
868                 matches_s = xfrm_policy_addr_delta(&policy->selector.saddr,
869                                                    &n->addr,
870                                                    n->prefixlen,
871                                                    family) == 0;
872                 matches_d = xfrm_policy_addr_delta(&policy->selector.daddr,
873                                                    &n->addr,
874                                                    n->prefixlen,
875                                                    family) == 0;
876                 if (matches_s && matches_d)
877                         continue;
878 
879                 WARN_ON_ONCE(!matches_s && !matches_d);
880                 if (matches_s)
881                         matched_s++;
882                 if (matches_d)
883                         matched_d++;
884                 WARN_ON_ONCE(matched_s && matched_d);
885         }
886 }
887 
888 static void xfrm_policy_inexact_node_reinsert(struct net *net,
889                                               struct xfrm_pol_inexact_node *n,
890                                               struct rb_root *new,
891                                               u16 family)
892 {
893         struct xfrm_pol_inexact_node *node;
894         struct rb_node **p, *parent;
895 
896         /* we should not have another subtree here */
897         WARN_ON_ONCE(!RB_EMPTY_ROOT(&n->root));
898 restart:
899         parent = NULL;
900         p = &new->rb_node;
901         while (*p) {
902                 u8 prefixlen;
903                 int delta;
904 
905                 parent = *p;
906                 node = rb_entry(*p, struct xfrm_pol_inexact_node, node);
907 
908                 prefixlen = min(node->prefixlen, n->prefixlen);
909 
910                 delta = xfrm_policy_addr_delta(&n->addr, &node->addr,
911                                                prefixlen, family);
912                 if (delta < 0) {
913                         p = &parent->rb_left;
914                 } else if (delta > 0) {
915                         p = &parent->rb_right;
916                 } else {
917                         bool same_prefixlen = node->prefixlen == n->prefixlen;
918                         struct xfrm_policy *tmp;
919 
920                         hlist_for_each_entry(tmp, &n->hhead, bydst) {
921                                 tmp->bydst_reinsert = true;
922                                 hlist_del_rcu(&tmp->bydst);
923                         }
924 
925                         node->prefixlen = prefixlen;
926 
927                         xfrm_policy_inexact_list_reinsert(net, node, family);
928 
929                         if (same_prefixlen) {
930                                 kfree_rcu(n, rcu);
931                                 return;
932                         }
933 
934                         rb_erase(*p, new);
935                         kfree_rcu(n, rcu);
936                         n = node;
937                         goto restart;
938                 }
939         }
940 
941         rb_link_node_rcu(&n->node, parent, p);
942         rb_insert_color(&n->node, new);
943 }
944 
945 /* merge nodes v and n */
946 static void xfrm_policy_inexact_node_merge(struct net *net,
947                                            struct xfrm_pol_inexact_node *v,
948                                            struct xfrm_pol_inexact_node *n,
949                                            u16 family)
950 {
951         struct xfrm_pol_inexact_node *node;
952         struct xfrm_policy *tmp;
953         struct rb_node *rnode;
954 
955         /* To-be-merged node v has a subtree.
956          *
957          * Dismantle it and insert its nodes to n->root.
958          */
959         while ((rnode = rb_first(&v->root)) != NULL) {
960                 node = rb_entry(rnode, struct xfrm_pol_inexact_node, node);
961                 rb_erase(&node->node, &v->root);
962                 xfrm_policy_inexact_node_reinsert(net, node, &n->root,
963                                                   family);
964         }
965 
966         hlist_for_each_entry(tmp, &v->hhead, bydst) {
967                 tmp->bydst_reinsert = true;
968                 hlist_del_rcu(&tmp->bydst);
969         }
970 
971         xfrm_policy_inexact_list_reinsert(net, n, family);
972 }
973 
974 static struct xfrm_pol_inexact_node *
975 xfrm_policy_inexact_insert_node(struct net *net,
976                                 struct rb_root *root,
977                                 xfrm_address_t *addr,
978                                 u16 family, u8 prefixlen, u8 dir)
979 {
980         struct xfrm_pol_inexact_node *cached = NULL;
981         struct rb_node **p, *parent = NULL;
982         struct xfrm_pol_inexact_node *node;
983 
984         p = &root->rb_node;
985         while (*p) {
986                 int delta;
987 
988                 parent = *p;
989                 node = rb_entry(*p, struct xfrm_pol_inexact_node, node);
990 
991                 delta = xfrm_policy_addr_delta(addr, &node->addr,
992                                                node->prefixlen,
993                                                family);
994                 if (delta == 0 && prefixlen >= node->prefixlen) {
995                         WARN_ON_ONCE(cached); /* ipsec policies got lost */
996                         return node;
997                 }
998 
999                 if (delta < 0)
1000                         p = &parent->rb_left;
1001                 else
1002                         p = &parent->rb_right;
1003 
1004                 if (prefixlen < node->prefixlen) {
1005                         delta = xfrm_policy_addr_delta(addr, &node->addr,
1006                                                        prefixlen,
1007                                                        family);
1008                         if (delta)
1009                                 continue;
1010 
1011                         /* This node is a subnet of the new prefix. It needs
1012                          * to be removed and re-inserted with the smaller
1013                          * prefix and all nodes that are now also covered
1014                          * by the reduced prefixlen.
1015                          */
1016                         rb_erase(&node->node, root);
1017 
1018                         if (!cached) {
1019                                 xfrm_pol_inexact_node_init(node, addr,
1020                                                            prefixlen);
1021                                 cached = node;
1022                         } else {
1023                                 /* This node also falls within the new
1024                                  * prefixlen. Merge the to-be-reinserted
1025                                  * node and this one.
1026                                  */
1027                                 xfrm_policy_inexact_node_merge(net, node,
1028                                                                cached, family);
1029                                 kfree_rcu(node, rcu);
1030                         }
1031 
1032                         /* restart */
1033                         p = &root->rb_node;
1034                         parent = NULL;
1035                 }
1036         }
1037 
1038         node = cached;
1039         if (!node) {
1040                 node = xfrm_pol_inexact_node_alloc(addr, prefixlen);
1041                 if (!node)
1042                         return NULL;
1043         }
1044 
1045         rb_link_node_rcu(&node->node, parent, p);
1046         rb_insert_color(&node->node, root);
1047 
1048         return node;
1049 }
1050 
1051 static void xfrm_policy_inexact_gc_tree(struct rb_root *r, bool rm)
1052 {
1053         struct xfrm_pol_inexact_node *node;
1054         struct rb_node *rn = rb_first(r);
1055 
1056         while (rn) {
1057                 node = rb_entry(rn, struct xfrm_pol_inexact_node, node);
1058 
1059                 xfrm_policy_inexact_gc_tree(&node->root, rm);
1060                 rn = rb_next(rn);
1061 
1062                 if (!hlist_empty(&node->hhead) || !RB_EMPTY_ROOT(&node->root)) {
1063                         WARN_ON_ONCE(rm);
1064                         continue;
1065                 }
1066 
1067                 rb_erase(&node->node, r);
1068                 kfree_rcu(node, rcu);
1069         }
1070 }
1071 
1072 static void __xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin *b, bool net_exit)
1073 {
1074         write_seqcount_begin(&b->count);
1075         xfrm_policy_inexact_gc_tree(&b->root_d, net_exit);
1076         xfrm_policy_inexact_gc_tree(&b->root_s, net_exit);
1077         write_seqcount_end(&b->count);
1078 
1079         if (!RB_EMPTY_ROOT(&b->root_d) || !RB_EMPTY_ROOT(&b->root_s) ||
1080             !hlist_empty(&b->hhead)) {
1081                 WARN_ON_ONCE(net_exit);
1082                 return;
1083         }
1084 
1085         if (rhashtable_remove_fast(&xfrm_policy_inexact_table, &b->head,
1086                                    xfrm_pol_inexact_params) == 0) {
1087                 list_del(&b->inexact_bins);
1088                 kfree_rcu(b, rcu);
1089         }
1090 }
1091 
1092 static void xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin *b)
1093 {
1094         struct net *net = read_pnet(&b->k.net);
1095 
1096         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1097         __xfrm_policy_inexact_prune_bin(b, false);
1098         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1099 }
1100 
1101 static void __xfrm_policy_inexact_flush(struct net *net)
1102 {
1103         struct xfrm_pol_inexact_bin *bin, *t;
1104 
1105         lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
1106 
1107         list_for_each_entry_safe(bin, t, &net->xfrm.inexact_bins, inexact_bins)
1108                 __xfrm_policy_inexact_prune_bin(bin, false);
1109 }
1110 
1111 static struct hlist_head *
1112 xfrm_policy_inexact_alloc_chain(struct xfrm_pol_inexact_bin *bin,
1113                                 struct xfrm_policy *policy, u8 dir)
1114 {
1115         struct xfrm_pol_inexact_node *n;
1116         struct net *net;
1117 
1118         net = xp_net(policy);
1119         lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
1120 
1121         if (xfrm_policy_inexact_insert_use_any_list(policy))
1122                 return &bin->hhead;
1123 
1124         if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.daddr,
1125                                                policy->family,
1126                                                policy->selector.prefixlen_d)) {
1127                 write_seqcount_begin(&bin->count);
1128                 n = xfrm_policy_inexact_insert_node(net,
1129                                                     &bin->root_s,
1130                                                     &policy->selector.saddr,
1131                                                     policy->family,
1132                                                     policy->selector.prefixlen_s,
1133                                                     dir);
1134                 write_seqcount_end(&bin->count);
1135                 if (!n)
1136                         return NULL;
1137 
1138                 return &n->hhead;
1139         }
1140 
1141         /* daddr is fixed */
1142         write_seqcount_begin(&bin->count);
1143         n = xfrm_policy_inexact_insert_node(net,
1144                                             &bin->root_d,
1145                                             &policy->selector.daddr,
1146                                             policy->family,
1147                                             policy->selector.prefixlen_d, dir);
1148         write_seqcount_end(&bin->count);
1149         if (!n)
1150                 return NULL;
1151 
1152         /* saddr is wildcard */
1153         if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.saddr,
1154                                                policy->family,
1155                                                policy->selector.prefixlen_s))
1156                 return &n->hhead;
1157 
1158         write_seqcount_begin(&bin->count);
1159         n = xfrm_policy_inexact_insert_node(net,
1160                                             &n->root,
1161                                             &policy->selector.saddr,
1162                                             policy->family,
1163                                             policy->selector.prefixlen_s, dir);
1164         write_seqcount_end(&bin->count);
1165         if (!n)
1166                 return NULL;
1167 
1168         return &n->hhead;
1169 }
1170 
1171 static struct xfrm_policy *
1172 xfrm_policy_inexact_insert(struct xfrm_policy *policy, u8 dir, int excl)
1173 {
1174         struct xfrm_pol_inexact_bin *bin;
1175         struct xfrm_policy *delpol;
1176         struct hlist_head *chain;
1177         struct net *net;
1178 
1179         bin = xfrm_policy_inexact_alloc_bin(policy, dir);
1180         if (!bin)
1181                 return ERR_PTR(-ENOMEM);
1182 
1183         net = xp_net(policy);
1184         lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
1185 
1186         chain = xfrm_policy_inexact_alloc_chain(bin, policy, dir);
1187         if (!chain) {
1188                 __xfrm_policy_inexact_prune_bin(bin, false);
1189                 return ERR_PTR(-ENOMEM);
1190         }
1191 
1192         delpol = xfrm_policy_insert_list(chain, policy, excl);
1193         if (delpol && excl) {
1194                 __xfrm_policy_inexact_prune_bin(bin, false);
1195                 return ERR_PTR(-EEXIST);
1196         }
1197 
1198         chain = &net->xfrm.policy_inexact[dir];
1199         xfrm_policy_insert_inexact_list(chain, policy);
1200 
1201         if (delpol)
1202                 __xfrm_policy_inexact_prune_bin(bin, false);
1203 
1204         return delpol;
1205 }
1206 
1207 static void xfrm_hash_rebuild(struct work_struct *work)
1208 {
1209         struct net *net = container_of(work, struct net,
1210                                        xfrm.policy_hthresh.work);
1211         unsigned int hmask;
1212         struct xfrm_policy *pol;
1213         struct xfrm_policy *policy;
1214         struct hlist_head *chain;
1215         struct hlist_head *odst;
1216         struct hlist_node *newpos;
1217         int i;
1218         int dir;
1219         unsigned seq;
1220         u8 lbits4, rbits4, lbits6, rbits6;
1221 
1222         mutex_lock(&hash_resize_mutex);
1223 
1224         /* read selector prefixlen thresholds */
1225         do {
1226                 seq = read_seqbegin(&net->xfrm.policy_hthresh.lock);
1227 
1228                 lbits4 = net->xfrm.policy_hthresh.lbits4;
1229                 rbits4 = net->xfrm.policy_hthresh.rbits4;
1230                 lbits6 = net->xfrm.policy_hthresh.lbits6;
1231                 rbits6 = net->xfrm.policy_hthresh.rbits6;
1232         } while (read_seqretry(&net->xfrm.policy_hthresh.lock, seq));
1233 
1234         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1235         write_seqcount_begin(&xfrm_policy_hash_generation);
1236 
1237         /* make sure that we can insert the indirect policies again before
1238          * we start with destructive action.
1239          */
1240         list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) {
1241                 struct xfrm_pol_inexact_bin *bin;
1242                 u8 dbits, sbits;
1243 
1244                 dir = xfrm_policy_id2dir(policy->index);
1245                 if (policy->walk.dead || dir >= XFRM_POLICY_MAX)
1246                         continue;
1247 
1248                 if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
1249                         if (policy->family == AF_INET) {
1250                                 dbits = rbits4;
1251                                 sbits = lbits4;
1252                         } else {
1253                                 dbits = rbits6;
1254                                 sbits = lbits6;
1255                         }
1256                 } else {
1257                         if (policy->family == AF_INET) {
1258                                 dbits = lbits4;
1259                                 sbits = rbits4;
1260                         } else {
1261                                 dbits = lbits6;
1262                                 sbits = rbits6;
1263                         }
1264                 }
1265 
1266                 if (policy->selector.prefixlen_d < dbits ||
1267                     policy->selector.prefixlen_s < sbits)
1268                         continue;
1269 
1270                 bin = xfrm_policy_inexact_alloc_bin(policy, dir);
1271                 if (!bin)
1272                         goto out_unlock;
1273 
1274                 if (!xfrm_policy_inexact_alloc_chain(bin, policy, dir))
1275                         goto out_unlock;
1276         }
1277 
1278         /* reset the bydst and inexact table in all directions */
1279         for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
1280                 struct hlist_node *n;
1281 
1282                 hlist_for_each_entry_safe(policy, n,
1283                                           &net->xfrm.policy_inexact[dir],
1284                                           bydst_inexact_list) {
1285                         hlist_del_rcu(&policy->bydst);
1286                         hlist_del_init(&policy->bydst_inexact_list);
1287                 }
1288 
1289                 hmask = net->xfrm.policy_bydst[dir].hmask;
1290                 odst = net->xfrm.policy_bydst[dir].table;
1291                 for (i = hmask; i >= 0; i--) {
1292                         hlist_for_each_entry_safe(policy, n, odst + i, bydst)
1293                                 hlist_del_rcu(&policy->bydst);
1294                 }
1295                 if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
1296                         /* dir out => dst = remote, src = local */
1297                         net->xfrm.policy_bydst[dir].dbits4 = rbits4;
1298                         net->xfrm.policy_bydst[dir].sbits4 = lbits4;
1299                         net->xfrm.policy_bydst[dir].dbits6 = rbits6;
1300                         net->xfrm.policy_bydst[dir].sbits6 = lbits6;
1301                 } else {
1302                         /* dir in/fwd => dst = local, src = remote */
1303                         net->xfrm.policy_bydst[dir].dbits4 = lbits4;
1304                         net->xfrm.policy_bydst[dir].sbits4 = rbits4;
1305                         net->xfrm.policy_bydst[dir].dbits6 = lbits6;
1306                         net->xfrm.policy_bydst[dir].sbits6 = rbits6;
1307                 }
1308         }
1309 
1310         /* re-insert all policies by order of creation */
1311         list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
1312                 if (policy->walk.dead)
1313                         continue;
1314                 dir = xfrm_policy_id2dir(policy->index);
1315                 if (dir >= XFRM_POLICY_MAX) {
1316                         /* skip socket policies */
1317                         continue;
1318                 }
1319                 newpos = NULL;
1320                 chain = policy_hash_bysel(net, &policy->selector,
1321                                           policy->family, dir);
1322 
1323                 if (!chain) {
1324                         void *p = xfrm_policy_inexact_insert(policy, dir, 0);
1325 
1326                         WARN_ONCE(IS_ERR(p), "reinsert: %ld\n", PTR_ERR(p));
1327                         continue;
1328                 }
1329 
1330                 hlist_for_each_entry(pol, chain, bydst) {
1331                         if (policy->priority >= pol->priority)
1332                                 newpos = &pol->bydst;
1333                         else
1334                                 break;
1335                 }
1336                 if (newpos)
1337                         hlist_add_behind_rcu(&policy->bydst, newpos);
1338                 else
1339                         hlist_add_head_rcu(&policy->bydst, chain);
1340         }
1341 
1342 out_unlock:
1343         __xfrm_policy_inexact_flush(net);
1344         write_seqcount_end(&xfrm_policy_hash_generation);
1345         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1346 
1347         mutex_unlock(&hash_resize_mutex);
1348 }
1349 
1350 void xfrm_policy_hash_rebuild(struct net *net)
1351 {
1352         schedule_work(&net->xfrm.policy_hthresh.work);
1353 }
1354 EXPORT_SYMBOL(xfrm_policy_hash_rebuild);
1355 
1356 /* Generate new index... KAME seems to generate them ordered by cost
1357  * of an absolute inpredictability of ordering of rules. This will not pass. */
1358 static u32 xfrm_gen_index(struct net *net, int dir, u32 index)
1359 {
1360         static u32 idx_generator;
1361 
1362         for (;;) {
1363                 struct hlist_head *list;
1364                 struct xfrm_policy *p;
1365                 u32 idx;
1366                 int found;
1367 
1368                 if (!index) {
1369                         idx = (idx_generator | dir);
1370                         idx_generator += 8;
1371                 } else {
1372                         idx = index;
1373                         index = 0;
1374                 }
1375 
1376                 if (idx == 0)
1377                         idx = 8;
1378                 list = net->xfrm.policy_byidx + idx_hash(net, idx);
1379                 found = 0;
1380                 hlist_for_each_entry(p, list, byidx) {
1381                         if (p->index == idx) {
1382                                 found = 1;
1383                                 break;
1384                         }
1385                 }
1386                 if (!found)
1387                         return idx;
1388         }
1389 }
1390 
1391 static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2)
1392 {
1393         u32 *p1 = (u32 *) s1;
1394         u32 *p2 = (u32 *) s2;
1395         int len = sizeof(struct xfrm_selector) / sizeof(u32);
1396         int i;
1397 
1398         for (i = 0; i < len; i++) {
1399                 if (p1[i] != p2[i])
1400                         return 1;
1401         }
1402 
1403         return 0;
1404 }
1405 
1406 static void xfrm_policy_requeue(struct xfrm_policy *old,
1407                                 struct xfrm_policy *new)
1408 {
1409         struct xfrm_policy_queue *pq = &old->polq;
1410         struct sk_buff_head list;
1411 
1412         if (skb_queue_empty(&pq->hold_queue))
1413                 return;
1414 
1415         __skb_queue_head_init(&list);
1416 
1417         spin_lock_bh(&pq->hold_queue.lock);
1418         skb_queue_splice_init(&pq->hold_queue, &list);
1419         if (del_timer(&pq->hold_timer))
1420                 xfrm_pol_put(old);
1421         spin_unlock_bh(&pq->hold_queue.lock);
1422 
1423         pq = &new->polq;
1424 
1425         spin_lock_bh(&pq->hold_queue.lock);
1426         skb_queue_splice(&list, &pq->hold_queue);
1427         pq->timeout = XFRM_QUEUE_TMO_MIN;
1428         if (!mod_timer(&pq->hold_timer, jiffies))
1429                 xfrm_pol_hold(new);
1430         spin_unlock_bh(&pq->hold_queue.lock);
1431 }
1432 
1433 static bool xfrm_policy_mark_match(struct xfrm_policy *policy,
1434                                    struct xfrm_policy *pol)
1435 {
1436         u32 mark = policy->mark.v & policy->mark.m;
1437 
1438         if (policy->mark.v == pol->mark.v && policy->mark.m == pol->mark.m)
1439                 return true;
1440 
1441         if ((mark & pol->mark.m) == pol->mark.v &&
1442             policy->priority == pol->priority)
1443                 return true;
1444 
1445         return false;
1446 }
1447 
1448 static u32 xfrm_pol_bin_key(const void *data, u32 len, u32 seed)
1449 {
1450         const struct xfrm_pol_inexact_key *k = data;
1451         u32 a = k->type << 24 | k->dir << 16 | k->family;
1452 
1453         return jhash_3words(a, k->if_id, net_hash_mix(read_pnet(&k->net)),
1454                             seed);
1455 }
1456 
1457 static u32 xfrm_pol_bin_obj(const void *data, u32 len, u32 seed)
1458 {
1459         const struct xfrm_pol_inexact_bin *b = data;
1460 
1461         return xfrm_pol_bin_key(&b->k, 0, seed);
1462 }
1463 
1464 static int xfrm_pol_bin_cmp(struct rhashtable_compare_arg *arg,
1465                             const void *ptr)
1466 {
1467         const struct xfrm_pol_inexact_key *key = arg->key;
1468         const struct xfrm_pol_inexact_bin *b = ptr;
1469         int ret;
1470 
1471         if (!net_eq(read_pnet(&b->k.net), read_pnet(&key->net)))
1472                 return -1;
1473 
1474         ret = b->k.dir ^ key->dir;
1475         if (ret)
1476                 return ret;
1477 
1478         ret = b->k.type ^ key->type;
1479         if (ret)
1480                 return ret;
1481 
1482         ret = b->k.family ^ key->family;
1483         if (ret)
1484                 return ret;
1485 
1486         return b->k.if_id ^ key->if_id;
1487 }
1488 
1489 static const struct rhashtable_params xfrm_pol_inexact_params = {
1490         .head_offset            = offsetof(struct xfrm_pol_inexact_bin, head),
1491         .hashfn                 = xfrm_pol_bin_key,
1492         .obj_hashfn             = xfrm_pol_bin_obj,
1493         .obj_cmpfn              = xfrm_pol_bin_cmp,
1494         .automatic_shrinking    = true,
1495 };
1496 
1497 static void xfrm_policy_insert_inexact_list(struct hlist_head *chain,
1498                                             struct xfrm_policy *policy)
1499 {
1500         struct xfrm_policy *pol, *delpol = NULL;
1501         struct hlist_node *newpos = NULL;
1502         int i = 0;
1503 
1504         hlist_for_each_entry(pol, chain, bydst_inexact_list) {
1505                 if (pol->type == policy->type &&
1506                     pol->if_id == policy->if_id &&
1507                     !selector_cmp(&pol->selector, &policy->selector) &&
1508                     xfrm_policy_mark_match(policy, pol) &&
1509                     xfrm_sec_ctx_match(pol->security, policy->security) &&
1510                     !WARN_ON(delpol)) {
1511                         delpol = pol;
1512                         if (policy->priority > pol->priority)
1513                                 continue;
1514                 } else if (policy->priority >= pol->priority) {
1515                         newpos = &pol->bydst_inexact_list;
1516                         continue;
1517                 }
1518                 if (delpol)
1519                         break;
1520         }
1521 
1522         if (newpos)
1523                 hlist_add_behind_rcu(&policy->bydst_inexact_list, newpos);
1524         else
1525                 hlist_add_head_rcu(&policy->bydst_inexact_list, chain);
1526 
1527         hlist_for_each_entry(pol, chain, bydst_inexact_list) {
1528                 pol->pos = i;
1529                 i++;
1530         }
1531 }
1532 
1533 static struct xfrm_policy *xfrm_policy_insert_list(struct hlist_head *chain,
1534                                                    struct xfrm_policy *policy,
1535                                                    bool excl)
1536 {
1537         struct xfrm_policy *pol, *newpos = NULL, *delpol = NULL;
1538 
1539         hlist_for_each_entry(pol, chain, bydst) {
1540                 if (pol->type == policy->type &&
1541                     pol->if_id == policy->if_id &&
1542                     !selector_cmp(&pol->selector, &policy->selector) &&
1543                     xfrm_policy_mark_match(policy, pol) &&
1544                     xfrm_sec_ctx_match(pol->security, policy->security) &&
1545                     !WARN_ON(delpol)) {
1546                         if (excl)
1547                                 return ERR_PTR(-EEXIST);
1548                         delpol = pol;
1549                         if (policy->priority > pol->priority)
1550                                 continue;
1551                 } else if (policy->priority >= pol->priority) {
1552                         newpos = pol;
1553                         continue;
1554                 }
1555                 if (delpol)
1556                         break;
1557         }
1558 
1559         if (newpos)
1560                 hlist_add_behind_rcu(&policy->bydst, &newpos->bydst);
1561         else
1562                 hlist_add_head_rcu(&policy->bydst, chain);
1563 
1564         return delpol;
1565 }
1566 
1567 int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
1568 {
1569         struct net *net = xp_net(policy);
1570         struct xfrm_policy *delpol;
1571         struct hlist_head *chain;
1572 
1573         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1574         chain = policy_hash_bysel(net, &policy->selector, policy->family, dir);
1575         if (chain)
1576                 delpol = xfrm_policy_insert_list(chain, policy, excl);
1577         else
1578                 delpol = xfrm_policy_inexact_insert(policy, dir, excl);
1579 
1580         if (IS_ERR(delpol)) {
1581                 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1582                 return PTR_ERR(delpol);
1583         }
1584 
1585         __xfrm_policy_link(policy, dir);
1586 
1587         /* After previous checking, family can either be AF_INET or AF_INET6 */
1588         if (policy->family == AF_INET)
1589                 rt_genid_bump_ipv4(net);
1590         else
1591                 rt_genid_bump_ipv6(net);
1592 
1593         if (delpol) {
1594                 xfrm_policy_requeue(delpol, policy);
1595                 __xfrm_policy_unlink(delpol, dir);
1596         }
1597         policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir, policy->index);
1598         hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index));
1599         policy->curlft.add_time = ktime_get_real_seconds();
1600         policy->curlft.use_time = 0;
1601         if (!mod_timer(&policy->timer, jiffies + HZ))
1602                 xfrm_pol_hold(policy);
1603         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1604 
1605         if (delpol)
1606                 xfrm_policy_kill(delpol);
1607         else if (xfrm_bydst_should_resize(net, dir, NULL))
1608                 schedule_work(&net->xfrm.policy_hash_work);
1609 
1610         return 0;
1611 }
1612 EXPORT_SYMBOL(xfrm_policy_insert);
1613 
1614 static struct xfrm_policy *
1615 __xfrm_policy_bysel_ctx(struct hlist_head *chain, u32 mark, u32 if_id,
1616                         u8 type, int dir,
1617                         struct xfrm_selector *sel,
1618                         struct xfrm_sec_ctx *ctx)
1619 {
1620         struct xfrm_policy *pol;
1621 
1622         if (!chain)
1623                 return NULL;
1624 
1625         hlist_for_each_entry(pol, chain, bydst) {
1626                 if (pol->type == type &&
1627                     pol->if_id == if_id &&
1628                     (mark & pol->mark.m) == pol->mark.v &&
1629                     !selector_cmp(sel, &pol->selector) &&
1630                     xfrm_sec_ctx_match(ctx, pol->security))
1631                         return pol;
1632         }
1633 
1634         return NULL;
1635 }
1636 
1637 struct xfrm_policy *xfrm_policy_bysel_ctx(struct net *net, u32 mark, u32 if_id,
1638                                           u8 type, int dir,
1639                                           struct xfrm_selector *sel,
1640                                           struct xfrm_sec_ctx *ctx, int delete,
1641                                           int *err)
1642 {
1643         struct xfrm_pol_inexact_bin *bin = NULL;
1644         struct xfrm_policy *pol, *ret = NULL;
1645         struct hlist_head *chain;
1646 
1647         *err = 0;
1648         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1649         chain = policy_hash_bysel(net, sel, sel->family, dir);
1650         if (!chain) {
1651                 struct xfrm_pol_inexact_candidates cand;
1652                 int i;
1653 
1654                 bin = xfrm_policy_inexact_lookup(net, type,
1655                                                  sel->family, dir, if_id);
1656                 if (!bin) {
1657                         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1658                         return NULL;
1659                 }
1660 
1661                 if (!xfrm_policy_find_inexact_candidates(&cand, bin,
1662                                                          &sel->saddr,
1663                                                          &sel->daddr)) {
1664                         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1665                         return NULL;
1666                 }
1667 
1668                 pol = NULL;
1669                 for (i = 0; i < ARRAY_SIZE(cand.res); i++) {
1670                         struct xfrm_policy *tmp;
1671 
1672                         tmp = __xfrm_policy_bysel_ctx(cand.res[i], mark,
1673                                                       if_id, type, dir,
1674                                                       sel, ctx);
1675                         if (!tmp)
1676                                 continue;
1677 
1678                         if (!pol || tmp->pos < pol->pos)
1679                                 pol = tmp;
1680                 }
1681         } else {
1682                 pol = __xfrm_policy_bysel_ctx(chain, mark, if_id, type, dir,
1683                                               sel, ctx);
1684         }
1685 
1686         if (pol) {
1687                 xfrm_pol_hold(pol);
1688                 if (delete) {
1689                         *err = security_xfrm_policy_delete(pol->security);
1690                         if (*err) {
1691                                 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1692                                 return pol;
1693                         }
1694                         __xfrm_policy_unlink(pol, dir);
1695                 }
1696                 ret = pol;
1697         }
1698         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1699 
1700         if (ret && delete)
1701                 xfrm_policy_kill(ret);
1702         if (bin && delete)
1703                 xfrm_policy_inexact_prune_bin(bin);
1704         return ret;
1705 }
1706 EXPORT_SYMBOL(xfrm_policy_bysel_ctx);
1707 
1708 struct xfrm_policy *xfrm_policy_byid(struct net *net, u32 mark, u32 if_id,
1709                                      u8 type, int dir, u32 id, int delete,
1710                                      int *err)
1711 {
1712         struct xfrm_policy *pol, *ret;
1713         struct hlist_head *chain;
1714 
1715         *err = -ENOENT;
1716         if (xfrm_policy_id2dir(id) != dir)
1717                 return NULL;
1718 
1719         *err = 0;
1720         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1721         chain = net->xfrm.policy_byidx + idx_hash(net, id);
1722         ret = NULL;
1723         hlist_for_each_entry(pol, chain, byidx) {
1724                 if (pol->type == type && pol->index == id &&
1725                     pol->if_id == if_id &&
1726                     (mark & pol->mark.m) == pol->mark.v) {
1727                         xfrm_pol_hold(pol);
1728                         if (delete) {
1729                                 *err = security_xfrm_policy_delete(
1730                                                                 pol->security);
1731                                 if (*err) {
1732                                         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1733                                         return pol;
1734                                 }
1735                                 __xfrm_policy_unlink(pol, dir);
1736                         }
1737                         ret = pol;
1738                         break;
1739                 }
1740         }
1741         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1742 
1743         if (ret && delete)
1744                 xfrm_policy_kill(ret);
1745         return ret;
1746 }
1747 EXPORT_SYMBOL(xfrm_policy_byid);
1748 
1749 #ifdef CONFIG_SECURITY_NETWORK_XFRM
1750 static inline int
1751 xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
1752 {
1753         struct xfrm_policy *pol;
1754         int err = 0;
1755 
1756         list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1757                 if (pol->walk.dead ||
1758                     xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX ||
1759                     pol->type != type)
1760                         continue;
1761 
1762                 err = security_xfrm_policy_delete(pol->security);
1763                 if (err) {
1764                         xfrm_audit_policy_delete(pol, 0, task_valid);
1765                         return err;
1766                 }
1767         }
1768         return err;
1769 }
1770 #else
1771 static inline int
1772 xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
1773 {
1774         return 0;
1775 }
1776 #endif
1777 
1778 int xfrm_policy_flush(struct net *net, u8 type, bool task_valid)
1779 {
1780         int dir, err = 0, cnt = 0;
1781         struct xfrm_policy *pol;
1782 
1783         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1784 
1785         err = xfrm_policy_flush_secctx_check(net, type, task_valid);
1786         if (err)
1787                 goto out;
1788 
1789 again:
1790         list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1791                 dir = xfrm_policy_id2dir(pol->index);
1792                 if (pol->walk.dead ||
1793                     dir >= XFRM_POLICY_MAX ||
1794                     pol->type != type)
1795                         continue;
1796 
1797                 __xfrm_policy_unlink(pol, dir);
1798                 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1799                 cnt++;
1800                 xfrm_audit_policy_delete(pol, 1, task_valid);
1801                 xfrm_policy_kill(pol);
1802                 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1803                 goto again;
1804         }
1805         if (cnt)
1806                 __xfrm_policy_inexact_flush(net);
1807         else
1808                 err = -ESRCH;
1809 out:
1810         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1811         return err;
1812 }
1813 EXPORT_SYMBOL(xfrm_policy_flush);
1814 
1815 int xfrm_policy_walk(struct net *net, struct xfrm_policy_walk *walk,
1816                      int (*func)(struct xfrm_policy *, int, int, void*),
1817                      void *data)
1818 {
1819         struct xfrm_policy *pol;
1820         struct xfrm_policy_walk_entry *x;
1821         int error = 0;
1822 
1823         if (walk->type >= XFRM_POLICY_TYPE_MAX &&
1824             walk->type != XFRM_POLICY_TYPE_ANY)
1825                 return -EINVAL;
1826 
1827         if (list_empty(&walk->walk.all) && walk->seq != 0)
1828                 return 0;
1829 
1830         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1831         if (list_empty(&walk->walk.all))
1832                 x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all);
1833         else
1834                 x = list_first_entry(&walk->walk.all,
1835                                      struct xfrm_policy_walk_entry, all);
1836 
1837         list_for_each_entry_from(x, &net->xfrm.policy_all, all) {
1838                 if (x->dead)
1839                         continue;
1840                 pol = container_of(x, struct xfrm_policy, walk);
1841                 if (walk->type != XFRM_POLICY_TYPE_ANY &&
1842                     walk->type != pol->type)
1843                         continue;
1844                 error = func(pol, xfrm_policy_id2dir(pol->index),
1845                              walk->seq, data);
1846                 if (error) {
1847                         list_move_tail(&walk->walk.all, &x->all);
1848                         goto out;
1849                 }
1850                 walk->seq++;
1851         }
1852         if (walk->seq == 0) {
1853                 error = -ENOENT;
1854                 goto out;
1855         }
1856         list_del_init(&walk->walk.all);
1857 out:
1858         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1859         return error;
1860 }
1861 EXPORT_SYMBOL(xfrm_policy_walk);
1862 
1863 void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type)
1864 {
1865         INIT_LIST_HEAD(&walk->walk.all);
1866         walk->walk.dead = 1;
1867         walk->type = type;
1868         walk->seq = 0;
1869 }
1870 EXPORT_SYMBOL(xfrm_policy_walk_init);
1871 
1872 void xfrm_policy_walk_done(struct xfrm_policy_walk *walk, struct net *net)
1873 {
1874         if (list_empty(&walk->walk.all))
1875                 return;
1876 
1877         spin_lock_bh(&net->xfrm.xfrm_policy_lock); /*FIXME where is net? */
1878         list_del(&walk->walk.all);
1879         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1880 }
1881 EXPORT_SYMBOL(xfrm_policy_walk_done);
1882 
1883 /*
1884  * Find policy to apply to this flow.
1885  *
1886  * Returns 0 if policy found, else an -errno.
1887  */
1888 static int xfrm_policy_match(const struct xfrm_policy *pol,
1889                              const struct flowi *fl,
1890                              u8 type, u16 family, int dir, u32 if_id)
1891 {
1892         const struct xfrm_selector *sel = &pol->selector;
1893         int ret = -ESRCH;
1894         bool match;
1895 
1896         if (pol->family != family ||
1897             pol->if_id != if_id ||
1898             (fl->flowi_mark & pol->mark.m) != pol->mark.v ||
1899             pol->type != type)
1900                 return ret;
1901 
1902         match = xfrm_selector_match(sel, fl, family);
1903         if (match)
1904                 ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid,
1905                                                   dir);
1906         return ret;
1907 }
1908 
1909 static struct xfrm_pol_inexact_node *
1910 xfrm_policy_lookup_inexact_addr(const struct rb_root *r,
1911                                 seqcount_t *count,
1912                                 const xfrm_address_t *addr, u16 family)
1913 {
1914         const struct rb_node *parent;
1915         int seq;
1916 
1917 again:
1918         seq = read_seqcount_begin(count);
1919 
1920         parent = rcu_dereference_raw(r->rb_node);
1921         while (parent) {
1922                 struct xfrm_pol_inexact_node *node;
1923                 int delta;
1924 
1925                 node = rb_entry(parent, struct xfrm_pol_inexact_node, node);
1926 
1927                 delta = xfrm_policy_addr_delta(addr, &node->addr,
1928                                                node->prefixlen, family);
1929                 if (delta < 0) {
1930                         parent = rcu_dereference_raw(parent->rb_left);
1931                         continue;
1932                 } else if (delta > 0) {
1933                         parent = rcu_dereference_raw(parent->rb_right);
1934                         continue;
1935                 }
1936 
1937                 return node;
1938         }
1939 
1940         if (read_seqcount_retry(count, seq))
1941                 goto again;
1942 
1943         return NULL;
1944 }
1945 
1946 static bool
1947 xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates *cand,
1948                                     struct xfrm_pol_inexact_bin *b,
1949                                     const xfrm_address_t *saddr,
1950                                     const xfrm_address_t *daddr)
1951 {
1952         struct xfrm_pol_inexact_node *n;
1953         u16 family;
1954 
1955         if (!b)
1956                 return false;
1957 
1958         family = b->k.family;
1959         memset(cand, 0, sizeof(*cand));
1960         cand->res[XFRM_POL_CAND_ANY] = &b->hhead;
1961 
1962         n = xfrm_policy_lookup_inexact_addr(&b->root_d, &b->count, daddr,
1963                                             family);
1964         if (n) {
1965                 cand->res[XFRM_POL_CAND_DADDR] = &n->hhead;
1966                 n = xfrm_policy_lookup_inexact_addr(&n->root, &b->count, saddr,
1967                                                     family);
1968                 if (n)
1969                         cand->res[XFRM_POL_CAND_BOTH] = &n->hhead;
1970         }
1971 
1972         n = xfrm_policy_lookup_inexact_addr(&b->root_s, &b->count, saddr,
1973                                             family);
1974         if (n)
1975                 cand->res[XFRM_POL_CAND_SADDR] = &n->hhead;
1976 
1977         return true;
1978 }
1979 
1980 static struct xfrm_pol_inexact_bin *
1981 xfrm_policy_inexact_lookup_rcu(struct net *net, u8 type, u16 family,
1982                                u8 dir, u32 if_id)
1983 {
1984         struct xfrm_pol_inexact_key k = {
1985                 .family = family,
1986                 .type = type,
1987                 .dir = dir,
1988                 .if_id = if_id,
1989         };
1990 
1991         write_pnet(&k.net, net);
1992 
1993         return rhashtable_lookup(&xfrm_policy_inexact_table, &k,
1994                                  xfrm_pol_inexact_params);
1995 }
1996 
1997 static struct xfrm_pol_inexact_bin *
1998 xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family,
1999                            u8 dir, u32 if_id)
2000 {
2001         struct xfrm_pol_inexact_bin *bin;
2002 
2003         lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
2004 
2005         rcu_read_lock();
2006         bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id);
2007         rcu_read_unlock();
2008 
2009         return bin;
2010 }
2011 
2012 static struct xfrm_policy *
2013 __xfrm_policy_eval_candidates(struct hlist_head *chain,
2014                               struct xfrm_policy *prefer,
2015                               const struct flowi *fl,
2016                               u8 type, u16 family, int dir, u32 if_id)
2017 {
2018         u32 priority = prefer ? prefer->priority : ~0u;
2019         struct xfrm_policy *pol;
2020 
2021         if (!chain)
2022                 return NULL;
2023 
2024         hlist_for_each_entry_rcu(pol, chain, bydst) {
2025                 int err;
2026 
2027                 if (pol->priority > priority)
2028                         break;
2029 
2030                 err = xfrm_policy_match(pol, fl, type, family, dir, if_id);
2031                 if (err) {
2032                         if (err != -ESRCH)
2033                                 return ERR_PTR(err);
2034 
2035                         continue;
2036                 }
2037 
2038                 if (prefer) {
2039                         /* matches.  Is it older than *prefer? */
2040                         if (pol->priority == priority &&
2041                             prefer->pos < pol->pos)
2042                                 return prefer;
2043                 }
2044 
2045                 return pol;
2046         }
2047 
2048         return NULL;
2049 }
2050 
2051 static struct xfrm_policy *
2052 xfrm_policy_eval_candidates(struct xfrm_pol_inexact_candidates *cand,
2053                             struct xfrm_policy *prefer,
2054                             const struct flowi *fl,
2055                             u8 type, u16 family, int dir, u32 if_id)
2056 {
2057         struct xfrm_policy *tmp;
2058         int i;
2059 
2060         for (i = 0; i < ARRAY_SIZE(cand->res); i++) {
2061                 tmp = __xfrm_policy_eval_candidates(cand->res[i],
2062                                                     prefer,
2063                                                     fl, type, family, dir,
2064                                                     if_id);
2065                 if (!tmp)
2066                         continue;
2067 
2068                 if (IS_ERR(tmp))
2069                         return tmp;
2070                 prefer = tmp;
2071         }
2072 
2073         return prefer;
2074 }
2075 
2076 static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type,
2077                                                      const struct flowi *fl,
2078                                                      u16 family, u8 dir,
2079                                                      u32 if_id)
2080 {
2081         struct xfrm_pol_inexact_candidates cand;
2082         const xfrm_address_t *daddr, *saddr;
2083         struct xfrm_pol_inexact_bin *bin;
2084         struct xfrm_policy *pol, *ret;
2085         struct hlist_head *chain;
2086         unsigned int sequence;
2087         int err;
2088 
2089         daddr = xfrm_flowi_daddr(fl, family);
2090         saddr = xfrm_flowi_saddr(fl, family);
2091         if (unlikely(!daddr || !saddr))
2092                 return NULL;
2093 
2094         rcu_read_lock();
2095  retry:
2096         do {
2097                 sequence = read_seqcount_begin(&xfrm_policy_hash_generation);
2098                 chain = policy_hash_direct(net, daddr, saddr, family, dir);
2099         } while (read_seqcount_retry(&xfrm_policy_hash_generation, sequence));
2100 
2101         ret = NULL;
2102         hlist_for_each_entry_rcu(pol, chain, bydst) {
2103                 err = xfrm_policy_match(pol, fl, type, family, dir, if_id);
2104                 if (err) {
2105                         if (err == -ESRCH)
2106                                 continue;
2107                         else {
2108                                 ret = ERR_PTR(err);
2109                                 goto fail;
2110                         }
2111                 } else {
2112                         ret = pol;
2113                         break;
2114                 }
2115         }
2116         bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id);
2117         if (!bin || !xfrm_policy_find_inexact_candidates(&cand, bin, saddr,
2118                                                          daddr))
2119                 goto skip_inexact;
2120 
2121         pol = xfrm_policy_eval_candidates(&cand, ret, fl, type,
2122                                           family, dir, if_id);
2123         if (pol) {
2124                 ret = pol;
2125                 if (IS_ERR(pol))
2126                         goto fail;
2127         }
2128 
2129 skip_inexact:
2130         if (read_seqcount_retry(&xfrm_policy_hash_generation, sequence))
2131                 goto retry;
2132 
2133         if (ret && !xfrm_pol_hold_rcu(ret))
2134                 goto retry;
2135 fail:
2136         rcu_read_unlock();
2137 
2138         return ret;
2139 }
2140 
2141 static struct xfrm_policy *xfrm_policy_lookup(struct net *net,
2142                                               const struct flowi *fl,
2143                                               u16 family, u8 dir, u32 if_id)
2144 {
2145 #ifdef CONFIG_XFRM_SUB_POLICY
2146         struct xfrm_policy *pol;
2147 
2148         pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family,
2149                                         dir, if_id);
2150         if (pol != NULL)
2151                 return pol;
2152 #endif
2153         return xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, fl, family,
2154                                          dir, if_id);
2155 }
2156 
2157 static struct xfrm_policy *xfrm_sk_policy_lookup(const struct sock *sk, int dir,
2158                                                  const struct flowi *fl,
2159                                                  u16 family, u32 if_id)
2160 {
2161         struct xfrm_policy *pol;
2162 
2163         rcu_read_lock();
2164  again:
2165         pol = rcu_dereference(sk->sk_policy[dir]);
2166         if (pol != NULL) {
2167                 bool match;
2168                 int err = 0;
2169 
2170                 if (pol->family != family) {
2171                         pol = NULL;
2172                         goto out;
2173                 }
2174 
2175                 match = xfrm_selector_match(&pol->selector, fl, family);
2176                 if (match) {
2177                         if ((sk->sk_mark & pol->mark.m) != pol->mark.v ||
2178                             pol->if_id != if_id) {
2179                                 pol = NULL;
2180                                 goto out;
2181                         }
2182                         err = security_xfrm_policy_lookup(pol->security,
2183                                                       fl->flowi_secid,
2184                                                       dir);
2185                         if (!err) {
2186                                 if (!xfrm_pol_hold_rcu(pol))
2187                                         goto again;
2188                         } else if (err == -ESRCH) {
2189                                 pol = NULL;
2190                         } else {
2191                                 pol = ERR_PTR(err);
2192                         }
2193                 } else
2194                         pol = NULL;
2195         }
2196 out:
2197         rcu_read_unlock();
2198         return pol;
2199 }
2200 
2201 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)
2202 {
2203         struct net *net = xp_net(pol);
2204 
2205         list_add(&pol->walk.all, &net->xfrm.policy_all);
2206         net->xfrm.policy_count[dir]++;
2207         xfrm_pol_hold(pol);
2208 }
2209 
2210 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
2211                                                 int dir)
2212 {
2213         struct net *net = xp_net(pol);
2214 
2215         if (list_empty(&pol->walk.all))
2216                 return NULL;
2217 
2218         /* Socket policies are not hashed. */
2219         if (!hlist_unhashed(&pol->bydst)) {
2220                 hlist_del_rcu(&pol->bydst);
2221                 hlist_del_init(&pol->bydst_inexact_list);
2222                 hlist_del(&pol->byidx);
2223         }
2224 
2225         list_del_init(&pol->walk.all);
2226         net->xfrm.policy_count[dir]--;
2227 
2228         return pol;
2229 }
2230 
2231 static void xfrm_sk_policy_link(struct xfrm_policy *pol, int dir)
2232 {
2233         __xfrm_policy_link(pol, XFRM_POLICY_MAX + dir);
2234 }
2235 
2236 static void xfrm_sk_policy_unlink(struct xfrm_policy *pol, int dir)
2237 {
2238         __xfrm_policy_unlink(pol, XFRM_POLICY_MAX + dir);
2239 }
2240 
2241 int xfrm_policy_delete(struct xfrm_policy *pol, int dir)
2242 {
2243         struct net *net = xp_net(pol);
2244 
2245         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
2246         pol = __xfrm_policy_unlink(pol, dir);
2247         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
2248         if (pol) {
2249                 xfrm_policy_kill(pol);
2250                 return 0;
2251         }
2252         return -ENOENT;
2253 }
2254 EXPORT_SYMBOL(xfrm_policy_delete);
2255 
2256 int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)
2257 {
2258         struct net *net = sock_net(sk);
2259         struct xfrm_policy *old_pol;
2260 
2261 #ifdef CONFIG_XFRM_SUB_POLICY
2262         if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)
2263                 return -EINVAL;
2264 #endif
2265 
2266         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
2267         old_pol = rcu_dereference_protected(sk->sk_policy[dir],
2268                                 lockdep_is_held(&net->xfrm.xfrm_policy_lock));
2269         if (pol) {
2270                 pol->curlft.add_time = ktime_get_real_seconds();
2271                 pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0);
2272                 xfrm_sk_policy_link(pol, dir);
2273         }
2274         rcu_assign_pointer(sk->sk_policy[dir], pol);
2275         if (old_pol) {
2276                 if (pol)
2277                         xfrm_policy_requeue(old_pol, pol);
2278 
2279                 /* Unlinking succeeds always. This is the only function
2280                  * allowed to delete or replace socket policy.
2281                  */
2282                 xfrm_sk_policy_unlink(old_pol, dir);
2283         }
2284         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
2285 
2286         if (old_pol) {
2287                 xfrm_policy_kill(old_pol);
2288         }
2289         return 0;
2290 }
2291 
2292 static struct xfrm_policy *clone_policy(const struct xfrm_policy *old, int dir)
2293 {
2294         struct xfrm_policy *newp = xfrm_policy_alloc(xp_net(old), GFP_ATOMIC);
2295         struct net *net = xp_net(old);
2296 
2297         if (newp) {
2298                 newp->selector = old->selector;
2299                 if (security_xfrm_policy_clone(old->security,
2300                                                &newp->security)) {
2301                         kfree(newp);
2302                         return NULL;  /* ENOMEM */
2303                 }
2304                 newp->lft = old->lft;
2305                 newp->curlft = old->curlft;
2306                 newp->mark = old->mark;
2307                 newp->if_id = old->if_id;
2308                 newp->action = old->action;
2309                 newp->flags = old->flags;
2310                 newp->xfrm_nr = old->xfrm_nr;
2311                 newp->index = old->index;
2312                 newp->type = old->type;
2313                 newp->family = old->family;
2314                 memcpy(newp->xfrm_vec, old->xfrm_vec,
2315                        newp->xfrm_nr*sizeof(struct xfrm_tmpl));
2316                 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
2317                 xfrm_sk_policy_link(newp, dir);
2318                 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
2319                 xfrm_pol_put(newp);
2320         }
2321         return newp;
2322 }
2323 
2324 int __xfrm_sk_clone_policy(struct sock *sk, const struct sock *osk)
2325 {
2326         const struct xfrm_policy *p;
2327         struct xfrm_policy *np;
2328         int i, ret = 0;
2329 
2330         rcu_read_lock();
2331         for (i = 0; i < 2; i++) {
2332                 p = rcu_dereference(osk->sk_policy[i]);
2333                 if (p) {
2334                         np = clone_policy(p, i);
2335                         if (unlikely(!np)) {
2336                                 ret = -ENOMEM;
2337                                 break;
2338                         }
2339                         rcu_assign_pointer(sk->sk_policy[i], np);
2340                 }
2341         }
2342         rcu_read_unlock();
2343         return ret;
2344 }
2345 
2346 static int
2347 xfrm_get_saddr(struct net *net, int oif, xfrm_address_t *local,
2348                xfrm_address_t *remote, unsigned short family, u32 mark)
2349 {
2350         int err;
2351         const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2352 
2353         if (unlikely(afinfo == NULL))
2354                 return -EINVAL;
2355         err = afinfo->get_saddr(net, oif, local, remote, mark);
2356         rcu_read_unlock();
2357         return err;
2358 }
2359 
2360 /* Resolve list of templates for the flow, given policy. */
2361 
2362 static int
2363 xfrm_tmpl_resolve_one(struct xfrm_policy *policy, const struct flowi *fl,
2364                       struct xfrm_state **xfrm, unsigned short family)
2365 {
2366         struct net *net = xp_net(policy);
2367         int nx;
2368         int i, error;
2369         xfrm_address_t *daddr = xfrm_flowi_daddr(fl, family);
2370         xfrm_address_t *saddr = xfrm_flowi_saddr(fl, family);
2371         xfrm_address_t tmp;
2372 
2373         for (nx = 0, i = 0; i < policy->xfrm_nr; i++) {
2374                 struct xfrm_state *x;
2375                 xfrm_address_t *remote = daddr;
2376                 xfrm_address_t *local  = saddr;
2377                 struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i];
2378 
2379                 if (tmpl->mode == XFRM_MODE_TUNNEL ||
2380                     tmpl->mode == XFRM_MODE_BEET) {
2381                         remote = &tmpl->id.daddr;
2382                         local = &tmpl->saddr;
2383                         if (xfrm_addr_any(local, tmpl->encap_family)) {
2384                                 error = xfrm_get_saddr(net, fl->flowi_oif,
2385                                                        &tmp, remote,
2386                                                        tmpl->encap_family, 0);
2387                                 if (error)
2388                                         goto fail;
2389                                 local = &tmp;
2390                         }
2391                 }
2392 
2393                 x = xfrm_state_find(remote, local, fl, tmpl, policy, &error,
2394                                     family, policy->if_id);
2395 
2396                 if (x && x->km.state == XFRM_STATE_VALID) {
2397                         xfrm[nx++] = x;
2398                         daddr = remote;
2399                         saddr = local;
2400                         continue;
2401                 }
2402                 if (x) {
2403                         error = (x->km.state == XFRM_STATE_ERROR ?
2404                                  -EINVAL : -EAGAIN);
2405                         xfrm_state_put(x);
2406                 } else if (error == -ESRCH) {
2407                         error = -EAGAIN;
2408                 }
2409 
2410                 if (!tmpl->optional)
2411                         goto fail;
2412         }
2413         return nx;
2414 
2415 fail:
2416         for (nx--; nx >= 0; nx--)
2417                 xfrm_state_put(xfrm[nx]);
2418         return error;
2419 }
2420 
2421 static int
2422 xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, const struct flowi *fl,
2423                   struct xfrm_state **xfrm, unsigned short family)
2424 {
2425         struct xfrm_state *tp[XFRM_MAX_DEPTH];
2426         struct xfrm_state **tpp = (npols > 1) ? tp : xfrm;
2427         int cnx = 0;
2428         int error;
2429         int ret;
2430         int i;
2431 
2432         for (i = 0; i < npols; i++) {
2433                 if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) {
2434                         error = -ENOBUFS;
2435                         goto fail;
2436                 }
2437 
2438                 ret = xfrm_tmpl_resolve_one(pols[i], fl, &tpp[cnx], family);
2439                 if (ret < 0) {
2440                         error = ret;
2441                         goto fail;
2442                 } else
2443                         cnx += ret;
2444         }
2445 
2446         /* found states are sorted for outbound processing */
2447         if (npols > 1)
2448                 xfrm_state_sort(xfrm, tpp, cnx, family);
2449 
2450         return cnx;
2451 
2452  fail:
2453         for (cnx--; cnx >= 0; cnx--)
2454                 xfrm_state_put(tpp[cnx]);
2455         return error;
2456 
2457 }
2458 
2459 static int xfrm_get_tos(const struct flowi *fl, int family)
2460 {
2461         if (family == AF_INET)
2462                 return IPTOS_RT_MASK & fl->u.ip4.flowi4_tos;
2463 
2464         return 0;
2465 }
2466 
2467 static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family)
2468 {
2469         const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2470         struct dst_ops *dst_ops;
2471         struct xfrm_dst *xdst;
2472 
2473         if (!afinfo)
2474                 return ERR_PTR(-EINVAL);
2475 
2476         switch (family) {
2477         case AF_INET:
2478                 dst_ops = &net->xfrm.xfrm4_dst_ops;
2479                 break;
2480 #if IS_ENABLED(CONFIG_IPV6)
2481         case AF_INET6:
2482                 dst_ops = &net->xfrm.xfrm6_dst_ops;
2483                 break;
2484 #endif
2485         default:
2486                 BUG();
2487         }
2488         xdst = dst_alloc(dst_ops, NULL, 1, DST_OBSOLETE_NONE, 0);
2489 
2490         if (likely(xdst)) {
2491                 struct dst_entry *dst = &xdst->u.dst;
2492 
2493                 memset(dst + 1, 0, sizeof(*xdst) - sizeof(*dst));
2494         } else
2495                 xdst = ERR_PTR(-ENOBUFS);
2496 
2497         rcu_read_unlock();
2498 
2499         return xdst;
2500 }
2501 
2502 static void xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst,
2503                            int nfheader_len)
2504 {
2505         if (dst->ops->family == AF_INET6) {
2506                 struct rt6_info *rt = (struct rt6_info *)dst;
2507                 path->path_cookie = rt6_get_cookie(rt);
2508                 path->u.rt6.rt6i_nfheader_len = nfheader_len;
2509         }
2510 }
2511 
2512 static inline int xfrm_fill_dst(struct xfrm_dst *xdst, struct net_device *dev,
2513                                 const struct flowi *fl)
2514 {
2515         const struct xfrm_policy_afinfo *afinfo =
2516                 xfrm_policy_get_afinfo(xdst->u.dst.ops->family);
2517         int err;
2518 
2519         if (!afinfo)
2520                 return -EINVAL;
2521 
2522         err = afinfo->fill_dst(xdst, dev, fl);
2523 
2524         rcu_read_unlock();
2525 
2526         return err;
2527 }
2528 
2529 
2530 /* Allocate chain of dst_entry's, attach known xfrm's, calculate
2531  * all the metrics... Shortly, bundle a bundle.
2532  */
2533 
2534 static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy,
2535                                             struct xfrm_state **xfrm,
2536                                             struct xfrm_dst **bundle,
2537                                             int nx,
2538                                             const struct flowi *fl,
2539                                             struct dst_entry *dst)
2540 {
2541         const struct xfrm_state_afinfo *afinfo;
2542         const struct xfrm_mode *inner_mode;
2543         struct net *net = xp_net(policy);
2544         unsigned long now = jiffies;
2545         struct net_device *dev;
2546         struct xfrm_dst *xdst_prev = NULL;
2547         struct xfrm_dst *xdst0 = NULL;
2548         int i = 0;
2549         int err;
2550         int header_len = 0;
2551         int nfheader_len = 0;
2552         int trailer_len = 0;
2553         int tos;
2554         int family = policy->selector.family;
2555         xfrm_address_t saddr, daddr;
2556 
2557         xfrm_flowi_addr_get(fl, &saddr, &daddr, family);
2558 
2559         tos = xfrm_get_tos(fl, family);
2560 
2561         dst_hold(dst);
2562 
2563         for (; i < nx; i++) {
2564                 struct xfrm_dst *xdst = xfrm_alloc_dst(net, family);
2565                 struct dst_entry *dst1 = &xdst->u.dst;
2566 
2567                 err = PTR_ERR(xdst);
2568                 if (IS_ERR(xdst)) {
2569                         dst_release(dst);
2570                         goto put_states;
2571                 }
2572 
2573                 bundle[i] = xdst;
2574                 if (!xdst_prev)
2575                         xdst0 = xdst;
2576                 else
2577                         /* Ref count is taken during xfrm_alloc_dst()
2578                          * No need to do dst_clone() on dst1
2579                          */
2580                         xfrm_dst_set_child(xdst_prev, &xdst->u.dst);
2581 
2582                 if (xfrm[i]->sel.family == AF_UNSPEC) {
2583                         inner_mode = xfrm_ip2inner_mode(xfrm[i],
2584                                                         xfrm_af2proto(family));
2585                         if (!inner_mode) {
2586                                 err = -EAFNOSUPPORT;
2587                                 dst_release(dst);
2588                                 goto put_states;
2589                         }
2590                 } else
2591                         inner_mode = &xfrm[i]->inner_mode;
2592 
2593                 xdst->route = dst;
2594                 dst_copy_metrics(dst1, dst);
2595 
2596                 if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) {
2597                         __u32 mark = 0;
2598 
2599                         if (xfrm[i]->props.smark.v || xfrm[i]->props.smark.m)
2600                                 mark = xfrm_smark_get(fl->flowi_mark, xfrm[i]);
2601 
2602                         family = xfrm[i]->props.family;
2603                         dst = xfrm_dst_lookup(xfrm[i], tos, fl->flowi_oif,
2604                                               &saddr, &daddr, family, mark);
2605                         err = PTR_ERR(dst);
2606                         if (IS_ERR(dst))
2607                                 goto put_states;
2608                 } else
2609                         dst_hold(dst);
2610 
2611                 dst1->xfrm = xfrm[i];
2612                 xdst->xfrm_genid = xfrm[i]->genid;
2613 
2614                 dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
2615                 dst1->flags |= DST_HOST;
2616                 dst1->lastuse = now;
2617 
2618                 dst1->input = dst_discard;
2619 
2620                 rcu_read_lock();
2621                 afinfo = xfrm_state_afinfo_get_rcu(inner_mode->family);
2622                 if (likely(afinfo))
2623                         dst1->output = afinfo->output;
2624                 else
2625                         dst1->output = dst_discard_out;
2626                 rcu_read_unlock();
2627 
2628                 xdst_prev = xdst;
2629 
2630                 header_len += xfrm[i]->props.header_len;
2631                 if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT)
2632                         nfheader_len += xfrm[i]->props.header_len;
2633                 trailer_len += xfrm[i]->props.trailer_len;
2634         }
2635 
2636         xfrm_dst_set_child(xdst_prev, dst);
2637         xdst0->path = dst;
2638 
2639         err = -ENODEV;
2640         dev = dst->dev;
2641         if (!dev)
2642                 goto free_dst;
2643 
2644         xfrm_init_path(xdst0, dst, nfheader_len);
2645         xfrm_init_pmtu(bundle, nx);
2646 
2647         for (xdst_prev = xdst0; xdst_prev != (struct xfrm_dst *)dst;
2648              xdst_prev = (struct xfrm_dst *) xfrm_dst_child(&xdst_prev->u.dst)) {
2649                 err = xfrm_fill_dst(xdst_prev, dev, fl);
2650                 if (err)
2651                         goto free_dst;
2652 
2653                 xdst_prev->u.dst.header_len = header_len;
2654                 xdst_prev->u.dst.trailer_len = trailer_len;
2655                 header_len -= xdst_prev->u.dst.xfrm->props.header_len;
2656                 trailer_len -= xdst_prev->u.dst.xfrm->props.trailer_len;
2657         }
2658 
2659         return &xdst0->u.dst;
2660 
2661 put_states:
2662         for (; i < nx; i++)
2663                 xfrm_state_put(xfrm[i]);
2664 free_dst:
2665         if (xdst0)
2666                 dst_release_immediate(&xdst0->u.dst);
2667 
2668         return ERR_PTR(err);
2669 }
2670 
2671 static int xfrm_expand_policies(const struct flowi *fl, u16 family,
2672                                 struct xfrm_policy **pols,
2673                                 int *num_pols, int *num_xfrms)
2674 {
2675         int i;
2676 
2677         if (*num_pols == 0 || !pols[0]) {
2678                 *num_pols = 0;
2679                 *num_xfrms = 0;
2680                 return 0;
2681         }
2682         if (IS_ERR(pols[0]))
2683                 return PTR_ERR(pols[0]);
2684 
2685         *num_xfrms = pols[0]->xfrm_nr;
2686 
2687 #ifdef CONFIG_XFRM_SUB_POLICY
2688         if (pols[0] && pols[0]->action == XFRM_POLICY_ALLOW &&
2689             pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
2690                 pols[1] = xfrm_policy_lookup_bytype(xp_net(pols[0]),
2691                                                     XFRM_POLICY_TYPE_MAIN,
2692                                                     fl, family,
2693                                                     XFRM_POLICY_OUT,
2694                                                     pols[0]->if_id);
2695                 if (pols[1]) {
2696                         if (IS_ERR(pols[1])) {
2697                                 xfrm_pols_put(pols, *num_pols);
2698                                 return PTR_ERR(pols[1]);
2699                         }
2700                         (*num_pols)++;
2701                         (*num_xfrms) += pols[1]->xfrm_nr;
2702                 }
2703         }
2704 #endif
2705         for (i = 0; i < *num_pols; i++) {
2706                 if (pols[i]->action != XFRM_POLICY_ALLOW) {
2707                         *num_xfrms = -1;
2708                         break;
2709                 }
2710         }
2711 
2712         return 0;
2713 
2714 }
2715 
2716 static struct xfrm_dst *
2717 xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols,
2718                                const struct flowi *fl, u16 family,
2719                                struct dst_entry *dst_orig)
2720 {
2721         struct net *net = xp_net(pols[0]);
2722         struct xfrm_state *xfrm[XFRM_MAX_DEPTH];
2723         struct xfrm_dst *bundle[XFRM_MAX_DEPTH];
2724         struct xfrm_dst *xdst;
2725         struct dst_entry *dst;
2726         int err;
2727 
2728         /* Try to instantiate a bundle */
2729         err = xfrm_tmpl_resolve(pols, num_pols, fl, xfrm, family);
2730         if (err <= 0) {
2731                 if (err == 0)
2732                         return NULL;
2733 
2734                 if (err != -EAGAIN)
2735                         XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
2736                 return ERR_PTR(err);
2737         }
2738 
2739         dst = xfrm_bundle_create(pols[0], xfrm, bundle, err, fl, dst_orig);
2740         if (IS_ERR(dst)) {
2741                 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLEGENERROR);
2742                 return ERR_CAST(dst);
2743         }
2744 
2745         xdst = (struct xfrm_dst *)dst;
2746         xdst->num_xfrms = err;
2747         xdst->num_pols = num_pols;
2748         memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
2749         xdst->policy_genid = atomic_read(&pols[0]->genid);
2750 
2751         return xdst;
2752 }
2753 
2754 static void xfrm_policy_queue_process(struct timer_list *t)
2755 {
2756         struct sk_buff *skb;
2757         struct sock *sk;
2758         struct dst_entry *dst;
2759         struct xfrm_policy *pol = from_timer(pol, t, polq.hold_timer);
2760         struct net *net = xp_net(pol);
2761         struct xfrm_policy_queue *pq = &pol->polq;
2762         struct flowi fl;
2763         struct sk_buff_head list;
2764 
2765         spin_lock(&pq->hold_queue.lock);
2766         skb = skb_peek(&pq->hold_queue);
2767         if (!skb) {
2768                 spin_unlock(&pq->hold_queue.lock);
2769                 goto out;
2770         }
2771         dst = skb_dst(skb);
2772         sk = skb->sk;
2773         xfrm_decode_session(skb, &fl, dst->ops->family);
2774         spin_unlock(&pq->hold_queue.lock);
2775 
2776         dst_hold(xfrm_dst_path(dst));
2777         dst = xfrm_lookup(net, xfrm_dst_path(dst), &fl, sk, XFRM_LOOKUP_QUEUE);
2778         if (IS_ERR(dst))
2779                 goto purge_queue;
2780 
2781         if (dst->flags & DST_XFRM_QUEUE) {
2782                 dst_release(dst);
2783 
2784                 if (pq->timeout >= XFRM_QUEUE_TMO_MAX)
2785                         goto purge_queue;
2786 
2787                 pq->timeout = pq->timeout << 1;
2788                 if (!mod_timer(&pq->hold_timer, jiffies + pq->timeout))
2789                         xfrm_pol_hold(pol);
2790                 goto out;
2791         }
2792 
2793         dst_release(dst);
2794 
2795         __skb_queue_head_init(&list);
2796 
2797         spin_lock(&pq->hold_queue.lock);
2798         pq->timeout = 0;
2799         skb_queue_splice_init(&pq->hold_queue, &list);
2800         spin_unlock(&pq->hold_queue.lock);
2801 
2802         while (!skb_queue_empty(&list)) {
2803                 skb = __skb_dequeue(&list);
2804 
2805                 xfrm_decode_session(skb, &fl, skb_dst(skb)->ops->family);
2806                 dst_hold(xfrm_dst_path(skb_dst(skb)));
2807                 dst = xfrm_lookup(net, xfrm_dst_path(skb_dst(skb)), &fl, skb->sk, 0);
2808                 if (IS_ERR(dst)) {
2809                         kfree_skb(skb);
2810                         continue;
2811                 }
2812 
2813                 nf_reset_ct(skb);
2814                 skb_dst_drop(skb);
2815                 skb_dst_set(skb, dst);
2816 
2817                 dst_output(net, skb->sk, skb);
2818         }
2819 
2820 out:
2821         xfrm_pol_put(pol);
2822         return;
2823 
2824 purge_queue:
2825         pq->timeout = 0;
2826         skb_queue_purge(&pq->hold_queue);
2827         xfrm_pol_put(pol);
2828 }
2829 
2830 static int xdst_queue_output(struct net *net, struct sock *sk, struct sk_buff *skb)
2831 {
2832         unsigned long sched_next;
2833         struct dst_entry *dst = skb_dst(skb);
2834         struct xfrm_dst *xdst = (struct xfrm_dst *) dst;
2835         struct xfrm_policy *pol = xdst->pols[0];
2836         struct xfrm_policy_queue *pq = &pol->polq;
2837 
2838         if (unlikely(skb_fclone_busy(sk, skb))) {
2839                 kfree_skb(skb);
2840                 return 0;
2841         }
2842 
2843         if (pq->hold_queue.qlen > XFRM_MAX_QUEUE_LEN) {
2844                 kfree_skb(skb);
2845                 return -EAGAIN;
2846         }
2847 
2848         skb_dst_force(skb);
2849 
2850         spin_lock_bh(&pq->hold_queue.lock);
2851 
2852         if (!pq->timeout)
2853                 pq->timeout = XFRM_QUEUE_TMO_MIN;
2854 
2855         sched_next = jiffies + pq->timeout;
2856 
2857         if (del_timer(&pq->hold_timer)) {
2858                 if (time_before(pq->hold_timer.expires, sched_next))
2859                         sched_next = pq->hold_timer.expires;
2860                 xfrm_pol_put(pol);
2861         }
2862 
2863         __skb_queue_tail(&pq->hold_queue, skb);
2864         if (!mod_timer(&pq->hold_timer, sched_next))
2865                 xfrm_pol_hold(pol);
2866 
2867         spin_unlock_bh(&pq->hold_queue.lock);
2868 
2869         return 0;
2870 }
2871 
2872 static struct xfrm_dst *xfrm_create_dummy_bundle(struct net *net,
2873                                                  struct xfrm_flo *xflo,
2874                                                  const struct flowi *fl,
2875                                                  int num_xfrms,
2876                                                  u16 family)
2877 {
2878         int err;
2879         struct net_device *dev;
2880         struct dst_entry *dst;
2881         struct dst_entry *dst1;
2882         struct xfrm_dst *xdst;
2883 
2884         xdst = xfrm_alloc_dst(net, family);
2885         if (IS_ERR(xdst))
2886                 return xdst;
2887 
2888         if (!(xflo->flags & XFRM_LOOKUP_QUEUE) ||
2889             net->xfrm.sysctl_larval_drop ||
2890             num_xfrms <= 0)
2891                 return xdst;
2892 
2893         dst = xflo->dst_orig;
2894         dst1 = &xdst->u.dst;
2895         dst_hold(dst);
2896         xdst->route = dst;
2897 
2898         dst_copy_metrics(dst1, dst);
2899 
2900         dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
2901         dst1->flags |= DST_HOST | DST_XFRM_QUEUE;
2902         dst1->lastuse = jiffies;
2903 
2904         dst1->input = dst_discard;
2905         dst1->output = xdst_queue_output;
2906 
2907         dst_hold(dst);
2908         xfrm_dst_set_child(xdst, dst);
2909         xdst->path = dst;
2910 
2911         xfrm_init_path((struct xfrm_dst *)dst1, dst, 0);
2912 
2913         err = -ENODEV;
2914         dev = dst->dev;
2915         if (!dev)
2916                 goto free_dst;
2917 
2918         err = xfrm_fill_dst(xdst, dev, fl);
2919         if (err)
2920                 goto free_dst;
2921 
2922 out:
2923         return xdst;
2924 
2925 free_dst:
2926         dst_release(dst1);
2927         xdst = ERR_PTR(err);
2928         goto out;
2929 }
2930 
2931 static struct xfrm_dst *xfrm_bundle_lookup(struct net *net,
2932                                            const struct flowi *fl,
2933                                            u16 family, u8 dir,
2934                                            struct xfrm_flo *xflo, u32 if_id)
2935 {
2936         struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
2937         int num_pols = 0, num_xfrms = 0, err;
2938         struct xfrm_dst *xdst;
2939 
2940         /* Resolve policies to use if we couldn't get them from
2941          * previous cache entry */
2942         num_pols = 1;
2943         pols[0] = xfrm_policy_lookup(net, fl, family, dir, if_id);
2944         err = xfrm_expand_policies(fl, family, pols,
2945                                            &num_pols, &num_xfrms);
2946         if (err < 0)
2947                 goto inc_error;
2948         if (num_pols == 0)
2949                 return NULL;
2950         if (num_xfrms <= 0)
2951                 goto make_dummy_bundle;
2952 
2953         xdst = xfrm_resolve_and_create_bundle(pols, num_pols, fl, family,
2954                                               xflo->dst_orig);
2955         if (IS_ERR(xdst)) {
2956                 err = PTR_ERR(xdst);
2957                 if (err == -EREMOTE) {
2958                         xfrm_pols_put(pols, num_pols);
2959                         return NULL;
2960                 }
2961 
2962                 if (err != -EAGAIN)
2963                         goto error;
2964                 goto make_dummy_bundle;
2965         } else if (xdst == NULL) {
2966                 num_xfrms = 0;
2967                 goto make_dummy_bundle;
2968         }
2969 
2970         return xdst;
2971 
2972 make_dummy_bundle:
2973         /* We found policies, but there's no bundles to instantiate:
2974          * either because the policy blocks, has no transformations or
2975          * we could not build template (no xfrm_states).*/
2976         xdst = xfrm_create_dummy_bundle(net, xflo, fl, num_xfrms, family);
2977         if (IS_ERR(xdst)) {
2978                 xfrm_pols_put(pols, num_pols);
2979                 return ERR_CAST(xdst);
2980         }
2981         xdst->num_pols = num_pols;
2982         xdst->num_xfrms = num_xfrms;
2983         memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
2984 
2985         return xdst;
2986 
2987 inc_error:
2988         XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
2989 error:
2990         xfrm_pols_put(pols, num_pols);
2991         return ERR_PTR(err);
2992 }
2993 
2994 static struct dst_entry *make_blackhole(struct net *net, u16 family,
2995                                         struct dst_entry *dst_orig)
2996 {
2997         const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2998         struct dst_entry *ret;
2999 
3000         if (!afinfo) {
3001                 dst_release(dst_orig);
3002                 return ERR_PTR(-EINVAL);
3003         } else {
3004                 ret = afinfo->blackhole_route(net, dst_orig);
3005         }
3006         rcu_read_unlock();
3007 
3008         return ret;
3009 }
3010 
3011 /* Finds/creates a bundle for given flow and if_id
3012  *
3013  * At the moment we eat a raw IP route. Mostly to speed up lookups
3014  * on interfaces with disabled IPsec.
3015  *
3016  * xfrm_lookup uses an if_id of 0 by default, and is provided for
3017  * compatibility
3018  */
3019 struct dst_entry *xfrm_lookup_with_ifid(struct net *net,
3020                                         struct dst_entry *dst_orig,
3021                                         const struct flowi *fl,
3022                                         const struct sock *sk,
3023                                         int flags, u32 if_id)
3024 {
3025         struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
3026         struct xfrm_dst *xdst;
3027         struct dst_entry *dst, *route;
3028         u16 family = dst_orig->ops->family;
3029         u8 dir = XFRM_POLICY_OUT;
3030         int i, err, num_pols, num_xfrms = 0, drop_pols = 0;
3031 
3032         dst = NULL;
3033         xdst = NULL;
3034         route = NULL;
3035 
3036         sk = sk_const_to_full_sk(sk);
3037         if (sk && sk->sk_policy[XFRM_POLICY_OUT]) {
3038                 num_pols = 1;
3039                 pols[0] = xfrm_sk_policy_lookup(sk, XFRM_POLICY_OUT, fl, family,
3040                                                 if_id);
3041                 err = xfrm_expand_policies(fl, family, pols,
3042                                            &num_pols, &num_xfrms);
3043                 if (err < 0)
3044                         goto dropdst;
3045 
3046                 if (num_pols) {
3047                         if (num_xfrms <= 0) {
3048                                 drop_pols = num_pols;
3049                                 goto no_transform;
3050                         }
3051 
3052                         xdst = xfrm_resolve_and_create_bundle(
3053                                         pols, num_pols, fl,
3054                                         family, dst_orig);
3055 
3056                         if (IS_ERR(xdst)) {
3057                                 xfrm_pols_put(pols, num_pols);
3058                                 err = PTR_ERR(xdst);
3059                                 if (err == -EREMOTE)
3060                                         goto nopol;
3061 
3062                                 goto dropdst;
3063                         } else if (xdst == NULL) {
3064                                 num_xfrms = 0;
3065                                 drop_pols = num_pols;
3066                                 goto no_transform;
3067                         }
3068 
3069                         route = xdst->route;
3070                 }
3071         }
3072 
3073         if (xdst == NULL) {
3074                 struct xfrm_flo xflo;
3075 
3076                 xflo.dst_orig = dst_orig;
3077                 xflo.flags = flags;
3078 
3079                 /* To accelerate a bit...  */
3080                 if ((dst_orig->flags & DST_NOXFRM) ||
3081                     !net->xfrm.policy_count[XFRM_POLICY_OUT])
3082                         goto nopol;
3083 
3084                 xdst = xfrm_bundle_lookup(net, fl, family, dir, &xflo, if_id);
3085                 if (xdst == NULL)
3086                         goto nopol;
3087                 if (IS_ERR(xdst)) {
3088                         err = PTR_ERR(xdst);
3089                         goto dropdst;
3090                 }
3091 
3092                 num_pols = xdst->num_pols;
3093                 num_xfrms = xdst->num_xfrms;
3094                 memcpy(pols, xdst->pols, sizeof(struct xfrm_policy *) * num_pols);
3095                 route = xdst->route;
3096         }
3097 
3098         dst = &xdst->u.dst;
3099         if (route == NULL && num_xfrms > 0) {
3100                 /* The only case when xfrm_bundle_lookup() returns a
3101                  * bundle with null route, is when the template could
3102                  * not be resolved. It means policies are there, but
3103                  * bundle could not be created, since we don't yet
3104                  * have the xfrm_state's. We need to wait for KM to
3105                  * negotiate new SA's or bail out with error.*/
3106                 if (net->xfrm.sysctl_larval_drop) {
3107                         XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
3108                         err = -EREMOTE;
3109                         goto error;
3110                 }
3111 
3112                 err = -EAGAIN;
3113 
3114                 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
3115                 goto error;
3116         }
3117 
3118 no_transform:
3119         if (num_pols == 0)
3120                 goto nopol;
3121 
3122         if ((flags & XFRM_LOOKUP_ICMP) &&
3123             !(pols[0]->flags & XFRM_POLICY_ICMP)) {
3124                 err = -ENOENT;
3125                 goto error;
3126         }
3127 
3128         for (i = 0; i < num_pols; i++)
3129                 pols[i]->curlft.use_time = ktime_get_real_seconds();
3130 
3131         if (num_xfrms < 0) {
3132                 /* Prohibit the flow */
3133                 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLBLOCK);
3134                 err = -EPERM;
3135                 goto error;
3136         } else if (num_xfrms > 0) {
3137                 /* Flow transformed */
3138                 dst_release(dst_orig);
3139         } else {
3140                 /* Flow passes untransformed */
3141                 dst_release(dst);
3142                 dst = dst_orig;
3143         }
3144 ok:
3145         xfrm_pols_put(pols, drop_pols);
3146         if (dst && dst->xfrm &&
3147             dst->xfrm->props.mode == XFRM_MODE_TUNNEL)
3148                 dst->flags |= DST_XFRM_TUNNEL;
3149         return dst;
3150 
3151 nopol:
3152         if (!(flags & XFRM_LOOKUP_ICMP)) {
3153                 dst = dst_orig;
3154                 goto ok;
3155         }
3156         err = -ENOENT;
3157 error:
3158         dst_release(dst);
3159 dropdst:
3160         if (!(flags & XFRM_LOOKUP_KEEP_DST_REF))
3161                 dst_release(dst_orig);
3162         xfrm_pols_put(pols, drop_pols);
3163         return ERR_PTR(err);
3164 }
3165 EXPORT_SYMBOL(xfrm_lookup_with_ifid);
3166 
3167 /* Main function: finds/creates a bundle for given flow.
3168  *
3169  * At the moment we eat a raw IP route. Mostly to speed up lookups
3170  * on interfaces with disabled IPsec.
3171  */
3172 struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig,
3173                               const struct flowi *fl, const struct sock *sk,
3174                               int flags)
3175 {
3176         return xfrm_lookup_with_ifid(net, dst_orig, fl, sk, flags, 0);
3177 }
3178 EXPORT_SYMBOL(xfrm_lookup);
3179 
3180 /* Callers of xfrm_lookup_route() must ensure a call to dst_output().
3181  * Otherwise we may send out blackholed packets.
3182  */
3183 struct dst_entry *xfrm_lookup_route(struct net *net, struct dst_entry *dst_orig,
3184                                     const struct flowi *fl,
3185                                     const struct sock *sk, int flags)
3186 {
3187         struct dst_entry *dst = xfrm_lookup(net, dst_orig, fl, sk,
3188                                             flags | XFRM_LOOKUP_QUEUE |
3189                                             XFRM_LOOKUP_KEEP_DST_REF);
3190 
3191         if (IS_ERR(dst) && PTR_ERR(dst) == -EREMOTE)
3192                 return make_blackhole(net, dst_orig->ops->family, dst_orig);
3193 
3194         if (IS_ERR(dst))
3195                 dst_release(dst_orig);
3196 
3197         return dst;
3198 }
3199 EXPORT_SYMBOL(xfrm_lookup_route);
3200 
3201 static inline int
3202 xfrm_secpath_reject(int idx, struct sk_buff *skb, const struct flowi *fl)
3203 {
3204         struct sec_path *sp = skb_sec_path(skb);
3205         struct xfrm_state *x;
3206 
3207         if (!sp || idx < 0 || idx >= sp->len)
3208                 return 0;
3209         x = sp->xvec[idx];
3210         if (!x->type->reject)
3211                 return 0;
3212         return x->type->reject(x, skb, fl);
3213 }
3214 
3215 /* When skb is transformed back to its "native" form, we have to
3216  * check policy restrictions. At the moment we make this in maximally
3217  * stupid way. Shame on me. :-) Of course, connected sockets must
3218  * have policy cached at them.
3219  */
3220 
3221 static inline int
3222 xfrm_state_ok(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x,
3223               unsigned short family)
3224 {
3225         if (xfrm_state_kern(x))
3226                 return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family);
3227         return  x->id.proto == tmpl->id.proto &&
3228                 (x->id.spi == tmpl->id.spi || !tmpl->id.spi) &&
3229                 (x->props.reqid == tmpl->reqid || !tmpl->reqid) &&
3230                 x->props.mode == tmpl->mode &&
3231                 (tmpl->allalgs || (tmpl->aalgos & (1<<x->props.aalgo)) ||
3232                  !(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) &&
3233                 !(x->props.mode != XFRM_MODE_TRANSPORT &&
3234                   xfrm_state_addr_cmp(tmpl, x, family));
3235 }
3236 
3237 /*
3238  * 0 or more than 0 is returned when validation is succeeded (either bypass
3239  * because of optional transport mode, or next index of the mathced secpath
3240  * state with the template.
3241  * -1 is returned when no matching template is found.
3242  * Otherwise "-2 - errored_index" is returned.
3243  */
3244 static inline int
3245 xfrm_policy_ok(const struct xfrm_tmpl *tmpl, const struct sec_path *sp, int start,
3246                unsigned short family)
3247 {
3248         int idx = start;
3249 
3250         if (tmpl->optional) {
3251                 if (tmpl->mode == XFRM_MODE_TRANSPORT)
3252                         return start;
3253         } else
3254                 start = -1;
3255         for (; idx < sp->len; idx++) {
3256                 if (xfrm_state_ok(tmpl, sp->xvec[idx], family))
3257                         return ++idx;
3258                 if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) {
3259                         if (start == -1)
3260                                 start = -2-idx;
3261                         break;
3262                 }
3263         }
3264         return start;
3265 }
3266 
3267 static void
3268 decode_session4(struct sk_buff *skb, struct flowi *fl, bool reverse)
3269 {
3270         const struct iphdr *iph = ip_hdr(skb);
3271         int ihl = iph->ihl;
3272         u8 *xprth = skb_network_header(skb) + ihl * 4;
3273         struct flowi4 *fl4 = &fl->u.ip4;
3274         int oif = 0;
3275 
3276         if (skb_dst(skb) && skb_dst(skb)->dev)
3277                 oif = skb_dst(skb)->dev->ifindex;
3278 
3279         memset(fl4, 0, sizeof(struct flowi4));
3280         fl4->flowi4_mark = skb->mark;
3281         fl4->flowi4_oif = reverse ? skb->skb_iif : oif;
3282 
3283         fl4->flowi4_proto = iph->protocol;
3284         fl4->daddr = reverse ? iph->saddr : iph->daddr;
3285         fl4->saddr = reverse ? iph->daddr : iph->saddr;
3286         fl4->flowi4_tos = iph->tos;
3287 
3288         if (!ip_is_fragment(iph)) {
3289                 switch (iph->protocol) {
3290                 case IPPROTO_UDP:
3291                 case IPPROTO_UDPLITE:
3292                 case IPPROTO_TCP:
3293                 case IPPROTO_SCTP:
3294                 case IPPROTO_DCCP:
3295                         if (xprth + 4 < skb->data ||
3296                             pskb_may_pull(skb, xprth + 4 - skb->data)) {
3297                                 __be16 *ports;
3298 
3299                                 xprth = skb_network_header(skb) + ihl * 4;
3300                                 ports = (__be16 *)xprth;
3301 
3302                                 fl4->fl4_sport = ports[!!reverse];
3303                                 fl4->fl4_dport = ports[!reverse];
3304                         }
3305                         break;
3306                 case IPPROTO_ICMP:
3307                         if (xprth + 2 < skb->data ||
3308                             pskb_may_pull(skb, xprth + 2 - skb->data)) {
3309                                 u8 *icmp;
3310 
3311                                 xprth = skb_network_header(skb) + ihl * 4;
3312                                 icmp = xprth;
3313 
3314                                 fl4->fl4_icmp_type = icmp[0];
3315                                 fl4->fl4_icmp_code = icmp[1];
3316                         }
3317                         break;
3318                 case IPPROTO_ESP:
3319                         if (xprth + 4 < skb->data ||
3320                             pskb_may_pull(skb, xprth + 4 - skb->data)) {
3321                                 __be32 *ehdr;
3322 
3323                                 xprth = skb_network_header(skb) + ihl * 4;
3324                                 ehdr = (__be32 *)xprth;
3325 
3326                                 fl4->fl4_ipsec_spi = ehdr[0];
3327                         }
3328                         break;
3329                 case IPPROTO_AH:
3330                         if (xprth + 8 < skb->data ||
3331                             pskb_may_pull(skb, xprth + 8 - skb->data)) {
3332                                 __be32 *ah_hdr;
3333 
3334                                 xprth = skb_network_header(skb) + ihl * 4;
3335                                 ah_hdr = (__be32 *)xprth;
3336 
3337                                 fl4->fl4_ipsec_spi = ah_hdr[1];
3338                         }
3339                         break;
3340                 case IPPROTO_COMP:
3341                         if (xprth + 4 < skb->data ||
3342                             pskb_may_pull(skb, xprth + 4 - skb->data)) {
3343                                 __be16 *ipcomp_hdr;
3344 
3345                                 xprth = skb_network_header(skb) + ihl * 4;
3346                                 ipcomp_hdr = (__be16 *)xprth;
3347 
3348                                 fl4->fl4_ipsec_spi = htonl(ntohs(ipcomp_hdr[1]));
3349                         }
3350                         break;
3351                 case IPPROTO_GRE:
3352                         if (xprth + 12 < skb->data ||
3353                             pskb_may_pull(skb, xprth + 12 - skb->data)) {
3354                                 __be16 *greflags;
3355                                 __be32 *gre_hdr;
3356 
3357                                 xprth = skb_network_header(skb) + ihl * 4;
3358                                 greflags = (__be16 *)xprth;
3359                                 gre_hdr = (__be32 *)xprth;
3360 
3361                                 if (greflags[0] & GRE_KEY) {
3362                                         if (greflags[0] & GRE_CSUM)
3363                                                 gre_hdr++;
3364                                         fl4->fl4_gre_key = gre_hdr[1];
3365                                 }
3366                         }
3367                         break;
3368                 default:
3369                         fl4->fl4_ipsec_spi = 0;
3370                         break;
3371                 }
3372         }
3373 }
3374 
3375 #if IS_ENABLED(CONFIG_IPV6)
3376 static void
3377 decode_session6(struct sk_buff *skb, struct flowi *fl, bool reverse)
3378 {
3379         struct flowi6 *fl6 = &fl->u.ip6;
3380         int onlyproto = 0;
3381         const struct ipv6hdr *hdr = ipv6_hdr(skb);
3382         u32 offset = sizeof(*hdr);
3383         struct ipv6_opt_hdr *exthdr;
3384         const unsigned char *nh = skb_network_header(skb);
3385         u16 nhoff = IP6CB(skb)->nhoff;
3386         int oif = 0;
3387         u8 nexthdr;
3388 
3389         if (!nhoff)
3390                 nhoff = offsetof(struct ipv6hdr, nexthdr);
3391 
3392         nexthdr = nh[nhoff];
3393 
3394         if (skb_dst(skb) && skb_dst(skb)->dev)
3395                 oif = skb_dst(skb)->dev->ifindex;
3396 
3397         memset(fl6, 0, sizeof(struct flowi6));
3398         fl6->flowi6_mark = skb->mark;
3399         fl6->flowi6_oif = reverse ? skb->skb_iif : oif;
3400 
3401         fl6->daddr = reverse ? hdr->saddr : hdr->daddr;
3402         fl6->saddr = reverse ? hdr->daddr : hdr->saddr;
3403 
3404         while (nh + offset + sizeof(*exthdr) < skb->data ||
3405                pskb_may_pull(skb, nh + offset + sizeof(*exthdr) - skb->data)) {
3406                 nh = skb_network_header(skb);
3407                 exthdr = (struct ipv6_opt_hdr *)(nh + offset);
3408 
3409                 switch (nexthdr) {
3410                 case NEXTHDR_FRAGMENT:
3411                         onlyproto = 1;
3412                         /* fall through */
3413                 case NEXTHDR_ROUTING:
3414                 case NEXTHDR_HOP:
3415                 case NEXTHDR_DEST:
3416                         offset += ipv6_optlen(exthdr);
3417                         nexthdr = exthdr->nexthdr;
3418                         exthdr = (struct ipv6_opt_hdr *)(nh + offset);
3419                         break;
3420                 case IPPROTO_UDP:
3421                 case IPPROTO_UDPLITE:
3422                 case IPPROTO_TCP:
3423                 case IPPROTO_SCTP:
3424                 case IPPROTO_DCCP:
3425                         if (!onlyproto && (nh + offset + 4 < skb->data ||
3426                              pskb_may_pull(skb, nh + offset + 4 - skb->data))) {
3427                                 __be16 *ports;
3428 
3429                                 nh = skb_network_header(skb);
3430                                 ports = (__be16 *)(nh + offset);
3431                                 fl6->fl6_sport = ports[!!reverse];
3432                                 fl6->fl6_dport = ports[!reverse];
3433                         }
3434                         fl6->flowi6_proto = nexthdr;
3435                         return;
3436                 case IPPROTO_ICMPV6:
3437                         if (!onlyproto && (nh + offset + 2 < skb->data ||
3438                             pskb_may_pull(skb, nh + offset + 2 - skb->data))) {
3439                                 u8 *icmp;
3440 
3441                                 nh = skb_network_header(skb);
3442                                 icmp = (u8 *)(nh + offset);
3443                                 fl6->fl6_icmp_type = icmp[0];
3444                                 fl6->fl6_icmp_code = icmp[1];
3445                         }
3446                         fl6->flowi6_proto = nexthdr;
3447                         return;
3448 #if IS_ENABLED(CONFIG_IPV6_MIP6)
3449                 case IPPROTO_MH:
3450                         offset += ipv6_optlen(exthdr);
3451                         if (!onlyproto && (nh + offset + 3 < skb->data ||
3452                             pskb_may_pull(skb, nh + offset + 3 - skb->data))) {
3453                                 struct ip6_mh *mh;
3454 
3455                                 nh = skb_network_header(skb);
3456                                 mh = (struct ip6_mh *)(nh + offset);
3457                                 fl6->fl6_mh_type = mh->ip6mh_type;
3458                         }
3459                         fl6->flowi6_proto = nexthdr;
3460                         return;
3461 #endif
3462                 /* XXX Why are there these headers? */
3463                 case IPPROTO_AH:
3464                 case IPPROTO_ESP:
3465                 case IPPROTO_COMP:
3466                 default:
3467                         fl6->fl6_ipsec_spi = 0;
3468                         fl6->flowi6_proto = nexthdr;
3469                         return;
3470                 }
3471         }
3472 }
3473 #endif
3474 
3475 int __xfrm_decode_session(struct sk_buff *skb, struct flowi *fl,
3476                           unsigned int family, int reverse)
3477 {
3478         switch (family) {
3479         case AF_INET:
3480                 decode_session4(skb, fl, reverse);
3481                 break;
3482 #if IS_ENABLED(CONFIG_IPV6)
3483         case AF_INET6:
3484                 decode_session6(skb, fl, reverse);
3485                 break;
3486 #endif
3487         default:
3488                 return -EAFNOSUPPORT;
3489         }
3490 
3491         return security_xfrm_decode_session(skb, &fl->flowi_secid);
3492 }
3493 EXPORT_SYMBOL(__xfrm_decode_session);
3494 
3495 static inline int secpath_has_nontransport(const struct sec_path *sp, int k, int *idxp)
3496 {
3497         for (; k < sp->len; k++) {
3498                 if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) {
3499                         *idxp = k;
3500                         return 1;
3501                 }
3502         }
3503 
3504         return 0;
3505 }
3506 
3507 int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,
3508                         unsigned short family)
3509 {
3510         struct net *net = dev_net(skb->dev);
3511         struct xfrm_policy *pol;
3512         struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
3513         int npols = 0;
3514         int xfrm_nr;
3515         int pi;
3516         int reverse;
3517         struct flowi fl;
3518         int xerr_idx = -1;
3519         const struct xfrm_if_cb *ifcb;
3520         struct sec_path *sp;
3521         struct xfrm_if *xi;
3522         u32 if_id = 0;
3523 
3524         rcu_read_lock();
3525         ifcb = xfrm_if_get_cb();
3526 
3527         if (ifcb) {
3528                 xi = ifcb->decode_session(skb, family);
3529                 if (xi) {
3530                         if_id = xi->p.if_id;
3531                         net = xi->net;
3532                 }
3533         }
3534         rcu_read_unlock();
3535 
3536         reverse = dir & ~XFRM_POLICY_MASK;
3537         dir &= XFRM_POLICY_MASK;
3538 
3539         if (__xfrm_decode_session(skb, &fl, family, reverse) < 0) {
3540                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
3541                 return 0;
3542         }
3543 
3544         nf_nat_decode_session(skb, &fl, family);
3545 
3546         /* First, check used SA against their selectors. */
3547         sp = skb_sec_path(skb);
3548         if (sp) {
3549                 int i;
3550 
3551                 for (i = sp->len - 1; i >= 0; i--) {
3552                         struct xfrm_state *x = sp->xvec[i];
3553                         if (!xfrm_selector_match(&x->sel, &fl, family)) {
3554                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
3555                                 return 0;
3556                         }
3557                 }
3558         }
3559 
3560         pol = NULL;
3561         sk = sk_to_full_sk(sk);
3562         if (sk && sk->sk_policy[dir]) {
3563                 pol = xfrm_sk_policy_lookup(sk, dir, &fl, family, if_id);
3564                 if (IS_ERR(pol)) {
3565                         XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
3566                         return 0;
3567                 }
3568         }
3569 
3570         if (!pol)
3571                 pol = xfrm_policy_lookup(net, &fl, family, dir, if_id);
3572 
3573         if (IS_ERR(pol)) {
3574                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
3575                 return 0;
3576         }
3577 
3578         if (!pol) {
3579                 if (sp && secpath_has_nontransport(sp, 0, &xerr_idx)) {
3580                         xfrm_secpath_reject(xerr_idx, skb, &fl);
3581                         XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS);
3582                         return 0;
3583                 }
3584                 return 1;
3585         }
3586 
3587         pol->curlft.use_time = ktime_get_real_seconds();
3588 
3589         pols[0] = pol;
3590         npols++;
3591 #ifdef CONFIG_XFRM_SUB_POLICY
3592         if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
3593                 pols[1] = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN,
3594                                                     &fl, family,
3595                                                     XFRM_POLICY_IN, if_id);
3596                 if (pols[1]) {
3597                         if (IS_ERR(pols[1])) {
3598                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
3599                                 return 0;
3600                         }
3601                         pols[1]->curlft.use_time = ktime_get_real_seconds();
3602                         npols++;
3603                 }
3604         }
3605 #endif
3606 
3607         if (pol->action == XFRM_POLICY_ALLOW) {
3608                 static struct sec_path dummy;
3609                 struct xfrm_tmpl *tp[XFRM_MAX_DEPTH];
3610                 struct xfrm_tmpl *stp[XFRM_MAX_DEPTH];
3611                 struct xfrm_tmpl **tpp = tp;
3612                 int ti = 0;
3613                 int i, k;
3614 
3615                 sp = skb_sec_path(skb);
3616                 if (!sp)
3617                         sp = &dummy;
3618 
3619                 for (pi = 0; pi < npols; pi++) {
3620                         if (pols[pi] != pol &&
3621                             pols[pi]->action != XFRM_POLICY_ALLOW) {
3622                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
3623                                 goto reject;
3624                         }
3625                         if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) {
3626                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
3627                                 goto reject_error;
3628                         }
3629                         for (i = 0; i < pols[pi]->xfrm_nr; i++)
3630                                 tpp[ti++] = &pols[pi]->xfrm_vec[i];
3631                 }
3632                 xfrm_nr = ti;
3633                 if (npols > 1) {
3634                         xfrm_tmpl_sort(stp, tpp, xfrm_nr, family);
3635                         tpp = stp;
3636                 }
3637 
3638                 /* For each tunnel xfrm, find the first matching tmpl.
3639                  * For each tmpl before that, find corresponding xfrm.
3640                  * Order is _important_. Later we will implement
3641                  * some barriers, but at the moment barriers
3642                  * are implied between each two transformations.
3643                  */
3644                 for (i = xfrm_nr-1, k = 0; i >= 0; i--) {
3645                         k = xfrm_policy_ok(tpp[i], sp, k, family);
3646                         if (k < 0) {
3647                                 if (k < -1)
3648                                         /* "-2 - errored_index" returned */
3649                                         xerr_idx = -(2+k);
3650                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
3651                                 goto reject;
3652                         }
3653                 }
3654 
3655                 if (secpath_has_nontransport(sp, k, &xerr_idx)) {
3656                         XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
3657                         goto reject;
3658                 }
3659 
3660                 xfrm_pols_put(pols, npols);
3661                 return 1;
3662         }
3663         XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
3664 
3665 reject:
3666         xfrm_secpath_reject(xerr_idx, skb, &fl);
3667 reject_error:
3668         xfrm_pols_put(pols, npols);
3669         return 0;
3670 }
3671 EXPORT_SYMBOL(__xfrm_policy_check);
3672 
3673 int __xfrm_route_forward(struct sk_buff *skb, unsigned short family)
3674 {
3675         struct net *net = dev_net(skb->dev);
3676         struct flowi fl;
3677         struct dst_entry *dst;
3678         int res = 1;
3679 
3680         if (xfrm_decode_session(skb, &fl, family) < 0) {
3681                 XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
3682                 return 0;
3683         }
3684 
3685         skb_dst_force(skb);
3686         if (!skb_dst(skb)) {
3687                 XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
3688                 return 0;
3689         }
3690 
3691         dst = xfrm_lookup(net, skb_dst(skb), &fl, NULL, XFRM_LOOKUP_QUEUE);
3692         if (IS_ERR(dst)) {
3693                 res = 0;
3694                 dst = NULL;
3695         }
3696         skb_dst_set(skb, dst);
3697         return res;
3698 }
3699 EXPORT_SYMBOL(__xfrm_route_forward);
3700 
3701 /* Optimize later using cookies and generation ids. */
3702 
3703 static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie)
3704 {
3705         /* Code (such as __xfrm4_bundle_create()) sets dst->obsolete
3706          * to DST_OBSOLETE_FORCE_CHK to force all XFRM destinations to
3707          * get validated by dst_ops->check on every use.  We do this
3708          * because when a normal route referenced by an XFRM dst is
3709          * obsoleted we do not go looking around for all parent
3710          * referencing XFRM dsts so that we can invalidate them.  It
3711          * is just too much work.  Instead we make the checks here on
3712          * every use.  For example:
3713          *
3714          *      XFRM dst A --> IPv4 dst X
3715          *
3716          * X is the "xdst->route" of A (X is also the "dst->path" of A
3717          * in this example).  If X is marked obsolete, "A" will not
3718          * notice.  That's what we are validating here via the
3719          * stale_bundle() check.
3720          *
3721          * When a dst is removed from the fib tree, DST_OBSOLETE_DEAD will
3722          * be marked on it.
3723          * This will force stale_bundle() to fail on any xdst bundle with
3724          * this dst linked in it.
3725          */
3726         if (dst->obsolete < 0 && !stale_bundle(dst))
3727                 return dst;
3728 
3729         return NULL;
3730 }
3731 
3732 static int stale_bundle(struct dst_entry *dst)
3733 {
3734         return !xfrm_bundle_ok((struct xfrm_dst *)dst);
3735 }
3736 
3737 void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev)
3738 {
3739         while ((dst = xfrm_dst_child(dst)) && dst->xfrm && dst->dev == dev) {
3740                 dst->dev = dev_net(dev)->loopback_dev;
3741                 dev_hold(dst->dev);
3742                 dev_put(dev);
3743         }
3744 }
3745 EXPORT_SYMBOL(xfrm_dst_ifdown);
3746 
3747 static void xfrm_link_failure(struct sk_buff *skb)
3748 {
3749         /* Impossible. Such dst must be popped before reaches point of failure. */
3750 }
3751 
3752 static struct dst_entry *xfrm_negative_advice(struct dst_entry *dst)
3753 {
3754         if (dst) {
3755                 if (dst->obsolete) {
3756                         dst_release(dst);
3757                         dst = NULL;
3758                 }
3759         }
3760         return dst;
3761 }
3762 
3763 static void xfrm_init_pmtu(struct xfrm_dst **bundle, int nr)
3764 {
3765         while (nr--) {
3766                 struct xfrm_dst *xdst = bundle[nr];
3767                 u32 pmtu, route_mtu_cached;
3768                 struct dst_entry *dst;
3769 
3770                 dst = &xdst->u.dst;
3771                 pmtu = dst_mtu(xfrm_dst_child(dst));
3772                 xdst->child_mtu_cached = pmtu;
3773 
3774                 pmtu = xfrm_state_mtu(dst->xfrm, pmtu);
3775 
3776                 route_mtu_cached = dst_mtu(xdst->route);
3777                 xdst->route_mtu_cached = route_mtu_cached;
3778 
3779                 if (pmtu > route_mtu_cached)
3780                         pmtu = route_mtu_cached;
3781 
3782                 dst_metric_set(dst, RTAX_MTU, pmtu);
3783         }
3784 }
3785 
3786 /* Check that the bundle accepts the flow and its components are
3787  * still valid.
3788  */
3789 
3790 static int xfrm_bundle_ok(struct xfrm_dst *first)
3791 {
3792         struct xfrm_dst *bundle[XFRM_MAX_DEPTH];
3793         struct dst_entry *dst = &first->u.dst;
3794         struct xfrm_dst *xdst;
3795         int start_from, nr;
3796         u32 mtu;
3797 
3798         if (!dst_check(xfrm_dst_path(dst), ((struct xfrm_dst *)dst)->path_cookie) ||
3799             (dst->dev && !netif_running(dst->dev)))
3800                 return 0;
3801 
3802         if (dst->flags & DST_XFRM_QUEUE)
3803                 return 1;
3804 
3805         start_from = nr = 0;
3806         do {
3807                 struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
3808 
3809                 if (dst->xfrm->km.state != XFRM_STATE_VALID)
3810                         return 0;
3811                 if (xdst->xfrm_genid != dst->xfrm->genid)
3812                         return 0;
3813                 if (xdst->num_pols > 0 &&
3814                     xdst->policy_genid != atomic_read(&xdst->pols[0]->genid))
3815                         return 0;
3816 
3817                 bundle[nr++] = xdst;
3818 
3819                 mtu = dst_mtu(xfrm_dst_child(dst));
3820                 if (xdst->child_mtu_cached != mtu) {
3821                         start_from = nr;
3822                         xdst->child_mtu_cached = mtu;
3823                 }
3824 
3825                 if (!dst_check(xdst->route, xdst->route_cookie))
3826                         return 0;
3827                 mtu = dst_mtu(xdst->route);
3828                 if (xdst->route_mtu_cached != mtu) {
3829                         start_from = nr;
3830                         xdst->route_mtu_cached = mtu;
3831                 }
3832 
3833                 dst = xfrm_dst_child(dst);
3834         } while (dst->xfrm);
3835 
3836         if (likely(!start_from))
3837                 return 1;
3838 
3839         xdst = bundle[start_from - 1];
3840         mtu = xdst->child_mtu_cached;
3841         while (start_from--) {
3842                 dst = &xdst->u.dst;
3843 
3844                 mtu = xfrm_state_mtu(dst->xfrm, mtu);
3845                 if (mtu > xdst->route_mtu_cached)
3846                         mtu = xdst->route_mtu_cached;
3847                 dst_metric_set(dst, RTAX_MTU, mtu);
3848                 if (!start_from)
3849                         break;
3850 
3851                 xdst = bundle[start_from - 1];
3852                 xdst->child_mtu_cached = mtu;
3853         }
3854 
3855         return 1;
3856 }
3857 
3858 static unsigned int xfrm_default_advmss(const struct dst_entry *dst)
3859 {
3860         return dst_metric_advmss(xfrm_dst_path(dst));
3861 }
3862 
3863 static unsigned int xfrm_mtu(const struct dst_entry *dst)
3864 {
3865         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
3866 
3867         return mtu ? : dst_mtu(xfrm_dst_path(dst));
3868 }
3869 
3870 static const void *xfrm_get_dst_nexthop(const struct dst_entry *dst,
3871                                         const void *daddr)
3872 {
3873         while (dst->xfrm) {
3874                 const struct xfrm_state *xfrm = dst->xfrm;
3875 
3876                 dst = xfrm_dst_child(dst);
3877 
3878                 if (xfrm->props.mode == XFRM_MODE_TRANSPORT)
3879                         continue;
3880                 if (xfrm->type->flags & XFRM_TYPE_REMOTE_COADDR)
3881                         daddr = xfrm->coaddr;
3882                 else if (!(xfrm->type->flags & XFRM_TYPE_LOCAL_COADDR))
3883                         daddr = &xfrm->id.daddr;
3884         }
3885         return daddr;
3886 }
3887 
3888 static struct neighbour *xfrm_neigh_lookup(const struct dst_entry *dst,
3889                                            struct sk_buff *skb,
3890                                            const void *daddr)
3891 {
3892         const struct dst_entry *path = xfrm_dst_path(dst);
3893 
3894         if (!skb)
3895                 daddr = xfrm_get_dst_nexthop(dst, daddr);
3896         return path->ops->neigh_lookup(path, skb, daddr);
3897 }
3898 
3899 static void xfrm_confirm_neigh(const struct dst_entry *dst, const void *daddr)
3900 {
3901         const struct dst_entry *path = xfrm_dst_path(dst);
3902 
3903         daddr = xfrm_get_dst_nexthop(dst, daddr);
3904         path->ops->confirm_neigh(path, daddr);
3905 }
3906 
3907 int xfrm_policy_register_afinfo(const struct xfrm_policy_afinfo *afinfo, int family)
3908 {
3909         int err = 0;
3910 
3911         if (WARN_ON(family >= ARRAY_SIZE(xfrm_policy_afinfo)))
3912                 return -EAFNOSUPPORT;
3913 
3914         spin_lock(&xfrm_policy_afinfo_lock);
3915         if (unlikely(xfrm_policy_afinfo[family] != NULL))
3916                 err = -EEXIST;
3917         else {
3918                 struct dst_ops *dst_ops = afinfo->dst_ops;
3919                 if (likely(dst_ops->kmem_cachep == NULL))
3920                         dst_ops->kmem_cachep = xfrm_dst_cache;
3921                 if (likely(dst_ops->check == NULL))
3922                         dst_ops->check = xfrm_dst_check;
3923                 if (likely(dst_ops->default_advmss == NULL))
3924                         dst_ops->default_advmss = xfrm_default_advmss;
3925                 if (likely(dst_ops->mtu == NULL))
3926                         dst_ops->mtu = xfrm_mtu;
3927                 if (likely(dst_ops->negative_advice == NULL))
3928                         dst_ops->negative_advice = xfrm_negative_advice;
3929                 if (likely(dst_ops->link_failure == NULL))
3930                         dst_ops->link_failure = xfrm_link_failure;
3931                 if (likely(dst_ops->neigh_lookup == NULL))
3932                         dst_ops->neigh_lookup = xfrm_neigh_lookup;
3933                 if (likely(!dst_ops->confirm_neigh))
3934                         dst_ops->confirm_neigh = xfrm_confirm_neigh;
3935                 rcu_assign_pointer(xfrm_policy_afinfo[family], afinfo);
3936         }
3937         spin_unlock(&xfrm_policy_afinfo_lock);
3938 
3939         return err;
3940 }
3941 EXPORT_SYMBOL(xfrm_policy_register_afinfo);
3942 
3943 void xfrm_policy_unregister_afinfo(const struct xfrm_policy_afinfo *afinfo)
3944 {
3945         struct dst_ops *dst_ops = afinfo->dst_ops;
3946         int i;
3947 
3948         for (i = 0; i < ARRAY_SIZE(xfrm_policy_afinfo); i++) {
3949                 if (xfrm_policy_afinfo[i] != afinfo)
3950                         continue;
3951                 RCU_INIT_POINTER(xfrm_policy_afinfo[i], NULL);
3952                 break;
3953         }
3954 
3955         synchronize_rcu();
3956 
3957         dst_ops->kmem_cachep = NULL;
3958         dst_ops->check = NULL;
3959         dst_ops->negative_advice = NULL;
3960         dst_ops->link_failure = NULL;
3961 }
3962 EXPORT_SYMBOL(xfrm_policy_unregister_afinfo);
3963 
3964 void xfrm_if_register_cb(const struct xfrm_if_cb *ifcb)
3965 {
3966         spin_lock(&xfrm_if_cb_lock);
3967         rcu_assign_pointer(xfrm_if_cb, ifcb);
3968         spin_unlock(&xfrm_if_cb_lock);
3969 }
3970 EXPORT_SYMBOL(xfrm_if_register_cb);
3971 
3972 void xfrm_if_unregister_cb(void)
3973 {
3974         RCU_INIT_POINTER(xfrm_if_cb, NULL);
3975         synchronize_rcu();
3976 }
3977 EXPORT_SYMBOL(xfrm_if_unregister_cb);
3978 
3979 #ifdef CONFIG_XFRM_STATISTICS
3980 static int __net_init xfrm_statistics_init(struct net *net)
3981 {
3982         int rv;
3983         net->mib.xfrm_statistics = alloc_percpu(struct linux_xfrm_mib);
3984         if (!net->mib.xfrm_statistics)
3985                 return -ENOMEM;
3986         rv = xfrm_proc_init(net);
3987         if (rv < 0)
3988                 free_percpu(net->mib.xfrm_statistics);
3989         return rv;
3990 }
3991 
3992 static void xfrm_statistics_fini(struct net *net)
3993 {
3994         xfrm_proc_fini(net);
3995         free_percpu(net->mib.xfrm_statistics);
3996 }
3997 #else
3998 static int __net_init xfrm_statistics_init(struct net *net)
3999 {
4000         return 0;
4001 }
4002 
4003 static void xfrm_statistics_fini(struct net *net)
4004 {
4005 }
4006 #endif
4007 
4008 static int __net_init xfrm_policy_init(struct net *net)
4009 {
4010         unsigned int hmask, sz;
4011         int dir, err;
4012 
4013         if (net_eq(net, &init_net)) {
4014                 xfrm_dst_cache = kmem_cache_create("xfrm_dst_cache",
4015                                            sizeof(struct xfrm_dst),
4016                                            0, SLAB_HWCACHE_ALIGN|SLAB_PANIC,
4017                                            NULL);
4018                 err = rhashtable_init(&xfrm_policy_inexact_table,
4019                                       &xfrm_pol_inexact_params);
4020                 BUG_ON(err);
4021         }
4022 
4023         hmask = 8 - 1;
4024         sz = (hmask+1) * sizeof(struct hlist_head);
4025 
4026         net->xfrm.policy_byidx = xfrm_hash_alloc(sz);
4027         if (!net->xfrm.policy_byidx)
4028                 goto out_byidx;
4029         net->xfrm.policy_idx_hmask = hmask;
4030 
4031         for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
4032                 struct xfrm_policy_hash *htab;
4033 
4034                 net->xfrm.policy_count[dir] = 0;
4035                 net->xfrm.policy_count[XFRM_POLICY_MAX + dir] = 0;
4036                 INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]);
4037 
4038                 htab = &net->xfrm.policy_bydst[dir];
4039                 htab->table = xfrm_hash_alloc(sz);
4040                 if (!htab->table)
4041                         goto out_bydst;
4042                 htab->hmask = hmask;
4043                 htab->dbits4 = 32;
4044                 htab->sbits4 = 32;
4045                 htab->dbits6 = 128;
4046                 htab->sbits6 = 128;
4047         }
4048         net->xfrm.policy_hthresh.lbits4 = 32;
4049         net->xfrm.policy_hthresh.rbits4 = 32;
4050         net->xfrm.policy_hthresh.lbits6 = 128;
4051         net->xfrm.policy_hthresh.rbits6 = 128;
4052 
4053         seqlock_init(&net->xfrm.policy_hthresh.lock);
4054 
4055         INIT_LIST_HEAD(&net->xfrm.policy_all);
4056         INIT_LIST_HEAD(&net->xfrm.inexact_bins);
4057         INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize);
4058         INIT_WORK(&net->xfrm.policy_hthresh.work, xfrm_hash_rebuild);
4059         return 0;
4060 
4061 out_bydst:
4062         for (dir--; dir >= 0; dir--) {
4063                 struct xfrm_policy_hash *htab;
4064 
4065                 htab = &net->xfrm.policy_bydst[dir];
4066                 xfrm_hash_free(htab->table, sz);
4067         }
4068         xfrm_hash_free(net->xfrm.policy_byidx, sz);
4069 out_byidx:
4070         return -ENOMEM;
4071 }
4072 
4073 static void xfrm_policy_fini(struct net *net)
4074 {
4075         struct xfrm_pol_inexact_bin *b, *t;
4076         unsigned int sz;
4077         int dir;
4078 
4079         flush_work(&net->xfrm.policy_hash_work);
4080 #ifdef CONFIG_XFRM_SUB_POLICY
4081         xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, false);
4082 #endif
4083         xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, false);
4084 
4085         WARN_ON(!list_empty(&net->xfrm.policy_all));
4086 
4087         for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
4088                 struct xfrm_policy_hash *htab;
4089 
4090                 WARN_ON(!hlist_empty(&net->xfrm.policy_inexact[dir]));
4091 
4092                 htab = &net->xfrm.policy_bydst[dir];
4093                 sz = (htab->hmask + 1) * sizeof(struct hlist_head);
4094                 WARN_ON(!hlist_empty(htab->table));
4095                 xfrm_hash_free(htab->table, sz);
4096         }
4097 
4098         sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head);
4099         WARN_ON(!hlist_empty(net->xfrm.policy_byidx));
4100         xfrm_hash_free(net->xfrm.policy_byidx, sz);
4101 
4102         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
4103         list_for_each_entry_safe(b, t, &net->xfrm.inexact_bins, inexact_bins)
4104                 __xfrm_policy_inexact_prune_bin(b, true);
4105         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
4106 }
4107 
4108 static int __net_init xfrm_net_init(struct net *net)
4109 {
4110         int rv;
4111 
4112         /* Initialize the per-net locks here */
4113         spin_lock_init(&net->xfrm.xfrm_state_lock);
4114         spin_lock_init(&net->xfrm.xfrm_policy_lock);
4115         mutex_init(&net->xfrm.xfrm_cfg_mutex);
4116 
4117         rv = xfrm_statistics_init(net);
4118         if (rv < 0)
4119                 goto out_statistics;
4120         rv = xfrm_state_init(net);
4121         if (rv < 0)
4122                 goto out_state;
4123         rv = xfrm_policy_init(net);
4124         if (rv < 0)
4125                 goto out_policy;
4126         rv = xfrm_sysctl_init(net);
4127         if (rv < 0)
4128                 goto out_sysctl;
4129 
4130         return 0;
4131 
4132 out_sysctl:
4133         xfrm_policy_fini(net);
4134 out_policy:
4135         xfrm_state_fini(net);
4136 out_state:
4137         xfrm_statistics_fini(net);
4138 out_statistics:
4139         return rv;
4140 }
4141 
4142 static void __net_exit xfrm_net_exit(struct net *net)
4143 {
4144         xfrm_sysctl_fini(net);
4145         xfrm_policy_fini(net);
4146         xfrm_state_fini(net);
4147         xfrm_statistics_fini(net);
4148 }
4149 
4150 static struct pernet_operations __net_initdata xfrm_net_ops = {
4151         .init = xfrm_net_init,
4152         .exit = xfrm_net_exit,
4153 };
4154 
4155 void __init xfrm_init(void)
4156 {
4157         register_pernet_subsys(&xfrm_net_ops);
4158         xfrm_dev_init();
4159         seqcount_init(&xfrm_policy_hash_generation);
4160         xfrm_input_init();
4161 
4162         RCU_INIT_POINTER(xfrm_if_cb, NULL);
4163         synchronize_rcu();
4164 }
4165 
4166 #ifdef CONFIG_AUDITSYSCALL
4167 static void xfrm_audit_common_policyinfo(struct xfrm_policy *xp,
4168                                          struct audit_buffer *audit_buf)
4169 {
4170         struct xfrm_sec_ctx *ctx = xp->security;
4171         struct xfrm_selector *sel = &xp->selector;
4172 
4173         if (ctx)
4174                 audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s",
4175                                  ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
4176 
4177         switch (sel->family) {
4178         case AF_INET:
4179                 audit_log_format(audit_buf, " src=%pI4", &sel->saddr.a4);
4180                 if (sel->prefixlen_s != 32)
4181                         audit_log_format(audit_buf, " src_prefixlen=%d",
4182                                          sel->prefixlen_s);
4183                 audit_log_format(audit_buf, " dst=%pI4", &sel->daddr.a4);
4184                 if (sel->prefixlen_d != 32)
4185                         audit_log_format(audit_buf, " dst_prefixlen=%d",
4186                                          sel->prefixlen_d);
4187                 break;
4188         case AF_INET6:
4189                 audit_log_format(audit_buf, " src=%pI6", sel->saddr.a6);
4190                 if (sel->prefixlen_s != 128)
4191                         audit_log_format(audit_buf, " src_prefixlen=%d",
4192                                          sel->prefixlen_s);
4193                 audit_log_format(audit_buf, " dst=%pI6", sel->daddr.a6);
4194                 if (sel->prefixlen_d != 128)
4195                         audit_log_format(audit_buf, " dst_prefixlen=%d",
4196                                          sel->prefixlen_d);
4197                 break;
4198         }
4199 }
4200 
4201 void xfrm_audit_policy_add(struct xfrm_policy *xp, int result, bool task_valid)
4202 {
4203         struct audit_buffer *audit_buf;
4204 
4205         audit_buf = xfrm_audit_start("SPD-add");
4206         if (audit_buf == NULL)
4207                 return;
4208         xfrm_audit_helper_usrinfo(task_valid, audit_buf);
4209         audit_log_format(audit_buf, " res=%u", result);
4210         xfrm_audit_common_policyinfo(xp, audit_buf);
4211         audit_log_end(audit_buf);
4212 }
4213 EXPORT_SYMBOL_GPL(xfrm_audit_policy_add);
4214 
4215 void xfrm_audit_policy_delete(struct xfrm_policy *xp, int result,
4216                               bool task_valid)
4217 {
4218         struct audit_buffer *audit_buf;
4219 
4220         audit_buf = xfrm_audit_start("SPD-delete");
4221         if (audit_buf == NULL)
4222                 return;
4223         xfrm_audit_helper_usrinfo(task_valid, audit_buf);
4224         audit_log_format(audit_buf, " res=%u", result);
4225         xfrm_audit_common_policyinfo(xp, audit_buf);
4226         audit_log_end(audit_buf);
4227 }
4228 EXPORT_SYMBOL_GPL(xfrm_audit_policy_delete);
4229 #endif
4230 
4231 #ifdef CONFIG_XFRM_MIGRATE
4232 static bool xfrm_migrate_selector_match(const struct xfrm_selector *sel_cmp,
4233                                         const struct xfrm_selector *sel_tgt)
4234 {
4235         if (sel_cmp->proto == IPSEC_ULPROTO_ANY) {
4236                 if (sel_tgt->family == sel_cmp->family &&
4237                     xfrm_addr_equal(&sel_tgt->daddr, &sel_cmp->daddr,
4238                                     sel_cmp->family) &&
4239                     xfrm_addr_equal(&sel_tgt->saddr, &sel_cmp->saddr,
4240                                     sel_cmp->family) &&
4241                     sel_tgt->prefixlen_d == sel_cmp->prefixlen_d &&
4242                     sel_tgt->prefixlen_s == sel_cmp->prefixlen_s) {
4243                         return true;
4244                 }
4245         } else {
4246                 if (memcmp(sel_tgt, sel_cmp, sizeof(*sel_tgt)) == 0) {
4247                         return true;
4248                 }
4249         }
4250         return false;
4251 }
4252 
4253 static struct xfrm_policy *xfrm_migrate_policy_find(const struct xfrm_selector *sel,
4254                                                     u8 dir, u8 type, struct net *net)
4255 {
4256         struct xfrm_policy *pol, *ret = NULL;
4257         struct hlist_head *chain;
4258         u32 priority = ~0U;
4259 
4260         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
4261         chain = policy_hash_direct(net, &sel->daddr, &sel->saddr, sel->family, dir);
4262         hlist_for_each_entry(pol, chain, bydst) {
4263                 if (xfrm_migrate_selector_match(sel, &pol->selector) &&
4264                     pol->type == type) {
4265                         ret = pol;
4266                         priority = ret->priority;
4267                         break;
4268                 }
4269         }
4270         chain = &net->xfrm.policy_inexact[dir];
4271         hlist_for_each_entry(pol, chain, bydst_inexact_list) {
4272                 if ((pol->priority >= priority) && ret)
4273                         break;
4274 
4275                 if (xfrm_migrate_selector_match(sel, &pol->selector) &&
4276                     pol->type == type) {
4277                         ret = pol;
4278                         break;
4279                 }
4280         }
4281 
4282         xfrm_pol_hold(ret);
4283 
4284         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
4285 
4286         return ret;
4287 }
4288 
4289 static int migrate_tmpl_match(const struct xfrm_migrate *m, const struct xfrm_tmpl *t)
4290 {
4291         int match = 0;
4292 
4293         if (t->mode == m->mode && t->id.proto == m->proto &&
4294             (m->reqid == 0 || t->reqid == m->reqid)) {
4295                 switch (t->mode) {
4296                 case XFRM_MODE_TUNNEL:
4297                 case XFRM_MODE_BEET:
4298                         if (xfrm_addr_equal(&t->id.daddr, &m->old_daddr,
4299                                             m->old_family) &&
4300                             xfrm_addr_equal(&t->saddr, &m->old_saddr,
4301                                             m->old_family)) {
4302                                 match = 1;
4303                         }
4304                         break;
4305                 case XFRM_MODE_TRANSPORT:
4306                         /* in case of transport mode, template does not store
4307                            any IP addresses, hence we just compare mode and
4308                            protocol */
4309                         match = 1;
4310                         break;
4311                 default:
4312                         break;
4313                 }
4314         }
4315         return match;
4316 }
4317 
4318 /* update endpoint address(es) of template(s) */
4319 static int xfrm_policy_migrate(struct xfrm_policy *pol,
4320                                struct xfrm_migrate *m, int num_migrate)
4321 {
4322         struct xfrm_migrate *mp;
4323         int i, j, n = 0;
4324 
4325         write_lock_bh(&pol->lock);
4326         if (unlikely(pol->walk.dead)) {
4327                 /* target policy has been deleted */
4328                 write_unlock_bh(&pol->lock);
4329                 return -ENOENT;
4330         }
4331 
4332         for (i = 0; i < pol->xfrm_nr; i++) {
4333                 for (j = 0, mp = m; j < num_migrate; j++, mp++) {
4334                         if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))
4335                                 continue;
4336                         n++;
4337                         if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL &&
4338                             pol->xfrm_vec[i].mode != XFRM_MODE_BEET)
4339                                 continue;
4340                         /* update endpoints */
4341                         memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,
4342                                sizeof(pol->xfrm_vec[i].id.daddr));
4343                         memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,
4344                                sizeof(pol->xfrm_vec[i].saddr));
4345                         pol->xfrm_vec[i].encap_family = mp->new_family;
4346                         /* flush bundles */
4347                         atomic_inc(&pol->genid);
4348                 }
4349         }
4350 
4351         write_unlock_bh(&pol->lock);
4352 
4353         if (!n)
4354                 return -ENODATA;
4355 
4356         return 0;
4357 }
4358 
4359 static int xfrm_migrate_check(const struct xfrm_migrate *m, int num_migrate)
4360 {
4361         int i, j;
4362 
4363         if (num_migrate < 1 || num_migrate > XFRM_MAX_DEPTH)
4364                 return -EINVAL;
4365 
4366         for (i = 0; i < num_migrate; i++) {
4367                 if (xfrm_addr_any(&m[i].new_daddr, m[i].new_family) ||
4368                     xfrm_addr_any(&m[i].new_saddr, m[i].new_family))
4369                         return -EINVAL;
4370 
4371                 /* check if there is any duplicated entry */
4372                 for (j = i + 1; j < num_migrate; j++) {
4373                         if (!memcmp(&m[i].old_daddr, &m[j].old_daddr,
4374                                     sizeof(m[i].old_daddr)) &&
4375                             !memcmp(&m[i].old_saddr, &m[j].old_saddr,
4376                                     sizeof(m[i].old_saddr)) &&
4377                             m[i].proto == m[j].proto &&
4378                             m[i].mode == m[j].mode &&
4379                             m[i].reqid == m[j].reqid &&
4380                             m[i].old_family == m[j].old_family)
4381                                 return -EINVAL;
4382                 }
4383         }
4384 
4385         return 0;
4386 }
4387 
4388 int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
4389                  struct xfrm_migrate *m, int num_migrate,
4390                  struct xfrm_kmaddress *k, struct net *net,
4391                  struct xfrm_encap_tmpl *encap)
4392 {
4393         int i, err, nx_cur = 0, nx_new = 0;
4394         struct xfrm_policy *pol = NULL;
4395         struct xfrm_state *x, *xc;
4396         struct xfrm_state *x_cur[XFRM_MAX_DEPTH];
4397         struct xfrm_state *x_new[XFRM_MAX_DEPTH];
4398         struct xfrm_migrate *mp;
4399 
4400         /* Stage 0 - sanity checks */
4401         if ((err = xfrm_migrate_check(m, num_migrate)) < 0)
4402                 goto out;
4403 
4404         if (dir >= XFRM_POLICY_MAX) {
4405                 err = -EINVAL;
4406                 goto out;
4407         }
4408 
4409         /* Stage 1 - find policy */
4410         if ((pol = xfrm_migrate_policy_find(sel, dir, type, net)) == NULL) {
4411                 err = -ENOENT;
4412                 goto out;
4413         }
4414 
4415         /* Stage 2 - find and update state(s) */
4416         for (i = 0, mp = m; i < num_migrate; i++, mp++) {
4417                 if ((x = xfrm_migrate_state_find(mp, net))) {
4418                         x_cur[nx_cur] = x;
4419                         nx_cur++;
4420                         xc = xfrm_state_migrate(x, mp, encap);
4421                         if (xc) {
4422                                 x_new[nx_new] = xc;
4423                                 nx_new++;
4424                         } else {
4425                                 err = -ENODATA;
4426                                 goto restore_state;
4427                         }
4428                 }
4429         }
4430 
4431         /* Stage 3 - update policy */
4432         if ((err = xfrm_policy_migrate(pol, m, num_migrate)) < 0)
4433                 goto restore_state;
4434 
4435         /* Stage 4 - delete old state(s) */
4436         if (nx_cur) {
4437                 xfrm_states_put(x_cur, nx_cur);
4438                 xfrm_states_delete(x_cur, nx_cur);
4439         }
4440 
4441         /* Stage 5 - announce */
4442         km_migrate(sel, dir, type, m, num_migrate, k, encap);
4443 
4444         xfrm_pol_put(pol);
4445 
4446         return 0;
4447 out:
4448         return err;
4449 
4450 restore_state:
4451         if (pol)
4452                 xfrm_pol_put(pol);
4453         if (nx_cur)
4454                 xfrm_states_put(x_cur, nx_cur);
4455         if (nx_new)
4456                 xfrm_states_delete(x_new, nx_new);
4457 
4458         return err;
4459 }
4460 EXPORT_SYMBOL(xfrm_migrate);
4461 #endif
4462 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp