~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/sched/cls_u32.c

Version: ~ [ linux-5.8-rc4 ] ~ [ linux-5.7.7 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.50 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.131 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.187 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.229 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.229 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * net/sched/cls_u32.c  Ugly (or Universal) 32bit key Packet Classifier.
  3  *
  4  *              This program is free software; you can redistribute it and/or
  5  *              modify it under the terms of the GNU General Public License
  6  *              as published by the Free Software Foundation; either version
  7  *              2 of the License, or (at your option) any later version.
  8  *
  9  * Authors:     Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 10  *
 11  *      The filters are packed to hash tables of key nodes
 12  *      with a set of 32bit key/mask pairs at every node.
 13  *      Nodes reference next level hash tables etc.
 14  *
 15  *      This scheme is the best universal classifier I managed to
 16  *      invent; it is not super-fast, but it is not slow (provided you
 17  *      program it correctly), and general enough.  And its relative
 18  *      speed grows as the number of rules becomes larger.
 19  *
 20  *      It seems that it represents the best middle point between
 21  *      speed and manageability both by human and by machine.
 22  *
 23  *      It is especially useful for link sharing combined with QoS;
 24  *      pure RSVP doesn't need such a general approach and can use
 25  *      much simpler (and faster) schemes, sort of cls_rsvp.c.
 26  *
 27  *      JHS: We should remove the CONFIG_NET_CLS_IND from here
 28  *      eventually when the meta match extension is made available
 29  *
 30  *      nfmark match added by Catalin(ux aka Dino) BOIE <catab at umbrella.ro>
 31  */
 32 
 33 #include <linux/module.h>
 34 #include <linux/slab.h>
 35 #include <linux/types.h>
 36 #include <linux/kernel.h>
 37 #include <linux/string.h>
 38 #include <linux/errno.h>
 39 #include <linux/percpu.h>
 40 #include <linux/rtnetlink.h>
 41 #include <linux/skbuff.h>
 42 #include <linux/bitmap.h>
 43 #include <linux/netdevice.h>
 44 #include <linux/hash.h>
 45 #include <net/netlink.h>
 46 #include <net/act_api.h>
 47 #include <net/pkt_cls.h>
 48 #include <linux/idr.h>
 49 
 50 struct tc_u_knode {
 51         struct tc_u_knode __rcu *next;
 52         u32                     handle;
 53         struct tc_u_hnode __rcu *ht_up;
 54         struct tcf_exts         exts;
 55 #ifdef CONFIG_NET_CLS_IND
 56         int                     ifindex;
 57 #endif
 58         u8                      fshift;
 59         struct tcf_result       res;
 60         struct tc_u_hnode __rcu *ht_down;
 61 #ifdef CONFIG_CLS_U32_PERF
 62         struct tc_u32_pcnt __percpu *pf;
 63 #endif
 64         u32                     flags;
 65 #ifdef CONFIG_CLS_U32_MARK
 66         u32                     val;
 67         u32                     mask;
 68         u32 __percpu            *pcpu_success;
 69 #endif
 70         struct tcf_proto        *tp;
 71         struct rcu_work         rwork;
 72         /* The 'sel' field MUST be the last field in structure to allow for
 73          * tc_u32_keys allocated at end of structure.
 74          */
 75         struct tc_u32_sel       sel;
 76 };
 77 
 78 struct tc_u_hnode {
 79         struct tc_u_hnode __rcu *next;
 80         u32                     handle;
 81         u32                     prio;
 82         struct tc_u_common      *tp_c;
 83         int                     refcnt;
 84         unsigned int            divisor;
 85         struct idr              handle_idr;
 86         struct rcu_head         rcu;
 87         u32                     flags;
 88         /* The 'ht' field MUST be the last field in structure to allow for
 89          * more entries allocated at end of structure.
 90          */
 91         struct tc_u_knode __rcu *ht[1];
 92 };
 93 
 94 struct tc_u_common {
 95         struct tc_u_hnode __rcu *hlist;
 96         void                    *ptr;
 97         int                     refcnt;
 98         struct idr              handle_idr;
 99         struct hlist_node       hnode;
100         struct rcu_head         rcu;
101 };
102 
103 static inline unsigned int u32_hash_fold(__be32 key,
104                                          const struct tc_u32_sel *sel,
105                                          u8 fshift)
106 {
107         unsigned int h = ntohl(key & sel->hmask) >> fshift;
108 
109         return h;
110 }
111 
112 static int u32_classify(struct sk_buff *skb, const struct tcf_proto *tp,
113                         struct tcf_result *res)
114 {
115         struct {
116                 struct tc_u_knode *knode;
117                 unsigned int      off;
118         } stack[TC_U32_MAXDEPTH];
119 
120         struct tc_u_hnode *ht = rcu_dereference_bh(tp->root);
121         unsigned int off = skb_network_offset(skb);
122         struct tc_u_knode *n;
123         int sdepth = 0;
124         int off2 = 0;
125         int sel = 0;
126 #ifdef CONFIG_CLS_U32_PERF
127         int j;
128 #endif
129         int i, r;
130 
131 next_ht:
132         n = rcu_dereference_bh(ht->ht[sel]);
133 
134 next_knode:
135         if (n) {
136                 struct tc_u32_key *key = n->sel.keys;
137 
138 #ifdef CONFIG_CLS_U32_PERF
139                 __this_cpu_inc(n->pf->rcnt);
140                 j = 0;
141 #endif
142 
143                 if (tc_skip_sw(n->flags)) {
144                         n = rcu_dereference_bh(n->next);
145                         goto next_knode;
146                 }
147 
148 #ifdef CONFIG_CLS_U32_MARK
149                 if ((skb->mark & n->mask) != n->val) {
150                         n = rcu_dereference_bh(n->next);
151                         goto next_knode;
152                 } else {
153                         __this_cpu_inc(*n->pcpu_success);
154                 }
155 #endif
156 
157                 for (i = n->sel.nkeys; i > 0; i--, key++) {
158                         int toff = off + key->off + (off2 & key->offmask);
159                         __be32 *data, hdata;
160 
161                         if (skb_headroom(skb) + toff > INT_MAX)
162                                 goto out;
163 
164                         data = skb_header_pointer(skb, toff, 4, &hdata);
165                         if (!data)
166                                 goto out;
167                         if ((*data ^ key->val) & key->mask) {
168                                 n = rcu_dereference_bh(n->next);
169                                 goto next_knode;
170                         }
171 #ifdef CONFIG_CLS_U32_PERF
172                         __this_cpu_inc(n->pf->kcnts[j]);
173                         j++;
174 #endif
175                 }
176 
177                 ht = rcu_dereference_bh(n->ht_down);
178                 if (!ht) {
179 check_terminal:
180                         if (n->sel.flags & TC_U32_TERMINAL) {
181 
182                                 *res = n->res;
183 #ifdef CONFIG_NET_CLS_IND
184                                 if (!tcf_match_indev(skb, n->ifindex)) {
185                                         n = rcu_dereference_bh(n->next);
186                                         goto next_knode;
187                                 }
188 #endif
189 #ifdef CONFIG_CLS_U32_PERF
190                                 __this_cpu_inc(n->pf->rhit);
191 #endif
192                                 r = tcf_exts_exec(skb, &n->exts, res);
193                                 if (r < 0) {
194                                         n = rcu_dereference_bh(n->next);
195                                         goto next_knode;
196                                 }
197 
198                                 return r;
199                         }
200                         n = rcu_dereference_bh(n->next);
201                         goto next_knode;
202                 }
203 
204                 /* PUSH */
205                 if (sdepth >= TC_U32_MAXDEPTH)
206                         goto deadloop;
207                 stack[sdepth].knode = n;
208                 stack[sdepth].off = off;
209                 sdepth++;
210 
211                 ht = rcu_dereference_bh(n->ht_down);
212                 sel = 0;
213                 if (ht->divisor) {
214                         __be32 *data, hdata;
215 
216                         data = skb_header_pointer(skb, off + n->sel.hoff, 4,
217                                                   &hdata);
218                         if (!data)
219                                 goto out;
220                         sel = ht->divisor & u32_hash_fold(*data, &n->sel,
221                                                           n->fshift);
222                 }
223                 if (!(n->sel.flags & (TC_U32_VAROFFSET | TC_U32_OFFSET | TC_U32_EAT)))
224                         goto next_ht;
225 
226                 if (n->sel.flags & (TC_U32_OFFSET | TC_U32_VAROFFSET)) {
227                         off2 = n->sel.off + 3;
228                         if (n->sel.flags & TC_U32_VAROFFSET) {
229                                 __be16 *data, hdata;
230 
231                                 data = skb_header_pointer(skb,
232                                                           off + n->sel.offoff,
233                                                           2, &hdata);
234                                 if (!data)
235                                         goto out;
236                                 off2 += ntohs(n->sel.offmask & *data) >>
237                                         n->sel.offshift;
238                         }
239                         off2 &= ~3;
240                 }
241                 if (n->sel.flags & TC_U32_EAT) {
242                         off += off2;
243                         off2 = 0;
244                 }
245 
246                 if (off < skb->len)
247                         goto next_ht;
248         }
249 
250         /* POP */
251         if (sdepth--) {
252                 n = stack[sdepth].knode;
253                 ht = rcu_dereference_bh(n->ht_up);
254                 off = stack[sdepth].off;
255                 goto check_terminal;
256         }
257 out:
258         return -1;
259 
260 deadloop:
261         net_warn_ratelimited("cls_u32: dead loop\n");
262         return -1;
263 }
264 
265 static struct tc_u_hnode *u32_lookup_ht(struct tc_u_common *tp_c, u32 handle)
266 {
267         struct tc_u_hnode *ht;
268 
269         for (ht = rtnl_dereference(tp_c->hlist);
270              ht;
271              ht = rtnl_dereference(ht->next))
272                 if (ht->handle == handle)
273                         break;
274 
275         return ht;
276 }
277 
278 static struct tc_u_knode *u32_lookup_key(struct tc_u_hnode *ht, u32 handle)
279 {
280         unsigned int sel;
281         struct tc_u_knode *n = NULL;
282 
283         sel = TC_U32_HASH(handle);
284         if (sel > ht->divisor)
285                 goto out;
286 
287         for (n = rtnl_dereference(ht->ht[sel]);
288              n;
289              n = rtnl_dereference(n->next))
290                 if (n->handle == handle)
291                         break;
292 out:
293         return n;
294 }
295 
296 
297 static void *u32_get(struct tcf_proto *tp, u32 handle)
298 {
299         struct tc_u_hnode *ht;
300         struct tc_u_common *tp_c = tp->data;
301 
302         if (TC_U32_HTID(handle) == TC_U32_ROOT)
303                 ht = rtnl_dereference(tp->root);
304         else
305                 ht = u32_lookup_ht(tp_c, TC_U32_HTID(handle));
306 
307         if (!ht)
308                 return NULL;
309 
310         if (TC_U32_KEY(handle) == 0)
311                 return ht;
312 
313         return u32_lookup_key(ht, handle);
314 }
315 
316 /* Protected by rtnl lock */
317 static u32 gen_new_htid(struct tc_u_common *tp_c, struct tc_u_hnode *ptr)
318 {
319         int id = idr_alloc_cyclic(&tp_c->handle_idr, ptr, 1, 0x7FF, GFP_KERNEL);
320         if (id < 0)
321                 return 0;
322         return (id | 0x800U) << 20;
323 }
324 
325 static struct hlist_head *tc_u_common_hash;
326 
327 #define U32_HASH_SHIFT 10
328 #define U32_HASH_SIZE (1 << U32_HASH_SHIFT)
329 
330 static void *tc_u_common_ptr(const struct tcf_proto *tp)
331 {
332         struct tcf_block *block = tp->chain->block;
333 
334         /* The block sharing is currently supported only
335          * for classless qdiscs. In that case we use block
336          * for tc_u_common identification. In case the
337          * block is not shared, block->q is a valid pointer
338          * and we can use that. That works for classful qdiscs.
339          */
340         if (tcf_block_shared(block))
341                 return block;
342         else
343                 return block->q;
344 }
345 
346 static unsigned int tc_u_hash(const struct tcf_proto *tp)
347 {
348         return hash_ptr(tc_u_common_ptr(tp), U32_HASH_SHIFT);
349 }
350 
351 static struct tc_u_common *tc_u_common_find(const struct tcf_proto *tp)
352 {
353         struct tc_u_common *tc;
354         unsigned int h;
355 
356         h = tc_u_hash(tp);
357         hlist_for_each_entry(tc, &tc_u_common_hash[h], hnode) {
358                 if (tc->ptr == tc_u_common_ptr(tp))
359                         return tc;
360         }
361         return NULL;
362 }
363 
364 static int u32_init(struct tcf_proto *tp)
365 {
366         struct tc_u_hnode *root_ht;
367         struct tc_u_common *tp_c;
368         unsigned int h;
369 
370         tp_c = tc_u_common_find(tp);
371 
372         root_ht = kzalloc(sizeof(*root_ht), GFP_KERNEL);
373         if (root_ht == NULL)
374                 return -ENOBUFS;
375 
376         root_ht->refcnt++;
377         root_ht->handle = tp_c ? gen_new_htid(tp_c, root_ht) : 0x80000000;
378         root_ht->prio = tp->prio;
379         idr_init(&root_ht->handle_idr);
380 
381         if (tp_c == NULL) {
382                 tp_c = kzalloc(sizeof(*tp_c), GFP_KERNEL);
383                 if (tp_c == NULL) {
384                         kfree(root_ht);
385                         return -ENOBUFS;
386                 }
387                 tp_c->ptr = tc_u_common_ptr(tp);
388                 INIT_HLIST_NODE(&tp_c->hnode);
389                 idr_init(&tp_c->handle_idr);
390 
391                 h = tc_u_hash(tp);
392                 hlist_add_head(&tp_c->hnode, &tc_u_common_hash[h]);
393         }
394 
395         tp_c->refcnt++;
396         RCU_INIT_POINTER(root_ht->next, tp_c->hlist);
397         rcu_assign_pointer(tp_c->hlist, root_ht);
398         root_ht->tp_c = tp_c;
399 
400         root_ht->refcnt++;
401         rcu_assign_pointer(tp->root, root_ht);
402         tp->data = tp_c;
403         return 0;
404 }
405 
406 static int u32_destroy_key(struct tcf_proto *tp, struct tc_u_knode *n,
407                            bool free_pf)
408 {
409         struct tc_u_hnode *ht = rtnl_dereference(n->ht_down);
410 
411         tcf_exts_destroy(&n->exts);
412         tcf_exts_put_net(&n->exts);
413         if (ht && --ht->refcnt == 0)
414                 kfree(ht);
415 #ifdef CONFIG_CLS_U32_PERF
416         if (free_pf)
417                 free_percpu(n->pf);
418 #endif
419 #ifdef CONFIG_CLS_U32_MARK
420         if (free_pf)
421                 free_percpu(n->pcpu_success);
422 #endif
423         kfree(n);
424         return 0;
425 }
426 
427 /* u32_delete_key_rcu should be called when free'ing a copied
428  * version of a tc_u_knode obtained from u32_init_knode(). When
429  * copies are obtained from u32_init_knode() the statistics are
430  * shared between the old and new copies to allow readers to
431  * continue to update the statistics during the copy. To support
432  * this the u32_delete_key_rcu variant does not free the percpu
433  * statistics.
434  */
435 static void u32_delete_key_work(struct work_struct *work)
436 {
437         struct tc_u_knode *key = container_of(to_rcu_work(work),
438                                               struct tc_u_knode,
439                                               rwork);
440         rtnl_lock();
441         u32_destroy_key(key->tp, key, false);
442         rtnl_unlock();
443 }
444 
445 /* u32_delete_key_freepf_rcu is the rcu callback variant
446  * that free's the entire structure including the statistics
447  * percpu variables. Only use this if the key is not a copy
448  * returned by u32_init_knode(). See u32_delete_key_rcu()
449  * for the variant that should be used with keys return from
450  * u32_init_knode()
451  */
452 static void u32_delete_key_freepf_work(struct work_struct *work)
453 {
454         struct tc_u_knode *key = container_of(to_rcu_work(work),
455                                               struct tc_u_knode,
456                                               rwork);
457         rtnl_lock();
458         u32_destroy_key(key->tp, key, true);
459         rtnl_unlock();
460 }
461 
462 static int u32_delete_key(struct tcf_proto *tp, struct tc_u_knode *key)
463 {
464         struct tc_u_knode __rcu **kp;
465         struct tc_u_knode *pkp;
466         struct tc_u_hnode *ht = rtnl_dereference(key->ht_up);
467 
468         if (ht) {
469                 kp = &ht->ht[TC_U32_HASH(key->handle)];
470                 for (pkp = rtnl_dereference(*kp); pkp;
471                      kp = &pkp->next, pkp = rtnl_dereference(*kp)) {
472                         if (pkp == key) {
473                                 RCU_INIT_POINTER(*kp, key->next);
474 
475                                 tcf_unbind_filter(tp, &key->res);
476                                 idr_remove(&ht->handle_idr, key->handle);
477                                 tcf_exts_get_net(&key->exts);
478                                 tcf_queue_work(&key->rwork, u32_delete_key_freepf_work);
479                                 return 0;
480                         }
481                 }
482         }
483         WARN_ON(1);
484         return 0;
485 }
486 
487 static void u32_clear_hw_hnode(struct tcf_proto *tp, struct tc_u_hnode *h,
488                                struct netlink_ext_ack *extack)
489 {
490         struct tcf_block *block = tp->chain->block;
491         struct tc_cls_u32_offload cls_u32 = {};
492 
493         tc_cls_common_offload_init(&cls_u32.common, tp, h->flags, extack);
494         cls_u32.command = TC_CLSU32_DELETE_HNODE;
495         cls_u32.hnode.divisor = h->divisor;
496         cls_u32.hnode.handle = h->handle;
497         cls_u32.hnode.prio = h->prio;
498 
499         tc_setup_cb_call(block, NULL, TC_SETUP_CLSU32, &cls_u32, false);
500 }
501 
502 static int u32_replace_hw_hnode(struct tcf_proto *tp, struct tc_u_hnode *h,
503                                 u32 flags, struct netlink_ext_ack *extack)
504 {
505         struct tcf_block *block = tp->chain->block;
506         struct tc_cls_u32_offload cls_u32 = {};
507         bool skip_sw = tc_skip_sw(flags);
508         bool offloaded = false;
509         int err;
510 
511         tc_cls_common_offload_init(&cls_u32.common, tp, flags, extack);
512         cls_u32.command = TC_CLSU32_NEW_HNODE;
513         cls_u32.hnode.divisor = h->divisor;
514         cls_u32.hnode.handle = h->handle;
515         cls_u32.hnode.prio = h->prio;
516 
517         err = tc_setup_cb_call(block, NULL, TC_SETUP_CLSU32, &cls_u32, skip_sw);
518         if (err < 0) {
519                 u32_clear_hw_hnode(tp, h, NULL);
520                 return err;
521         } else if (err > 0) {
522                 offloaded = true;
523         }
524 
525         if (skip_sw && !offloaded)
526                 return -EINVAL;
527 
528         return 0;
529 }
530 
531 static void u32_remove_hw_knode(struct tcf_proto *tp, struct tc_u_knode *n,
532                                 struct netlink_ext_ack *extack)
533 {
534         struct tcf_block *block = tp->chain->block;
535         struct tc_cls_u32_offload cls_u32 = {};
536 
537         tc_cls_common_offload_init(&cls_u32.common, tp, n->flags, extack);
538         cls_u32.command = TC_CLSU32_DELETE_KNODE;
539         cls_u32.knode.handle = n->handle;
540 
541         tc_setup_cb_call(block, NULL, TC_SETUP_CLSU32, &cls_u32, false);
542         tcf_block_offload_dec(block, &n->flags);
543 }
544 
545 static int u32_replace_hw_knode(struct tcf_proto *tp, struct tc_u_knode *n,
546                                 u32 flags, struct netlink_ext_ack *extack)
547 {
548         struct tc_u_hnode *ht = rtnl_dereference(n->ht_down);
549         struct tcf_block *block = tp->chain->block;
550         struct tc_cls_u32_offload cls_u32 = {};
551         bool skip_sw = tc_skip_sw(flags);
552         int err;
553 
554         tc_cls_common_offload_init(&cls_u32.common, tp, flags, extack);
555         cls_u32.command = TC_CLSU32_REPLACE_KNODE;
556         cls_u32.knode.handle = n->handle;
557         cls_u32.knode.fshift = n->fshift;
558 #ifdef CONFIG_CLS_U32_MARK
559         cls_u32.knode.val = n->val;
560         cls_u32.knode.mask = n->mask;
561 #else
562         cls_u32.knode.val = 0;
563         cls_u32.knode.mask = 0;
564 #endif
565         cls_u32.knode.sel = &n->sel;
566         cls_u32.knode.exts = &n->exts;
567         if (n->ht_down)
568                 cls_u32.knode.link_handle = ht->handle;
569 
570         err = tc_setup_cb_call(block, NULL, TC_SETUP_CLSU32, &cls_u32, skip_sw);
571         if (err < 0) {
572                 u32_remove_hw_knode(tp, n, NULL);
573                 return err;
574         } else if (err > 0) {
575                 tcf_block_offload_inc(block, &n->flags);
576         }
577 
578         if (skip_sw && !(n->flags & TCA_CLS_FLAGS_IN_HW))
579                 return -EINVAL;
580 
581         return 0;
582 }
583 
584 static void u32_clear_hnode(struct tcf_proto *tp, struct tc_u_hnode *ht,
585                             struct netlink_ext_ack *extack)
586 {
587         struct tc_u_knode *n;
588         unsigned int h;
589 
590         for (h = 0; h <= ht->divisor; h++) {
591                 while ((n = rtnl_dereference(ht->ht[h])) != NULL) {
592                         RCU_INIT_POINTER(ht->ht[h],
593                                          rtnl_dereference(n->next));
594                         tcf_unbind_filter(tp, &n->res);
595                         u32_remove_hw_knode(tp, n, extack);
596                         idr_remove(&ht->handle_idr, n->handle);
597                         if (tcf_exts_get_net(&n->exts))
598                                 tcf_queue_work(&n->rwork, u32_delete_key_freepf_work);
599                         else
600                                 u32_destroy_key(n->tp, n, true);
601                 }
602         }
603 }
604 
605 static int u32_destroy_hnode(struct tcf_proto *tp, struct tc_u_hnode *ht,
606                              struct netlink_ext_ack *extack)
607 {
608         struct tc_u_common *tp_c = tp->data;
609         struct tc_u_hnode __rcu **hn;
610         struct tc_u_hnode *phn;
611 
612         WARN_ON(--ht->refcnt);
613 
614         u32_clear_hnode(tp, ht, extack);
615 
616         hn = &tp_c->hlist;
617         for (phn = rtnl_dereference(*hn);
618              phn;
619              hn = &phn->next, phn = rtnl_dereference(*hn)) {
620                 if (phn == ht) {
621                         u32_clear_hw_hnode(tp, ht, extack);
622                         idr_destroy(&ht->handle_idr);
623                         idr_remove(&tp_c->handle_idr, ht->handle);
624                         RCU_INIT_POINTER(*hn, ht->next);
625                         kfree_rcu(ht, rcu);
626                         return 0;
627                 }
628         }
629 
630         return -ENOENT;
631 }
632 
633 static bool ht_empty(struct tc_u_hnode *ht)
634 {
635         unsigned int h;
636 
637         for (h = 0; h <= ht->divisor; h++)
638                 if (rcu_access_pointer(ht->ht[h]))
639                         return false;
640 
641         return true;
642 }
643 
644 static void u32_destroy(struct tcf_proto *tp, struct netlink_ext_ack *extack)
645 {
646         struct tc_u_common *tp_c = tp->data;
647         struct tc_u_hnode *root_ht = rtnl_dereference(tp->root);
648 
649         WARN_ON(root_ht == NULL);
650 
651         if (root_ht && --root_ht->refcnt == 1)
652                 u32_destroy_hnode(tp, root_ht, extack);
653 
654         if (--tp_c->refcnt == 0) {
655                 struct tc_u_hnode *ht;
656 
657                 hlist_del(&tp_c->hnode);
658 
659                 while ((ht = rtnl_dereference(tp_c->hlist)) != NULL) {
660                         u32_clear_hnode(tp, ht, extack);
661                         RCU_INIT_POINTER(tp_c->hlist, ht->next);
662 
663                         /* u32_destroy_key() will later free ht for us, if it's
664                          * still referenced by some knode
665                          */
666                         if (--ht->refcnt == 0)
667                                 kfree_rcu(ht, rcu);
668                 }
669 
670                 idr_destroy(&tp_c->handle_idr);
671                 kfree(tp_c);
672         }
673 
674         tp->data = NULL;
675 }
676 
677 static int u32_delete(struct tcf_proto *tp, void *arg, bool *last,
678                       struct netlink_ext_ack *extack)
679 {
680         struct tc_u_hnode *ht = arg;
681         struct tc_u_hnode *root_ht = rtnl_dereference(tp->root);
682         struct tc_u_common *tp_c = tp->data;
683         int ret = 0;
684 
685         if (ht == NULL)
686                 goto out;
687 
688         if (TC_U32_KEY(ht->handle)) {
689                 u32_remove_hw_knode(tp, (struct tc_u_knode *)ht, extack);
690                 ret = u32_delete_key(tp, (struct tc_u_knode *)ht);
691                 goto out;
692         }
693 
694         if (root_ht == ht) {
695                 NL_SET_ERR_MSG_MOD(extack, "Not allowed to delete root node");
696                 return -EINVAL;
697         }
698 
699         if (ht->refcnt == 1) {
700                 u32_destroy_hnode(tp, ht, extack);
701         } else {
702                 NL_SET_ERR_MSG_MOD(extack, "Can not delete in-use filter");
703                 return -EBUSY;
704         }
705 
706 out:
707         *last = true;
708         if (root_ht) {
709                 if (root_ht->refcnt > 2) {
710                         *last = false;
711                         goto ret;
712                 }
713                 if (root_ht->refcnt == 2) {
714                         if (!ht_empty(root_ht)) {
715                                 *last = false;
716                                 goto ret;
717                         }
718                 }
719         }
720 
721         if (tp_c->refcnt > 1) {
722                 *last = false;
723                 goto ret;
724         }
725 
726         if (tp_c->refcnt == 1) {
727                 struct tc_u_hnode *ht;
728 
729                 for (ht = rtnl_dereference(tp_c->hlist);
730                      ht;
731                      ht = rtnl_dereference(ht->next))
732                         if (!ht_empty(ht)) {
733                                 *last = false;
734                                 break;
735                         }
736         }
737 
738 ret:
739         return ret;
740 }
741 
742 static u32 gen_new_kid(struct tc_u_hnode *ht, u32 htid)
743 {
744         u32 index = htid | 0x800;
745         u32 max = htid | 0xFFF;
746 
747         if (idr_alloc_u32(&ht->handle_idr, NULL, &index, max, GFP_KERNEL)) {
748                 index = htid + 1;
749                 if (idr_alloc_u32(&ht->handle_idr, NULL, &index, max,
750                                  GFP_KERNEL))
751                         index = max;
752         }
753 
754         return index;
755 }
756 
757 static const struct nla_policy u32_policy[TCA_U32_MAX + 1] = {
758         [TCA_U32_CLASSID]       = { .type = NLA_U32 },
759         [TCA_U32_HASH]          = { .type = NLA_U32 },
760         [TCA_U32_LINK]          = { .type = NLA_U32 },
761         [TCA_U32_DIVISOR]       = { .type = NLA_U32 },
762         [TCA_U32_SEL]           = { .len = sizeof(struct tc_u32_sel) },
763         [TCA_U32_INDEV]         = { .type = NLA_STRING, .len = IFNAMSIZ },
764         [TCA_U32_MARK]          = { .len = sizeof(struct tc_u32_mark) },
765         [TCA_U32_FLAGS]         = { .type = NLA_U32 },
766 };
767 
768 static int u32_set_parms(struct net *net, struct tcf_proto *tp,
769                          unsigned long base, struct tc_u_hnode *ht,
770                          struct tc_u_knode *n, struct nlattr **tb,
771                          struct nlattr *est, bool ovr,
772                          struct netlink_ext_ack *extack)
773 {
774         int err;
775 
776         err = tcf_exts_validate(net, tp, tb, est, &n->exts, ovr, extack);
777         if (err < 0)
778                 return err;
779 
780         if (tb[TCA_U32_LINK]) {
781                 u32 handle = nla_get_u32(tb[TCA_U32_LINK]);
782                 struct tc_u_hnode *ht_down = NULL, *ht_old;
783 
784                 if (TC_U32_KEY(handle)) {
785                         NL_SET_ERR_MSG_MOD(extack, "u32 Link handle must be a hash table");
786                         return -EINVAL;
787                 }
788 
789                 if (handle) {
790                         ht_down = u32_lookup_ht(ht->tp_c, handle);
791 
792                         if (!ht_down) {
793                                 NL_SET_ERR_MSG_MOD(extack, "Link hash table not found");
794                                 return -EINVAL;
795                         }
796                         ht_down->refcnt++;
797                 }
798 
799                 ht_old = rtnl_dereference(n->ht_down);
800                 rcu_assign_pointer(n->ht_down, ht_down);
801 
802                 if (ht_old)
803                         ht_old->refcnt--;
804         }
805         if (tb[TCA_U32_CLASSID]) {
806                 n->res.classid = nla_get_u32(tb[TCA_U32_CLASSID]);
807                 tcf_bind_filter(tp, &n->res, base);
808         }
809 
810 #ifdef CONFIG_NET_CLS_IND
811         if (tb[TCA_U32_INDEV]) {
812                 int ret;
813                 ret = tcf_change_indev(net, tb[TCA_U32_INDEV], extack);
814                 if (ret < 0)
815                         return -EINVAL;
816                 n->ifindex = ret;
817         }
818 #endif
819         return 0;
820 }
821 
822 static void u32_replace_knode(struct tcf_proto *tp, struct tc_u_common *tp_c,
823                               struct tc_u_knode *n)
824 {
825         struct tc_u_knode __rcu **ins;
826         struct tc_u_knode *pins;
827         struct tc_u_hnode *ht;
828 
829         if (TC_U32_HTID(n->handle) == TC_U32_ROOT)
830                 ht = rtnl_dereference(tp->root);
831         else
832                 ht = u32_lookup_ht(tp_c, TC_U32_HTID(n->handle));
833 
834         ins = &ht->ht[TC_U32_HASH(n->handle)];
835 
836         /* The node must always exist for it to be replaced if this is not the
837          * case then something went very wrong elsewhere.
838          */
839         for (pins = rtnl_dereference(*ins); ;
840              ins = &pins->next, pins = rtnl_dereference(*ins))
841                 if (pins->handle == n->handle)
842                         break;
843 
844         idr_replace(&ht->handle_idr, n, n->handle);
845         RCU_INIT_POINTER(n->next, pins->next);
846         rcu_assign_pointer(*ins, n);
847 }
848 
849 static struct tc_u_knode *u32_init_knode(struct tcf_proto *tp,
850                                          struct tc_u_knode *n)
851 {
852         struct tc_u_hnode *ht = rtnl_dereference(n->ht_down);
853         struct tc_u32_sel *s = &n->sel;
854         struct tc_u_knode *new;
855 
856         new = kzalloc(sizeof(*n) + s->nkeys*sizeof(struct tc_u32_key),
857                       GFP_KERNEL);
858 
859         if (!new)
860                 return NULL;
861 
862         RCU_INIT_POINTER(new->next, n->next);
863         new->handle = n->handle;
864         RCU_INIT_POINTER(new->ht_up, n->ht_up);
865 
866 #ifdef CONFIG_NET_CLS_IND
867         new->ifindex = n->ifindex;
868 #endif
869         new->fshift = n->fshift;
870         new->res = n->res;
871         new->flags = n->flags;
872         RCU_INIT_POINTER(new->ht_down, ht);
873 
874         /* bump reference count as long as we hold pointer to structure */
875         if (ht)
876                 ht->refcnt++;
877 
878 #ifdef CONFIG_CLS_U32_PERF
879         /* Statistics may be incremented by readers during update
880          * so we must keep them in tact. When the node is later destroyed
881          * a special destroy call must be made to not free the pf memory.
882          */
883         new->pf = n->pf;
884 #endif
885 
886 #ifdef CONFIG_CLS_U32_MARK
887         new->val = n->val;
888         new->mask = n->mask;
889         /* Similarly success statistics must be moved as pointers */
890         new->pcpu_success = n->pcpu_success;
891 #endif
892         new->tp = tp;
893         memcpy(&new->sel, s, sizeof(*s) + s->nkeys*sizeof(struct tc_u32_key));
894 
895         if (tcf_exts_init(&new->exts, TCA_U32_ACT, TCA_U32_POLICE)) {
896                 kfree(new);
897                 return NULL;
898         }
899 
900         return new;
901 }
902 
903 static int u32_change(struct net *net, struct sk_buff *in_skb,
904                       struct tcf_proto *tp, unsigned long base, u32 handle,
905                       struct nlattr **tca, void **arg, bool ovr,
906                       struct netlink_ext_ack *extack)
907 {
908         struct tc_u_common *tp_c = tp->data;
909         struct tc_u_hnode *ht;
910         struct tc_u_knode *n;
911         struct tc_u32_sel *s;
912         struct nlattr *opt = tca[TCA_OPTIONS];
913         struct nlattr *tb[TCA_U32_MAX + 1];
914         u32 htid, flags = 0;
915         size_t sel_size;
916         int err;
917 #ifdef CONFIG_CLS_U32_PERF
918         size_t size;
919 #endif
920 
921         if (!opt) {
922                 if (handle) {
923                         NL_SET_ERR_MSG_MOD(extack, "Filter handle requires options");
924                         return -EINVAL;
925                 } else {
926                         return 0;
927                 }
928         }
929 
930         err = nla_parse_nested(tb, TCA_U32_MAX, opt, u32_policy, extack);
931         if (err < 0)
932                 return err;
933 
934         if (tb[TCA_U32_FLAGS]) {
935                 flags = nla_get_u32(tb[TCA_U32_FLAGS]);
936                 if (!tc_flags_valid(flags)) {
937                         NL_SET_ERR_MSG_MOD(extack, "Invalid filter flags");
938                         return -EINVAL;
939                 }
940         }
941 
942         n = *arg;
943         if (n) {
944                 struct tc_u_knode *new;
945 
946                 if (TC_U32_KEY(n->handle) == 0) {
947                         NL_SET_ERR_MSG_MOD(extack, "Key node id cannot be zero");
948                         return -EINVAL;
949                 }
950 
951                 if ((n->flags ^ flags) &
952                     ~(TCA_CLS_FLAGS_IN_HW | TCA_CLS_FLAGS_NOT_IN_HW)) {
953                         NL_SET_ERR_MSG_MOD(extack, "Key node flags do not match passed flags");
954                         return -EINVAL;
955                 }
956 
957                 new = u32_init_knode(tp, n);
958                 if (!new)
959                         return -ENOMEM;
960 
961                 err = u32_set_parms(net, tp, base,
962                                     rtnl_dereference(n->ht_up), new, tb,
963                                     tca[TCA_RATE], ovr, extack);
964 
965                 if (err) {
966                         u32_destroy_key(tp, new, false);
967                         return err;
968                 }
969 
970                 err = u32_replace_hw_knode(tp, new, flags, extack);
971                 if (err) {
972                         u32_destroy_key(tp, new, false);
973                         return err;
974                 }
975 
976                 if (!tc_in_hw(new->flags))
977                         new->flags |= TCA_CLS_FLAGS_NOT_IN_HW;
978 
979                 u32_replace_knode(tp, tp_c, new);
980                 tcf_unbind_filter(tp, &n->res);
981                 tcf_exts_get_net(&n->exts);
982                 tcf_queue_work(&n->rwork, u32_delete_key_work);
983                 return 0;
984         }
985 
986         if (tb[TCA_U32_DIVISOR]) {
987                 unsigned int divisor = nla_get_u32(tb[TCA_U32_DIVISOR]);
988 
989                 if (--divisor > 0x100) {
990                         NL_SET_ERR_MSG_MOD(extack, "Exceeded maximum 256 hash buckets");
991                         return -EINVAL;
992                 }
993                 if (TC_U32_KEY(handle)) {
994                         NL_SET_ERR_MSG_MOD(extack, "Divisor can only be used on a hash table");
995                         return -EINVAL;
996                 }
997                 ht = kzalloc(sizeof(*ht) + divisor*sizeof(void *), GFP_KERNEL);
998                 if (ht == NULL)
999                         return -ENOBUFS;
1000                 if (handle == 0) {
1001                         handle = gen_new_htid(tp->data, ht);
1002                         if (handle == 0) {
1003                                 kfree(ht);
1004                                 return -ENOMEM;
1005                         }
1006                 } else {
1007                         err = idr_alloc_u32(&tp_c->handle_idr, ht, &handle,
1008                                             handle, GFP_KERNEL);
1009                         if (err) {
1010                                 kfree(ht);
1011                                 return err;
1012                         }
1013                 }
1014                 ht->tp_c = tp_c;
1015                 ht->refcnt = 1;
1016                 ht->divisor = divisor;
1017                 ht->handle = handle;
1018                 ht->prio = tp->prio;
1019                 idr_init(&ht->handle_idr);
1020                 ht->flags = flags;
1021 
1022                 err = u32_replace_hw_hnode(tp, ht, flags, extack);
1023                 if (err) {
1024                         idr_remove(&tp_c->handle_idr, handle);
1025                         kfree(ht);
1026                         return err;
1027                 }
1028 
1029                 RCU_INIT_POINTER(ht->next, tp_c->hlist);
1030                 rcu_assign_pointer(tp_c->hlist, ht);
1031                 *arg = ht;
1032 
1033                 return 0;
1034         }
1035 
1036         if (tb[TCA_U32_HASH]) {
1037                 htid = nla_get_u32(tb[TCA_U32_HASH]);
1038                 if (TC_U32_HTID(htid) == TC_U32_ROOT) {
1039                         ht = rtnl_dereference(tp->root);
1040                         htid = ht->handle;
1041                 } else {
1042                         ht = u32_lookup_ht(tp->data, TC_U32_HTID(htid));
1043                         if (!ht) {
1044                                 NL_SET_ERR_MSG_MOD(extack, "Specified hash table not found");
1045                                 return -EINVAL;
1046                         }
1047                 }
1048         } else {
1049                 ht = rtnl_dereference(tp->root);
1050                 htid = ht->handle;
1051         }
1052 
1053         if (ht->divisor < TC_U32_HASH(htid)) {
1054                 NL_SET_ERR_MSG_MOD(extack, "Specified hash table buckets exceed configured value");
1055                 return -EINVAL;
1056         }
1057 
1058         if (handle) {
1059                 if (TC_U32_HTID(handle) && TC_U32_HTID(handle ^ htid)) {
1060                         NL_SET_ERR_MSG_MOD(extack, "Handle specified hash table address mismatch");
1061                         return -EINVAL;
1062                 }
1063                 handle = htid | TC_U32_NODE(handle);
1064                 err = idr_alloc_u32(&ht->handle_idr, NULL, &handle, handle,
1065                                     GFP_KERNEL);
1066                 if (err)
1067                         return err;
1068         } else
1069                 handle = gen_new_kid(ht, htid);
1070 
1071         if (tb[TCA_U32_SEL] == NULL) {
1072                 NL_SET_ERR_MSG_MOD(extack, "Selector not specified");
1073                 err = -EINVAL;
1074                 goto erridr;
1075         }
1076 
1077         s = nla_data(tb[TCA_U32_SEL]);
1078         sel_size = struct_size(s, keys, s->nkeys);
1079         if (nla_len(tb[TCA_U32_SEL]) < sel_size) {
1080                 err = -EINVAL;
1081                 goto erridr;
1082         }
1083 
1084         n = kzalloc(offsetof(typeof(*n), sel) + sel_size, GFP_KERNEL);
1085         if (n == NULL) {
1086                 err = -ENOBUFS;
1087                 goto erridr;
1088         }
1089 
1090 #ifdef CONFIG_CLS_U32_PERF
1091         size = sizeof(struct tc_u32_pcnt) + s->nkeys * sizeof(u64);
1092         n->pf = __alloc_percpu(size, __alignof__(struct tc_u32_pcnt));
1093         if (!n->pf) {
1094                 err = -ENOBUFS;
1095                 goto errfree;
1096         }
1097 #endif
1098 
1099         memcpy(&n->sel, s, sel_size);
1100         RCU_INIT_POINTER(n->ht_up, ht);
1101         n->handle = handle;
1102         n->fshift = s->hmask ? ffs(ntohl(s->hmask)) - 1 : 0;
1103         n->flags = flags;
1104         n->tp = tp;
1105 
1106         err = tcf_exts_init(&n->exts, TCA_U32_ACT, TCA_U32_POLICE);
1107         if (err < 0)
1108                 goto errout;
1109 
1110 #ifdef CONFIG_CLS_U32_MARK
1111         n->pcpu_success = alloc_percpu(u32);
1112         if (!n->pcpu_success) {
1113                 err = -ENOMEM;
1114                 goto errout;
1115         }
1116 
1117         if (tb[TCA_U32_MARK]) {
1118                 struct tc_u32_mark *mark;
1119 
1120                 mark = nla_data(tb[TCA_U32_MARK]);
1121                 n->val = mark->val;
1122                 n->mask = mark->mask;
1123         }
1124 #endif
1125 
1126         err = u32_set_parms(net, tp, base, ht, n, tb, tca[TCA_RATE], ovr,
1127                             extack);
1128         if (err == 0) {
1129                 struct tc_u_knode __rcu **ins;
1130                 struct tc_u_knode *pins;
1131 
1132                 err = u32_replace_hw_knode(tp, n, flags, extack);
1133                 if (err)
1134                         goto errhw;
1135 
1136                 if (!tc_in_hw(n->flags))
1137                         n->flags |= TCA_CLS_FLAGS_NOT_IN_HW;
1138 
1139                 ins = &ht->ht[TC_U32_HASH(handle)];
1140                 for (pins = rtnl_dereference(*ins); pins;
1141                      ins = &pins->next, pins = rtnl_dereference(*ins))
1142                         if (TC_U32_NODE(handle) < TC_U32_NODE(pins->handle))
1143                                 break;
1144 
1145                 RCU_INIT_POINTER(n->next, pins);
1146                 rcu_assign_pointer(*ins, n);
1147                 *arg = n;
1148                 return 0;
1149         }
1150 
1151 errhw:
1152 #ifdef CONFIG_CLS_U32_MARK
1153         free_percpu(n->pcpu_success);
1154 #endif
1155 
1156 errout:
1157         tcf_exts_destroy(&n->exts);
1158 #ifdef CONFIG_CLS_U32_PERF
1159 errfree:
1160         free_percpu(n->pf);
1161 #endif
1162         kfree(n);
1163 erridr:
1164         idr_remove(&ht->handle_idr, handle);
1165         return err;
1166 }
1167 
1168 static void u32_walk(struct tcf_proto *tp, struct tcf_walker *arg)
1169 {
1170         struct tc_u_common *tp_c = tp->data;
1171         struct tc_u_hnode *ht;
1172         struct tc_u_knode *n;
1173         unsigned int h;
1174 
1175         if (arg->stop)
1176                 return;
1177 
1178         for (ht = rtnl_dereference(tp_c->hlist);
1179              ht;
1180              ht = rtnl_dereference(ht->next)) {
1181                 if (ht->prio != tp->prio)
1182                         continue;
1183                 if (arg->count >= arg->skip) {
1184                         if (arg->fn(tp, ht, arg) < 0) {
1185                                 arg->stop = 1;
1186                                 return;
1187                         }
1188                 }
1189                 arg->count++;
1190                 for (h = 0; h <= ht->divisor; h++) {
1191                         for (n = rtnl_dereference(ht->ht[h]);
1192                              n;
1193                              n = rtnl_dereference(n->next)) {
1194                                 if (arg->count < arg->skip) {
1195                                         arg->count++;
1196                                         continue;
1197                                 }
1198                                 if (arg->fn(tp, n, arg) < 0) {
1199                                         arg->stop = 1;
1200                                         return;
1201                                 }
1202                                 arg->count++;
1203                         }
1204                 }
1205         }
1206 }
1207 
1208 static void u32_bind_class(void *fh, u32 classid, unsigned long cl)
1209 {
1210         struct tc_u_knode *n = fh;
1211 
1212         if (n && n->res.classid == classid)
1213                 n->res.class = cl;
1214 }
1215 
1216 static int u32_dump(struct net *net, struct tcf_proto *tp, void *fh,
1217                     struct sk_buff *skb, struct tcmsg *t)
1218 {
1219         struct tc_u_knode *n = fh;
1220         struct tc_u_hnode *ht_up, *ht_down;
1221         struct nlattr *nest;
1222 
1223         if (n == NULL)
1224                 return skb->len;
1225 
1226         t->tcm_handle = n->handle;
1227 
1228         nest = nla_nest_start(skb, TCA_OPTIONS);
1229         if (nest == NULL)
1230                 goto nla_put_failure;
1231 
1232         if (TC_U32_KEY(n->handle) == 0) {
1233                 struct tc_u_hnode *ht = fh;
1234                 u32 divisor = ht->divisor + 1;
1235 
1236                 if (nla_put_u32(skb, TCA_U32_DIVISOR, divisor))
1237                         goto nla_put_failure;
1238         } else {
1239 #ifdef CONFIG_CLS_U32_PERF
1240                 struct tc_u32_pcnt *gpf;
1241                 int cpu;
1242 #endif
1243 
1244                 if (nla_put(skb, TCA_U32_SEL,
1245                             sizeof(n->sel) + n->sel.nkeys*sizeof(struct tc_u32_key),
1246                             &n->sel))
1247                         goto nla_put_failure;
1248 
1249                 ht_up = rtnl_dereference(n->ht_up);
1250                 if (ht_up) {
1251                         u32 htid = n->handle & 0xFFFFF000;
1252                         if (nla_put_u32(skb, TCA_U32_HASH, htid))
1253                                 goto nla_put_failure;
1254                 }
1255                 if (n->res.classid &&
1256                     nla_put_u32(skb, TCA_U32_CLASSID, n->res.classid))
1257                         goto nla_put_failure;
1258 
1259                 ht_down = rtnl_dereference(n->ht_down);
1260                 if (ht_down &&
1261                     nla_put_u32(skb, TCA_U32_LINK, ht_down->handle))
1262                         goto nla_put_failure;
1263 
1264                 if (n->flags && nla_put_u32(skb, TCA_U32_FLAGS, n->flags))
1265                         goto nla_put_failure;
1266 
1267 #ifdef CONFIG_CLS_U32_MARK
1268                 if ((n->val || n->mask)) {
1269                         struct tc_u32_mark mark = {.val = n->val,
1270                                                    .mask = n->mask,
1271                                                    .success = 0};
1272                         int cpum;
1273 
1274                         for_each_possible_cpu(cpum) {
1275                                 __u32 cnt = *per_cpu_ptr(n->pcpu_success, cpum);
1276 
1277                                 mark.success += cnt;
1278                         }
1279 
1280                         if (nla_put(skb, TCA_U32_MARK, sizeof(mark), &mark))
1281                                 goto nla_put_failure;
1282                 }
1283 #endif
1284 
1285                 if (tcf_exts_dump(skb, &n->exts) < 0)
1286                         goto nla_put_failure;
1287 
1288 #ifdef CONFIG_NET_CLS_IND
1289                 if (n->ifindex) {
1290                         struct net_device *dev;
1291                         dev = __dev_get_by_index(net, n->ifindex);
1292                         if (dev && nla_put_string(skb, TCA_U32_INDEV, dev->name))
1293                                 goto nla_put_failure;
1294                 }
1295 #endif
1296 #ifdef CONFIG_CLS_U32_PERF
1297                 gpf = kzalloc(sizeof(struct tc_u32_pcnt) +
1298                               n->sel.nkeys * sizeof(u64),
1299                               GFP_KERNEL);
1300                 if (!gpf)
1301                         goto nla_put_failure;
1302 
1303                 for_each_possible_cpu(cpu) {
1304                         int i;
1305                         struct tc_u32_pcnt *pf = per_cpu_ptr(n->pf, cpu);
1306 
1307                         gpf->rcnt += pf->rcnt;
1308                         gpf->rhit += pf->rhit;
1309                         for (i = 0; i < n->sel.nkeys; i++)
1310                                 gpf->kcnts[i] += pf->kcnts[i];
1311                 }
1312 
1313                 if (nla_put_64bit(skb, TCA_U32_PCNT,
1314                                   sizeof(struct tc_u32_pcnt) +
1315                                   n->sel.nkeys * sizeof(u64),
1316                                   gpf, TCA_U32_PAD)) {
1317                         kfree(gpf);
1318                         goto nla_put_failure;
1319                 }
1320                 kfree(gpf);
1321 #endif
1322         }
1323 
1324         nla_nest_end(skb, nest);
1325 
1326         if (TC_U32_KEY(n->handle))
1327                 if (tcf_exts_dump_stats(skb, &n->exts) < 0)
1328                         goto nla_put_failure;
1329         return skb->len;
1330 
1331 nla_put_failure:
1332         nla_nest_cancel(skb, nest);
1333         return -1;
1334 }
1335 
1336 static struct tcf_proto_ops cls_u32_ops __read_mostly = {
1337         .kind           =       "u32",
1338         .classify       =       u32_classify,
1339         .init           =       u32_init,
1340         .destroy        =       u32_destroy,
1341         .get            =       u32_get,
1342         .change         =       u32_change,
1343         .delete         =       u32_delete,
1344         .walk           =       u32_walk,
1345         .dump           =       u32_dump,
1346         .bind_class     =       u32_bind_class,
1347         .owner          =       THIS_MODULE,
1348 };
1349 
1350 static int __init init_u32(void)
1351 {
1352         int i, ret;
1353 
1354         pr_info("u32 classifier\n");
1355 #ifdef CONFIG_CLS_U32_PERF
1356         pr_info("    Performance counters on\n");
1357 #endif
1358 #ifdef CONFIG_NET_CLS_IND
1359         pr_info("    input device check on\n");
1360 #endif
1361 #ifdef CONFIG_NET_CLS_ACT
1362         pr_info("    Actions configured\n");
1363 #endif
1364         tc_u_common_hash = kvmalloc_array(U32_HASH_SIZE,
1365                                           sizeof(struct hlist_head),
1366                                           GFP_KERNEL);
1367         if (!tc_u_common_hash)
1368                 return -ENOMEM;
1369 
1370         for (i = 0; i < U32_HASH_SIZE; i++)
1371                 INIT_HLIST_HEAD(&tc_u_common_hash[i]);
1372 
1373         ret = register_tcf_proto_ops(&cls_u32_ops);
1374         if (ret)
1375                 kvfree(tc_u_common_hash);
1376         return ret;
1377 }
1378 
1379 static void __exit exit_u32(void)
1380 {
1381         unregister_tcf_proto_ops(&cls_u32_ops);
1382         kvfree(tc_u_common_hash);
1383 }
1384 
1385 module_init(init_u32)
1386 module_exit(exit_u32)
1387 MODULE_LICENSE("GPL");
1388 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp