~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/net/sched/sch_generic.c

Version: ~ [ linux-5.9 ] ~ [ linux-5.8.14 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.70 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.150 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.200 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.238 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.238 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-or-later
  2 /*
  3  * net/sched/sch_generic.c      Generic packet scheduler routines.
  4  *
  5  * Authors:     Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  6  *              Jamal Hadi Salim, <hadi@cyberus.ca> 990601
  7  *              - Ingress support
  8  */
  9 
 10 #include <linux/bitops.h>
 11 #include <linux/module.h>
 12 #include <linux/types.h>
 13 #include <linux/kernel.h>
 14 #include <linux/sched.h>
 15 #include <linux/string.h>
 16 #include <linux/errno.h>
 17 #include <linux/netdevice.h>
 18 #include <linux/skbuff.h>
 19 #include <linux/rtnetlink.h>
 20 #include <linux/init.h>
 21 #include <linux/rcupdate.h>
 22 #include <linux/list.h>
 23 #include <linux/slab.h>
 24 #include <linux/if_vlan.h>
 25 #include <linux/skb_array.h>
 26 #include <linux/if_macvlan.h>
 27 #include <net/sch_generic.h>
 28 #include <net/pkt_sched.h>
 29 #include <net/dst.h>
 30 #include <trace/events/qdisc.h>
 31 #include <trace/events/net.h>
 32 #include <net/xfrm.h>
 33 
 34 /* Qdisc to use by default */
 35 const struct Qdisc_ops *default_qdisc_ops = &pfifo_fast_ops;
 36 EXPORT_SYMBOL(default_qdisc_ops);
 37 
 38 /* Main transmission queue. */
 39 
 40 /* Modifications to data participating in scheduling must be protected with
 41  * qdisc_lock(qdisc) spinlock.
 42  *
 43  * The idea is the following:
 44  * - enqueue, dequeue are serialized via qdisc root lock
 45  * - ingress filtering is also serialized via qdisc root lock
 46  * - updates to tree and tree walking are only done under the rtnl mutex.
 47  */
 48 
 49 #define SKB_XOFF_MAGIC ((struct sk_buff *)1UL)
 50 
 51 static inline struct sk_buff *__skb_dequeue_bad_txq(struct Qdisc *q)
 52 {
 53         const struct netdev_queue *txq = q->dev_queue;
 54         spinlock_t *lock = NULL;
 55         struct sk_buff *skb;
 56 
 57         if (q->flags & TCQ_F_NOLOCK) {
 58                 lock = qdisc_lock(q);
 59                 spin_lock(lock);
 60         }
 61 
 62         skb = skb_peek(&q->skb_bad_txq);
 63         if (skb) {
 64                 /* check the reason of requeuing without tx lock first */
 65                 txq = skb_get_tx_queue(txq->dev, skb);
 66                 if (!netif_xmit_frozen_or_stopped(txq)) {
 67                         skb = __skb_dequeue(&q->skb_bad_txq);
 68                         if (qdisc_is_percpu_stats(q)) {
 69                                 qdisc_qstats_cpu_backlog_dec(q, skb);
 70                                 qdisc_qstats_cpu_qlen_dec(q);
 71                         } else {
 72                                 qdisc_qstats_backlog_dec(q, skb);
 73                                 q->q.qlen--;
 74                         }
 75                 } else {
 76                         skb = SKB_XOFF_MAGIC;
 77                 }
 78         }
 79 
 80         if (lock)
 81                 spin_unlock(lock);
 82 
 83         return skb;
 84 }
 85 
 86 static inline struct sk_buff *qdisc_dequeue_skb_bad_txq(struct Qdisc *q)
 87 {
 88         struct sk_buff *skb = skb_peek(&q->skb_bad_txq);
 89 
 90         if (unlikely(skb))
 91                 skb = __skb_dequeue_bad_txq(q);
 92 
 93         return skb;
 94 }
 95 
 96 static inline void qdisc_enqueue_skb_bad_txq(struct Qdisc *q,
 97                                              struct sk_buff *skb)
 98 {
 99         spinlock_t *lock = NULL;
100 
101         if (q->flags & TCQ_F_NOLOCK) {
102                 lock = qdisc_lock(q);
103                 spin_lock(lock);
104         }
105 
106         __skb_queue_tail(&q->skb_bad_txq, skb);
107 
108         if (qdisc_is_percpu_stats(q)) {
109                 qdisc_qstats_cpu_backlog_inc(q, skb);
110                 qdisc_qstats_cpu_qlen_inc(q);
111         } else {
112                 qdisc_qstats_backlog_inc(q, skb);
113                 q->q.qlen++;
114         }
115 
116         if (lock)
117                 spin_unlock(lock);
118 }
119 
120 static inline void dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q)
121 {
122         spinlock_t *lock = NULL;
123 
124         if (q->flags & TCQ_F_NOLOCK) {
125                 lock = qdisc_lock(q);
126                 spin_lock(lock);
127         }
128 
129         while (skb) {
130                 struct sk_buff *next = skb->next;
131 
132                 __skb_queue_tail(&q->gso_skb, skb);
133 
134                 /* it's still part of the queue */
135                 if (qdisc_is_percpu_stats(q)) {
136                         qdisc_qstats_cpu_requeues_inc(q);
137                         qdisc_qstats_cpu_backlog_inc(q, skb);
138                         qdisc_qstats_cpu_qlen_inc(q);
139                 } else {
140                         q->qstats.requeues++;
141                         qdisc_qstats_backlog_inc(q, skb);
142                         q->q.qlen++;
143                 }
144 
145                 skb = next;
146         }
147         if (lock)
148                 spin_unlock(lock);
149         __netif_schedule(q);
150 }
151 
152 static void try_bulk_dequeue_skb(struct Qdisc *q,
153                                  struct sk_buff *skb,
154                                  const struct netdev_queue *txq,
155                                  int *packets)
156 {
157         int bytelimit = qdisc_avail_bulklimit(txq) - skb->len;
158 
159         while (bytelimit > 0) {
160                 struct sk_buff *nskb = q->dequeue(q);
161 
162                 if (!nskb)
163                         break;
164 
165                 bytelimit -= nskb->len; /* covers GSO len */
166                 skb->next = nskb;
167                 skb = nskb;
168                 (*packets)++; /* GSO counts as one pkt */
169         }
170         skb_mark_not_on_list(skb);
171 }
172 
173 /* This variant of try_bulk_dequeue_skb() makes sure
174  * all skbs in the chain are for the same txq
175  */
176 static void try_bulk_dequeue_skb_slow(struct Qdisc *q,
177                                       struct sk_buff *skb,
178                                       int *packets)
179 {
180         int mapping = skb_get_queue_mapping(skb);
181         struct sk_buff *nskb;
182         int cnt = 0;
183 
184         do {
185                 nskb = q->dequeue(q);
186                 if (!nskb)
187                         break;
188                 if (unlikely(skb_get_queue_mapping(nskb) != mapping)) {
189                         qdisc_enqueue_skb_bad_txq(q, nskb);
190                         break;
191                 }
192                 skb->next = nskb;
193                 skb = nskb;
194         } while (++cnt < 8);
195         (*packets) += cnt;
196         skb_mark_not_on_list(skb);
197 }
198 
199 /* Note that dequeue_skb can possibly return a SKB list (via skb->next).
200  * A requeued skb (via q->gso_skb) can also be a SKB list.
201  */
202 static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate,
203                                    int *packets)
204 {
205         const struct netdev_queue *txq = q->dev_queue;
206         struct sk_buff *skb = NULL;
207 
208         *packets = 1;
209         if (unlikely(!skb_queue_empty(&q->gso_skb))) {
210                 spinlock_t *lock = NULL;
211 
212                 if (q->flags & TCQ_F_NOLOCK) {
213                         lock = qdisc_lock(q);
214                         spin_lock(lock);
215                 }
216 
217                 skb = skb_peek(&q->gso_skb);
218 
219                 /* skb may be null if another cpu pulls gso_skb off in between
220                  * empty check and lock.
221                  */
222                 if (!skb) {
223                         if (lock)
224                                 spin_unlock(lock);
225                         goto validate;
226                 }
227 
228                 /* skb in gso_skb were already validated */
229                 *validate = false;
230                 if (xfrm_offload(skb))
231                         *validate = true;
232                 /* check the reason of requeuing without tx lock first */
233                 txq = skb_get_tx_queue(txq->dev, skb);
234                 if (!netif_xmit_frozen_or_stopped(txq)) {
235                         skb = __skb_dequeue(&q->gso_skb);
236                         if (qdisc_is_percpu_stats(q)) {
237                                 qdisc_qstats_cpu_backlog_dec(q, skb);
238                                 qdisc_qstats_cpu_qlen_dec(q);
239                         } else {
240                                 qdisc_qstats_backlog_dec(q, skb);
241                                 q->q.qlen--;
242                         }
243                 } else {
244                         skb = NULL;
245                 }
246                 if (lock)
247                         spin_unlock(lock);
248                 goto trace;
249         }
250 validate:
251         *validate = true;
252 
253         if ((q->flags & TCQ_F_ONETXQUEUE) &&
254             netif_xmit_frozen_or_stopped(txq))
255                 return skb;
256 
257         skb = qdisc_dequeue_skb_bad_txq(q);
258         if (unlikely(skb)) {
259                 if (skb == SKB_XOFF_MAGIC)
260                         return NULL;
261                 goto bulk;
262         }
263         skb = q->dequeue(q);
264         if (skb) {
265 bulk:
266                 if (qdisc_may_bulk(q))
267                         try_bulk_dequeue_skb(q, skb, txq, packets);
268                 else
269                         try_bulk_dequeue_skb_slow(q, skb, packets);
270         }
271 trace:
272         trace_qdisc_dequeue(q, txq, *packets, skb);
273         return skb;
274 }
275 
276 /*
277  * Transmit possibly several skbs, and handle the return status as
278  * required. Owning running seqcount bit guarantees that
279  * only one CPU can execute this function.
280  *
281  * Returns to the caller:
282  *                              false  - hardware queue frozen backoff
283  *                              true   - feel free to send more pkts
284  */
285 bool sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q,
286                      struct net_device *dev, struct netdev_queue *txq,
287                      spinlock_t *root_lock, bool validate)
288 {
289         int ret = NETDEV_TX_BUSY;
290         bool again = false;
291 
292         /* And release qdisc */
293         if (root_lock)
294                 spin_unlock(root_lock);
295 
296         /* Note that we validate skb (GSO, checksum, ...) outside of locks */
297         if (validate)
298                 skb = validate_xmit_skb_list(skb, dev, &again);
299 
300 #ifdef CONFIG_XFRM_OFFLOAD
301         if (unlikely(again)) {
302                 if (root_lock)
303                         spin_lock(root_lock);
304 
305                 dev_requeue_skb(skb, q);
306                 return false;
307         }
308 #endif
309 
310         if (likely(skb)) {
311                 HARD_TX_LOCK(dev, txq, smp_processor_id());
312                 if (!netif_xmit_frozen_or_stopped(txq))
313                         skb = dev_hard_start_xmit(skb, dev, txq, &ret);
314 
315                 HARD_TX_UNLOCK(dev, txq);
316         } else {
317                 if (root_lock)
318                         spin_lock(root_lock);
319                 return true;
320         }
321 
322         if (root_lock)
323                 spin_lock(root_lock);
324 
325         if (!dev_xmit_complete(ret)) {
326                 /* Driver returned NETDEV_TX_BUSY - requeue skb */
327                 if (unlikely(ret != NETDEV_TX_BUSY))
328                         net_warn_ratelimited("BUG %s code %d qlen %d\n",
329                                              dev->name, ret, q->q.qlen);
330 
331                 dev_requeue_skb(skb, q);
332                 return false;
333         }
334 
335         return true;
336 }
337 
338 /*
339  * NOTE: Called under qdisc_lock(q) with locally disabled BH.
340  *
341  * running seqcount guarantees only one CPU can process
342  * this qdisc at a time. qdisc_lock(q) serializes queue accesses for
343  * this queue.
344  *
345  *  netif_tx_lock serializes accesses to device driver.
346  *
347  *  qdisc_lock(q) and netif_tx_lock are mutually exclusive,
348  *  if one is grabbed, another must be free.
349  *
350  * Note, that this procedure can be called by a watchdog timer
351  *
352  * Returns to the caller:
353  *                              0  - queue is empty or throttled.
354  *                              >0 - queue is not empty.
355  *
356  */
357 static inline bool qdisc_restart(struct Qdisc *q, int *packets)
358 {
359         spinlock_t *root_lock = NULL;
360         struct netdev_queue *txq;
361         struct net_device *dev;
362         struct sk_buff *skb;
363         bool validate;
364 
365         /* Dequeue packet */
366         skb = dequeue_skb(q, &validate, packets);
367         if (unlikely(!skb))
368                 return false;
369 
370         if (!(q->flags & TCQ_F_NOLOCK))
371                 root_lock = qdisc_lock(q);
372 
373         dev = qdisc_dev(q);
374         txq = skb_get_tx_queue(dev, skb);
375 
376         return sch_direct_xmit(skb, q, dev, txq, root_lock, validate);
377 }
378 
379 void __qdisc_run(struct Qdisc *q)
380 {
381         int quota = dev_tx_weight;
382         int packets;
383 
384         while (qdisc_restart(q, &packets)) {
385                 /*
386                  * Ordered by possible occurrence: Postpone processing if
387                  * 1. we've exceeded packet quota
388                  * 2. another process needs the CPU;
389                  */
390                 quota -= packets;
391                 if (quota <= 0 || need_resched()) {
392                         __netif_schedule(q);
393                         break;
394                 }
395         }
396 }
397 
398 unsigned long dev_trans_start(struct net_device *dev)
399 {
400         unsigned long val, res;
401         unsigned int i;
402 
403         if (is_vlan_dev(dev))
404                 dev = vlan_dev_real_dev(dev);
405         else if (netif_is_macvlan(dev))
406                 dev = macvlan_dev_real_dev(dev);
407         res = netdev_get_tx_queue(dev, 0)->trans_start;
408         for (i = 1; i < dev->num_tx_queues; i++) {
409                 val = netdev_get_tx_queue(dev, i)->trans_start;
410                 if (val && time_after(val, res))
411                         res = val;
412         }
413 
414         return res;
415 }
416 EXPORT_SYMBOL(dev_trans_start);
417 
418 static void dev_watchdog(struct timer_list *t)
419 {
420         struct net_device *dev = from_timer(dev, t, watchdog_timer);
421 
422         netif_tx_lock(dev);
423         if (!qdisc_tx_is_noop(dev)) {
424                 if (netif_device_present(dev) &&
425                     netif_running(dev) &&
426                     netif_carrier_ok(dev)) {
427                         int some_queue_timedout = 0;
428                         unsigned int i;
429                         unsigned long trans_start;
430 
431                         for (i = 0; i < dev->num_tx_queues; i++) {
432                                 struct netdev_queue *txq;
433 
434                                 txq = netdev_get_tx_queue(dev, i);
435                                 trans_start = txq->trans_start;
436                                 if (netif_xmit_stopped(txq) &&
437                                     time_after(jiffies, (trans_start +
438                                                          dev->watchdog_timeo))) {
439                                         some_queue_timedout = 1;
440                                         txq->trans_timeout++;
441                                         break;
442                                 }
443                         }
444 
445                         if (some_queue_timedout) {
446                                 trace_net_dev_xmit_timeout(dev, i);
447                                 WARN_ONCE(1, KERN_INFO "NETDEV WATCHDOG: %s (%s): transmit queue %u timed out\n",
448                                        dev->name, netdev_drivername(dev), i);
449                                 dev->netdev_ops->ndo_tx_timeout(dev);
450                         }
451                         if (!mod_timer(&dev->watchdog_timer,
452                                        round_jiffies(jiffies +
453                                                      dev->watchdog_timeo)))
454                                 dev_hold(dev);
455                 }
456         }
457         netif_tx_unlock(dev);
458 
459         dev_put(dev);
460 }
461 
462 void __netdev_watchdog_up(struct net_device *dev)
463 {
464         if (dev->netdev_ops->ndo_tx_timeout) {
465                 if (dev->watchdog_timeo <= 0)
466                         dev->watchdog_timeo = 5*HZ;
467                 if (!mod_timer(&dev->watchdog_timer,
468                                round_jiffies(jiffies + dev->watchdog_timeo)))
469                         dev_hold(dev);
470         }
471 }
472 
473 static void dev_watchdog_up(struct net_device *dev)
474 {
475         __netdev_watchdog_up(dev);
476 }
477 
478 static void dev_watchdog_down(struct net_device *dev)
479 {
480         netif_tx_lock_bh(dev);
481         if (del_timer(&dev->watchdog_timer))
482                 dev_put(dev);
483         netif_tx_unlock_bh(dev);
484 }
485 
486 /**
487  *      netif_carrier_on - set carrier
488  *      @dev: network device
489  *
490  * Device has detected acquisition of carrier.
491  */
492 void netif_carrier_on(struct net_device *dev)
493 {
494         if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
495                 if (dev->reg_state == NETREG_UNINITIALIZED)
496                         return;
497                 atomic_inc(&dev->carrier_up_count);
498                 linkwatch_fire_event(dev);
499                 if (netif_running(dev))
500                         __netdev_watchdog_up(dev);
501         }
502 }
503 EXPORT_SYMBOL(netif_carrier_on);
504 
505 /**
506  *      netif_carrier_off - clear carrier
507  *      @dev: network device
508  *
509  * Device has detected loss of carrier.
510  */
511 void netif_carrier_off(struct net_device *dev)
512 {
513         if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
514                 if (dev->reg_state == NETREG_UNINITIALIZED)
515                         return;
516                 atomic_inc(&dev->carrier_down_count);
517                 linkwatch_fire_event(dev);
518         }
519 }
520 EXPORT_SYMBOL(netif_carrier_off);
521 
522 /* "NOOP" scheduler: the best scheduler, recommended for all interfaces
523    under all circumstances. It is difficult to invent anything faster or
524    cheaper.
525  */
526 
527 static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
528                         struct sk_buff **to_free)
529 {
530         __qdisc_drop(skb, to_free);
531         return NET_XMIT_CN;
532 }
533 
534 static struct sk_buff *noop_dequeue(struct Qdisc *qdisc)
535 {
536         return NULL;
537 }
538 
539 struct Qdisc_ops noop_qdisc_ops __read_mostly = {
540         .id             =       "noop",
541         .priv_size      =       0,
542         .enqueue        =       noop_enqueue,
543         .dequeue        =       noop_dequeue,
544         .peek           =       noop_dequeue,
545         .owner          =       THIS_MODULE,
546 };
547 
548 static struct netdev_queue noop_netdev_queue = {
549         RCU_POINTER_INITIALIZER(qdisc, &noop_qdisc),
550         .qdisc_sleeping =       &noop_qdisc,
551 };
552 
553 struct Qdisc noop_qdisc = {
554         .enqueue        =       noop_enqueue,
555         .dequeue        =       noop_dequeue,
556         .flags          =       TCQ_F_BUILTIN,
557         .ops            =       &noop_qdisc_ops,
558         .q.lock         =       __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock),
559         .dev_queue      =       &noop_netdev_queue,
560         .running        =       SEQCNT_ZERO(noop_qdisc.running),
561         .busylock       =       __SPIN_LOCK_UNLOCKED(noop_qdisc.busylock),
562         .gso_skb = {
563                 .next = (struct sk_buff *)&noop_qdisc.gso_skb,
564                 .prev = (struct sk_buff *)&noop_qdisc.gso_skb,
565                 .qlen = 0,
566                 .lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.gso_skb.lock),
567         },
568         .skb_bad_txq = {
569                 .next = (struct sk_buff *)&noop_qdisc.skb_bad_txq,
570                 .prev = (struct sk_buff *)&noop_qdisc.skb_bad_txq,
571                 .qlen = 0,
572                 .lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.skb_bad_txq.lock),
573         },
574 };
575 EXPORT_SYMBOL(noop_qdisc);
576 
577 static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt,
578                         struct netlink_ext_ack *extack)
579 {
580         /* register_qdisc() assigns a default of noop_enqueue if unset,
581          * but __dev_queue_xmit() treats noqueue only as such
582          * if this is NULL - so clear it here. */
583         qdisc->enqueue = NULL;
584         return 0;
585 }
586 
587 struct Qdisc_ops noqueue_qdisc_ops __read_mostly = {
588         .id             =       "noqueue",
589         .priv_size      =       0,
590         .init           =       noqueue_init,
591         .enqueue        =       noop_enqueue,
592         .dequeue        =       noop_dequeue,
593         .peek           =       noop_dequeue,
594         .owner          =       THIS_MODULE,
595 };
596 
597 static const u8 prio2band[TC_PRIO_MAX + 1] = {
598         1, 2, 2, 2, 1, 2, 0, 0 , 1, 1, 1, 1, 1, 1, 1, 1
599 };
600 
601 /* 3-band FIFO queue: old style, but should be a bit faster than
602    generic prio+fifo combination.
603  */
604 
605 #define PFIFO_FAST_BANDS 3
606 
607 /*
608  * Private data for a pfifo_fast scheduler containing:
609  *      - rings for priority bands
610  */
611 struct pfifo_fast_priv {
612         struct skb_array q[PFIFO_FAST_BANDS];
613 };
614 
615 static inline struct skb_array *band2list(struct pfifo_fast_priv *priv,
616                                           int band)
617 {
618         return &priv->q[band];
619 }
620 
621 static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
622                               struct sk_buff **to_free)
623 {
624         int band = prio2band[skb->priority & TC_PRIO_MAX];
625         struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
626         struct skb_array *q = band2list(priv, band);
627         unsigned int pkt_len = qdisc_pkt_len(skb);
628         int err;
629 
630         err = skb_array_produce(q, skb);
631 
632         if (unlikely(err)) {
633                 if (qdisc_is_percpu_stats(qdisc))
634                         return qdisc_drop_cpu(skb, qdisc, to_free);
635                 else
636                         return qdisc_drop(skb, qdisc, to_free);
637         }
638 
639         qdisc_update_stats_at_enqueue(qdisc, pkt_len);
640         return NET_XMIT_SUCCESS;
641 }
642 
643 static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
644 {
645         struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
646         struct sk_buff *skb = NULL;
647         int band;
648 
649         for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) {
650                 struct skb_array *q = band2list(priv, band);
651 
652                 if (__skb_array_empty(q))
653                         continue;
654 
655                 skb = __skb_array_consume(q);
656         }
657         if (likely(skb)) {
658                 qdisc_update_stats_at_dequeue(qdisc, skb);
659         } else {
660                 qdisc->empty = true;
661         }
662 
663         return skb;
664 }
665 
666 static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)
667 {
668         struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
669         struct sk_buff *skb = NULL;
670         int band;
671 
672         for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) {
673                 struct skb_array *q = band2list(priv, band);
674 
675                 skb = __skb_array_peek(q);
676         }
677 
678         return skb;
679 }
680 
681 static void pfifo_fast_reset(struct Qdisc *qdisc)
682 {
683         int i, band;
684         struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
685 
686         for (band = 0; band < PFIFO_FAST_BANDS; band++) {
687                 struct skb_array *q = band2list(priv, band);
688                 struct sk_buff *skb;
689 
690                 /* NULL ring is possible if destroy path is due to a failed
691                  * skb_array_init() in pfifo_fast_init() case.
692                  */
693                 if (!q->ring.queue)
694                         continue;
695 
696                 while ((skb = __skb_array_consume(q)) != NULL)
697                         kfree_skb(skb);
698         }
699 
700         if (qdisc_is_percpu_stats(qdisc)) {
701                 for_each_possible_cpu(i) {
702                         struct gnet_stats_queue *q;
703 
704                         q = per_cpu_ptr(qdisc->cpu_qstats, i);
705                         q->backlog = 0;
706                         q->qlen = 0;
707                 }
708         }
709 }
710 
711 static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
712 {
713         struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS };
714 
715         memcpy(&opt.priomap, prio2band, TC_PRIO_MAX + 1);
716         if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt))
717                 goto nla_put_failure;
718         return skb->len;
719 
720 nla_put_failure:
721         return -1;
722 }
723 
724 static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt,
725                            struct netlink_ext_ack *extack)
726 {
727         unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len;
728         struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
729         int prio;
730 
731         /* guard against zero length rings */
732         if (!qlen)
733                 return -EINVAL;
734 
735         for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
736                 struct skb_array *q = band2list(priv, prio);
737                 int err;
738 
739                 err = skb_array_init(q, qlen, GFP_KERNEL);
740                 if (err)
741                         return -ENOMEM;
742         }
743 
744         /* Can by-pass the queue discipline */
745         qdisc->flags |= TCQ_F_CAN_BYPASS;
746         return 0;
747 }
748 
749 static void pfifo_fast_destroy(struct Qdisc *sch)
750 {
751         struct pfifo_fast_priv *priv = qdisc_priv(sch);
752         int prio;
753 
754         for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
755                 struct skb_array *q = band2list(priv, prio);
756 
757                 /* NULL ring is possible if destroy path is due to a failed
758                  * skb_array_init() in pfifo_fast_init() case.
759                  */
760                 if (!q->ring.queue)
761                         continue;
762                 /* Destroy ring but no need to kfree_skb because a call to
763                  * pfifo_fast_reset() has already done that work.
764                  */
765                 ptr_ring_cleanup(&q->ring, NULL);
766         }
767 }
768 
769 static int pfifo_fast_change_tx_queue_len(struct Qdisc *sch,
770                                           unsigned int new_len)
771 {
772         struct pfifo_fast_priv *priv = qdisc_priv(sch);
773         struct skb_array *bands[PFIFO_FAST_BANDS];
774         int prio;
775 
776         for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
777                 struct skb_array *q = band2list(priv, prio);
778 
779                 bands[prio] = q;
780         }
781 
782         return skb_array_resize_multiple(bands, PFIFO_FAST_BANDS, new_len,
783                                          GFP_KERNEL);
784 }
785 
786 struct Qdisc_ops pfifo_fast_ops __read_mostly = {
787         .id             =       "pfifo_fast",
788         .priv_size      =       sizeof(struct pfifo_fast_priv),
789         .enqueue        =       pfifo_fast_enqueue,
790         .dequeue        =       pfifo_fast_dequeue,
791         .peek           =       pfifo_fast_peek,
792         .init           =       pfifo_fast_init,
793         .destroy        =       pfifo_fast_destroy,
794         .reset          =       pfifo_fast_reset,
795         .dump           =       pfifo_fast_dump,
796         .change_tx_queue_len =  pfifo_fast_change_tx_queue_len,
797         .owner          =       THIS_MODULE,
798         .static_flags   =       TCQ_F_NOLOCK | TCQ_F_CPUSTATS,
799 };
800 EXPORT_SYMBOL(pfifo_fast_ops);
801 
802 static struct lock_class_key qdisc_tx_busylock;
803 static struct lock_class_key qdisc_running_key;
804 
805 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
806                           const struct Qdisc_ops *ops,
807                           struct netlink_ext_ack *extack)
808 {
809         void *p;
810         struct Qdisc *sch;
811         unsigned int size = QDISC_ALIGN(sizeof(*sch)) + ops->priv_size;
812         int err = -ENOBUFS;
813         struct net_device *dev;
814 
815         if (!dev_queue) {
816                 NL_SET_ERR_MSG(extack, "No device queue given");
817                 err = -EINVAL;
818                 goto errout;
819         }
820 
821         dev = dev_queue->dev;
822         p = kzalloc_node(size, GFP_KERNEL,
823                          netdev_queue_numa_node_read(dev_queue));
824 
825         if (!p)
826                 goto errout;
827         sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p);
828         /* if we got non aligned memory, ask more and do alignment ourself */
829         if (sch != p) {
830                 kfree(p);
831                 p = kzalloc_node(size + QDISC_ALIGNTO - 1, GFP_KERNEL,
832                                  netdev_queue_numa_node_read(dev_queue));
833                 if (!p)
834                         goto errout;
835                 sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p);
836                 sch->padded = (char *) sch - (char *) p;
837         }
838         __skb_queue_head_init(&sch->gso_skb);
839         __skb_queue_head_init(&sch->skb_bad_txq);
840         qdisc_skb_head_init(&sch->q);
841         spin_lock_init(&sch->q.lock);
842 
843         if (ops->static_flags & TCQ_F_CPUSTATS) {
844                 sch->cpu_bstats =
845                         netdev_alloc_pcpu_stats(struct gnet_stats_basic_cpu);
846                 if (!sch->cpu_bstats)
847                         goto errout1;
848 
849                 sch->cpu_qstats = alloc_percpu(struct gnet_stats_queue);
850                 if (!sch->cpu_qstats) {
851                         free_percpu(sch->cpu_bstats);
852                         goto errout1;
853                 }
854         }
855 
856         spin_lock_init(&sch->busylock);
857         lockdep_set_class(&sch->busylock,
858                           dev->qdisc_tx_busylock ?: &qdisc_tx_busylock);
859 
860         /* seqlock has the same scope of busylock, for NOLOCK qdisc */
861         spin_lock_init(&sch->seqlock);
862         lockdep_set_class(&sch->busylock,
863                           dev->qdisc_tx_busylock ?: &qdisc_tx_busylock);
864 
865         seqcount_init(&sch->running);
866         lockdep_set_class(&sch->running,
867                           dev->qdisc_running_key ?: &qdisc_running_key);
868 
869         sch->ops = ops;
870         sch->flags = ops->static_flags;
871         sch->enqueue = ops->enqueue;
872         sch->dequeue = ops->dequeue;
873         sch->dev_queue = dev_queue;
874         sch->empty = true;
875         dev_hold(dev);
876         refcount_set(&sch->refcnt, 1);
877 
878         return sch;
879 errout1:
880         kfree(p);
881 errout:
882         return ERR_PTR(err);
883 }
884 
885 struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
886                                 const struct Qdisc_ops *ops,
887                                 unsigned int parentid,
888                                 struct netlink_ext_ack *extack)
889 {
890         struct Qdisc *sch;
891 
892         if (!try_module_get(ops->owner)) {
893                 NL_SET_ERR_MSG(extack, "Failed to increase module reference counter");
894                 return NULL;
895         }
896 
897         sch = qdisc_alloc(dev_queue, ops, extack);
898         if (IS_ERR(sch)) {
899                 module_put(ops->owner);
900                 return NULL;
901         }
902         sch->parent = parentid;
903 
904         if (!ops->init || ops->init(sch, NULL, extack) == 0)
905                 return sch;
906 
907         qdisc_put(sch);
908         return NULL;
909 }
910 EXPORT_SYMBOL(qdisc_create_dflt);
911 
912 /* Under qdisc_lock(qdisc) and BH! */
913 
914 void qdisc_reset(struct Qdisc *qdisc)
915 {
916         const struct Qdisc_ops *ops = qdisc->ops;
917         struct sk_buff *skb, *tmp;
918 
919         if (ops->reset)
920                 ops->reset(qdisc);
921 
922         skb_queue_walk_safe(&qdisc->gso_skb, skb, tmp) {
923                 __skb_unlink(skb, &qdisc->gso_skb);
924                 kfree_skb_list(skb);
925         }
926 
927         skb_queue_walk_safe(&qdisc->skb_bad_txq, skb, tmp) {
928                 __skb_unlink(skb, &qdisc->skb_bad_txq);
929                 kfree_skb_list(skb);
930         }
931 
932         qdisc->q.qlen = 0;
933         qdisc->qstats.backlog = 0;
934 }
935 EXPORT_SYMBOL(qdisc_reset);
936 
937 void qdisc_free(struct Qdisc *qdisc)
938 {
939         if (qdisc_is_percpu_stats(qdisc)) {
940                 free_percpu(qdisc->cpu_bstats);
941                 free_percpu(qdisc->cpu_qstats);
942         }
943 
944         kfree((char *) qdisc - qdisc->padded);
945 }
946 
947 static void qdisc_free_cb(struct rcu_head *head)
948 {
949         struct Qdisc *q = container_of(head, struct Qdisc, rcu);
950 
951         qdisc_free(q);
952 }
953 
954 static void qdisc_destroy(struct Qdisc *qdisc)
955 {
956         const struct Qdisc_ops  *ops = qdisc->ops;
957         struct sk_buff *skb, *tmp;
958 
959 #ifdef CONFIG_NET_SCHED
960         qdisc_hash_del(qdisc);
961 
962         qdisc_put_stab(rtnl_dereference(qdisc->stab));
963 #endif
964         gen_kill_estimator(&qdisc->rate_est);
965         if (ops->reset)
966                 ops->reset(qdisc);
967         if (ops->destroy)
968                 ops->destroy(qdisc);
969 
970         module_put(ops->owner);
971         dev_put(qdisc_dev(qdisc));
972 
973         skb_queue_walk_safe(&qdisc->gso_skb, skb, tmp) {
974                 __skb_unlink(skb, &qdisc->gso_skb);
975                 kfree_skb_list(skb);
976         }
977 
978         skb_queue_walk_safe(&qdisc->skb_bad_txq, skb, tmp) {
979                 __skb_unlink(skb, &qdisc->skb_bad_txq);
980                 kfree_skb_list(skb);
981         }
982 
983         call_rcu(&qdisc->rcu, qdisc_free_cb);
984 }
985 
986 void qdisc_put(struct Qdisc *qdisc)
987 {
988         if (!qdisc)
989                 return;
990 
991         if (qdisc->flags & TCQ_F_BUILTIN ||
992             !refcount_dec_and_test(&qdisc->refcnt))
993                 return;
994 
995         qdisc_destroy(qdisc);
996 }
997 EXPORT_SYMBOL(qdisc_put);
998 
999 /* Version of qdisc_put() that is called with rtnl mutex unlocked.
1000  * Intended to be used as optimization, this function only takes rtnl lock if
1001  * qdisc reference counter reached zero.
1002  */
1003 
1004 void qdisc_put_unlocked(struct Qdisc *qdisc)
1005 {
1006         if (qdisc->flags & TCQ_F_BUILTIN ||
1007             !refcount_dec_and_rtnl_lock(&qdisc->refcnt))
1008                 return;
1009 
1010         qdisc_destroy(qdisc);
1011         rtnl_unlock();
1012 }
1013 EXPORT_SYMBOL(qdisc_put_unlocked);
1014 
1015 /* Attach toplevel qdisc to device queue. */
1016 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
1017                               struct Qdisc *qdisc)
1018 {
1019         struct Qdisc *oqdisc = dev_queue->qdisc_sleeping;
1020         spinlock_t *root_lock;
1021 
1022         root_lock = qdisc_lock(oqdisc);
1023         spin_lock_bh(root_lock);
1024 
1025         /* ... and graft new one */
1026         if (qdisc == NULL)
1027                 qdisc = &noop_qdisc;
1028         dev_queue->qdisc_sleeping = qdisc;
1029         rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);
1030 
1031         spin_unlock_bh(root_lock);
1032 
1033         return oqdisc;
1034 }
1035 EXPORT_SYMBOL(dev_graft_qdisc);
1036 
1037 static void attach_one_default_qdisc(struct net_device *dev,
1038                                      struct netdev_queue *dev_queue,
1039                                      void *_unused)
1040 {
1041         struct Qdisc *qdisc;
1042         const struct Qdisc_ops *ops = default_qdisc_ops;
1043 
1044         if (dev->priv_flags & IFF_NO_QUEUE)
1045                 ops = &noqueue_qdisc_ops;
1046 
1047         qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT, NULL);
1048         if (!qdisc) {
1049                 netdev_info(dev, "activation failed\n");
1050                 return;
1051         }
1052         if (!netif_is_multiqueue(dev))
1053                 qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
1054         dev_queue->qdisc_sleeping = qdisc;
1055 }
1056 
1057 static void attach_default_qdiscs(struct net_device *dev)
1058 {
1059         struct netdev_queue *txq;
1060         struct Qdisc *qdisc;
1061 
1062         txq = netdev_get_tx_queue(dev, 0);
1063 
1064         if (!netif_is_multiqueue(dev) ||
1065             dev->priv_flags & IFF_NO_QUEUE) {
1066                 netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
1067                 dev->qdisc = txq->qdisc_sleeping;
1068                 qdisc_refcount_inc(dev->qdisc);
1069         } else {
1070                 qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL);
1071                 if (qdisc) {
1072                         dev->qdisc = qdisc;
1073                         qdisc->ops->attach(qdisc);
1074                 }
1075         }
1076 #ifdef CONFIG_NET_SCHED
1077         if (dev->qdisc != &noop_qdisc)
1078                 qdisc_hash_add(dev->qdisc, false);
1079 #endif
1080 }
1081 
1082 static void transition_one_qdisc(struct net_device *dev,
1083                                  struct netdev_queue *dev_queue,
1084                                  void *_need_watchdog)
1085 {
1086         struct Qdisc *new_qdisc = dev_queue->qdisc_sleeping;
1087         int *need_watchdog_p = _need_watchdog;
1088 
1089         if (!(new_qdisc->flags & TCQ_F_BUILTIN))
1090                 clear_bit(__QDISC_STATE_DEACTIVATED, &new_qdisc->state);
1091 
1092         rcu_assign_pointer(dev_queue->qdisc, new_qdisc);
1093         if (need_watchdog_p) {
1094                 dev_queue->trans_start = 0;
1095                 *need_watchdog_p = 1;
1096         }
1097 }
1098 
1099 void dev_activate(struct net_device *dev)
1100 {
1101         int need_watchdog;
1102 
1103         /* No queueing discipline is attached to device;
1104          * create default one for devices, which need queueing
1105          * and noqueue_qdisc for virtual interfaces
1106          */
1107 
1108         if (dev->qdisc == &noop_qdisc)
1109                 attach_default_qdiscs(dev);
1110 
1111         if (!netif_carrier_ok(dev))
1112                 /* Delay activation until next carrier-on event */
1113                 return;
1114 
1115         need_watchdog = 0;
1116         netdev_for_each_tx_queue(dev, transition_one_qdisc, &need_watchdog);
1117         if (dev_ingress_queue(dev))
1118                 transition_one_qdisc(dev, dev_ingress_queue(dev), NULL);
1119 
1120         if (need_watchdog) {
1121                 netif_trans_update(dev);
1122                 dev_watchdog_up(dev);
1123         }
1124 }
1125 EXPORT_SYMBOL(dev_activate);
1126 
1127 static void dev_deactivate_queue(struct net_device *dev,
1128                                  struct netdev_queue *dev_queue,
1129                                  void *_qdisc_default)
1130 {
1131         struct Qdisc *qdisc_default = _qdisc_default;
1132         struct Qdisc *qdisc;
1133 
1134         qdisc = rtnl_dereference(dev_queue->qdisc);
1135         if (qdisc) {
1136                 bool nolock = qdisc->flags & TCQ_F_NOLOCK;
1137 
1138                 if (nolock)
1139                         spin_lock_bh(&qdisc->seqlock);
1140                 spin_lock_bh(qdisc_lock(qdisc));
1141 
1142                 if (!(qdisc->flags & TCQ_F_BUILTIN))
1143                         set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state);
1144 
1145                 rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
1146                 qdisc_reset(qdisc);
1147 
1148                 spin_unlock_bh(qdisc_lock(qdisc));
1149                 if (nolock)
1150                         spin_unlock_bh(&qdisc->seqlock);
1151         }
1152 }
1153 
1154 static bool some_qdisc_is_busy(struct net_device *dev)
1155 {
1156         unsigned int i;
1157 
1158         for (i = 0; i < dev->num_tx_queues; i++) {
1159                 struct netdev_queue *dev_queue;
1160                 spinlock_t *root_lock;
1161                 struct Qdisc *q;
1162                 int val;
1163 
1164                 dev_queue = netdev_get_tx_queue(dev, i);
1165                 q = dev_queue->qdisc_sleeping;
1166 
1167                 root_lock = qdisc_lock(q);
1168                 spin_lock_bh(root_lock);
1169 
1170                 val = (qdisc_is_running(q) ||
1171                        test_bit(__QDISC_STATE_SCHED, &q->state));
1172 
1173                 spin_unlock_bh(root_lock);
1174 
1175                 if (val)
1176                         return true;
1177         }
1178         return false;
1179 }
1180 
1181 static void dev_qdisc_reset(struct net_device *dev,
1182                             struct netdev_queue *dev_queue,
1183                             void *none)
1184 {
1185         struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
1186 
1187         if (qdisc)
1188                 qdisc_reset(qdisc);
1189 }
1190 
1191 /**
1192  *      dev_deactivate_many - deactivate transmissions on several devices
1193  *      @head: list of devices to deactivate
1194  *
1195  *      This function returns only when all outstanding transmissions
1196  *      have completed, unless all devices are in dismantle phase.
1197  */
1198 void dev_deactivate_many(struct list_head *head)
1199 {
1200         struct net_device *dev;
1201 
1202         list_for_each_entry(dev, head, close_list) {
1203                 netdev_for_each_tx_queue(dev, dev_deactivate_queue,
1204                                          &noop_qdisc);
1205                 if (dev_ingress_queue(dev))
1206                         dev_deactivate_queue(dev, dev_ingress_queue(dev),
1207                                              &noop_qdisc);
1208 
1209                 dev_watchdog_down(dev);
1210         }
1211 
1212         /* Wait for outstanding qdisc-less dev_queue_xmit calls.
1213          * This is avoided if all devices are in dismantle phase :
1214          * Caller will call synchronize_net() for us
1215          */
1216         synchronize_net();
1217 
1218         /* Wait for outstanding qdisc_run calls. */
1219         list_for_each_entry(dev, head, close_list) {
1220                 while (some_qdisc_is_busy(dev))
1221                         yield();
1222                 /* The new qdisc is assigned at this point so we can safely
1223                  * unwind stale skb lists and qdisc statistics
1224                  */
1225                 netdev_for_each_tx_queue(dev, dev_qdisc_reset, NULL);
1226                 if (dev_ingress_queue(dev))
1227                         dev_qdisc_reset(dev, dev_ingress_queue(dev), NULL);
1228         }
1229 }
1230 
1231 void dev_deactivate(struct net_device *dev)
1232 {
1233         LIST_HEAD(single);
1234 
1235         list_add(&dev->close_list, &single);
1236         dev_deactivate_many(&single);
1237         list_del(&single);
1238 }
1239 EXPORT_SYMBOL(dev_deactivate);
1240 
1241 static int qdisc_change_tx_queue_len(struct net_device *dev,
1242                                      struct netdev_queue *dev_queue)
1243 {
1244         struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
1245         const struct Qdisc_ops *ops = qdisc->ops;
1246 
1247         if (ops->change_tx_queue_len)
1248                 return ops->change_tx_queue_len(qdisc, dev->tx_queue_len);
1249         return 0;
1250 }
1251 
1252 int dev_qdisc_change_tx_queue_len(struct net_device *dev)
1253 {
1254         bool up = dev->flags & IFF_UP;
1255         unsigned int i;
1256         int ret = 0;
1257 
1258         if (up)
1259                 dev_deactivate(dev);
1260 
1261         for (i = 0; i < dev->num_tx_queues; i++) {
1262                 ret = qdisc_change_tx_queue_len(dev, &dev->_tx[i]);
1263 
1264                 /* TODO: revert changes on a partial failure */
1265                 if (ret)
1266                         break;
1267         }
1268 
1269         if (up)
1270                 dev_activate(dev);
1271         return ret;
1272 }
1273 
1274 static void dev_init_scheduler_queue(struct net_device *dev,
1275                                      struct netdev_queue *dev_queue,
1276                                      void *_qdisc)
1277 {
1278         struct Qdisc *qdisc = _qdisc;
1279 
1280         rcu_assign_pointer(dev_queue->qdisc, qdisc);
1281         dev_queue->qdisc_sleeping = qdisc;
1282 }
1283 
1284 void dev_init_scheduler(struct net_device *dev)
1285 {
1286         dev->qdisc = &noop_qdisc;
1287         netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc);
1288         if (dev_ingress_queue(dev))
1289                 dev_init_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
1290 
1291         timer_setup(&dev->watchdog_timer, dev_watchdog, 0);
1292 }
1293 
1294 static void shutdown_scheduler_queue(struct net_device *dev,
1295                                      struct netdev_queue *dev_queue,
1296                                      void *_qdisc_default)
1297 {
1298         struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
1299         struct Qdisc *qdisc_default = _qdisc_default;
1300 
1301         if (qdisc) {
1302                 rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
1303                 dev_queue->qdisc_sleeping = qdisc_default;
1304 
1305                 qdisc_put(qdisc);
1306         }
1307 }
1308 
1309 void dev_shutdown(struct net_device *dev)
1310 {
1311         netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc);
1312         if (dev_ingress_queue(dev))
1313                 shutdown_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
1314         qdisc_put(dev->qdisc);
1315         dev->qdisc = &noop_qdisc;
1316 
1317         WARN_ON(timer_pending(&dev->watchdog_timer));
1318 }
1319 
1320 void psched_ratecfg_precompute(struct psched_ratecfg *r,
1321                                const struct tc_ratespec *conf,
1322                                u64 rate64)
1323 {
1324         memset(r, 0, sizeof(*r));
1325         r->overhead = conf->overhead;
1326         r->rate_bytes_ps = max_t(u64, conf->rate, rate64);
1327         r->linklayer = (conf->linklayer & TC_LINKLAYER_MASK);
1328         r->mult = 1;
1329         /*
1330          * The deal here is to replace a divide by a reciprocal one
1331          * in fast path (a reciprocal divide is a multiply and a shift)
1332          *
1333          * Normal formula would be :
1334          *  time_in_ns = (NSEC_PER_SEC * len) / rate_bps
1335          *
1336          * We compute mult/shift to use instead :
1337          *  time_in_ns = (len * mult) >> shift;
1338          *
1339          * We try to get the highest possible mult value for accuracy,
1340          * but have to make sure no overflows will ever happen.
1341          */
1342         if (r->rate_bytes_ps > 0) {
1343                 u64 factor = NSEC_PER_SEC;
1344 
1345                 for (;;) {
1346                         r->mult = div64_u64(factor, r->rate_bytes_ps);
1347                         if (r->mult & (1U << 31) || factor & (1ULL << 63))
1348                                 break;
1349                         factor <<= 1;
1350                         r->shift++;
1351                 }
1352         }
1353 }
1354 EXPORT_SYMBOL(psched_ratecfg_precompute);
1355 
1356 static void mini_qdisc_rcu_func(struct rcu_head *head)
1357 {
1358 }
1359 
1360 void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp,
1361                           struct tcf_proto *tp_head)
1362 {
1363         /* Protected with chain0->filter_chain_lock.
1364          * Can't access chain directly because tp_head can be NULL.
1365          */
1366         struct mini_Qdisc *miniq_old =
1367                 rcu_dereference_protected(*miniqp->p_miniq, 1);
1368         struct mini_Qdisc *miniq;
1369 
1370         if (!tp_head) {
1371                 RCU_INIT_POINTER(*miniqp->p_miniq, NULL);
1372                 /* Wait for flying RCU callback before it is freed. */
1373                 rcu_barrier();
1374                 return;
1375         }
1376 
1377         miniq = !miniq_old || miniq_old == &miniqp->miniq2 ?
1378                 &miniqp->miniq1 : &miniqp->miniq2;
1379 
1380         /* We need to make sure that readers won't see the miniq
1381          * we are about to modify. So wait until previous call_rcu callback
1382          * is done.
1383          */
1384         rcu_barrier();
1385         miniq->filter_list = tp_head;
1386         rcu_assign_pointer(*miniqp->p_miniq, miniq);
1387 
1388         if (miniq_old)
1389                 /* This is counterpart of the rcu barriers above. We need to
1390                  * block potential new user of miniq_old until all readers
1391                  * are not seeing it.
1392                  */
1393                 call_rcu(&miniq_old->rcu, mini_qdisc_rcu_func);
1394 }
1395 EXPORT_SYMBOL(mini_qdisc_pair_swap);
1396 
1397 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,
1398                           struct mini_Qdisc __rcu **p_miniq)
1399 {
1400         miniqp->miniq1.cpu_bstats = qdisc->cpu_bstats;
1401         miniqp->miniq1.cpu_qstats = qdisc->cpu_qstats;
1402         miniqp->miniq2.cpu_bstats = qdisc->cpu_bstats;
1403         miniqp->miniq2.cpu_qstats = qdisc->cpu_qstats;
1404         miniqp->p_miniq = p_miniq;
1405 }
1406 EXPORT_SYMBOL(mini_qdisc_pair_init);
1407 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp