~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/softirq.c

Version: ~ [ linux-5.4-rc7 ] ~ [ linux-5.3.10 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.83 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.153 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.200 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.200 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.76 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *      linux/kernel/softirq.c
  3  *
  4  *      Copyright (C) 1992 Linus Torvalds
  5  *
  6  *      Distribute under GPLv2.
  7  *
  8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
  9  */
 10 
 11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 12 
 13 #include <linux/export.h>
 14 #include <linux/kernel_stat.h>
 15 #include <linux/interrupt.h>
 16 #include <linux/init.h>
 17 #include <linux/mm.h>
 18 #include <linux/notifier.h>
 19 #include <linux/percpu.h>
 20 #include <linux/cpu.h>
 21 #include <linux/freezer.h>
 22 #include <linux/kthread.h>
 23 #include <linux/rcupdate.h>
 24 #include <linux/ftrace.h>
 25 #include <linux/smp.h>
 26 #include <linux/smpboot.h>
 27 #include <linux/tick.h>
 28 #include <linux/irq.h>
 29 
 30 #define CREATE_TRACE_POINTS
 31 #include <trace/events/irq.h>
 32 
 33 /*
 34    - No shared variables, all the data are CPU local.
 35    - If a softirq needs serialization, let it serialize itself
 36      by its own spinlocks.
 37    - Even if softirq is serialized, only local cpu is marked for
 38      execution. Hence, we get something sort of weak cpu binding.
 39      Though it is still not clear, will it result in better locality
 40      or will not.
 41 
 42    Examples:
 43    - NET RX softirq. It is multithreaded and does not require
 44      any global serialization.
 45    - NET TX softirq. It kicks software netdevice queues, hence
 46      it is logically serialized per device, but this serialization
 47      is invisible to common code.
 48    - Tasklets: serialized wrt itself.
 49  */
 50 
 51 #ifndef __ARCH_IRQ_STAT
 52 DEFINE_PER_CPU_ALIGNED(irq_cpustat_t, irq_stat);
 53 EXPORT_PER_CPU_SYMBOL(irq_stat);
 54 #endif
 55 
 56 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
 57 
 58 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
 59 
 60 const char * const softirq_to_name[NR_SOFTIRQS] = {
 61         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL",
 62         "TASKLET", "SCHED", "HRTIMER", "RCU"
 63 };
 64 
 65 /*
 66  * we cannot loop indefinitely here to avoid userspace starvation,
 67  * but we also don't want to introduce a worst case 1/HZ latency
 68  * to the pending events, so lets the scheduler to balance
 69  * the softirq load for us.
 70  */
 71 static void wakeup_softirqd(void)
 72 {
 73         /* Interrupts are disabled: no need to stop preemption */
 74         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
 75 
 76         if (tsk && tsk->state != TASK_RUNNING)
 77                 wake_up_process(tsk);
 78 }
 79 
 80 /*
 81  * If ksoftirqd is scheduled, we do not want to process pending softirqs
 82  * right now. Let ksoftirqd handle this at its own rate, to get fairness,
 83  * unless we're doing some of the synchronous softirqs.
 84  */
 85 #define SOFTIRQ_NOW_MASK ((1 << HI_SOFTIRQ) | (1 << TASKLET_SOFTIRQ))
 86 static bool ksoftirqd_running(unsigned long pending)
 87 {
 88         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
 89 
 90         if (pending & SOFTIRQ_NOW_MASK)
 91                 return false;
 92         return tsk && (tsk->state == TASK_RUNNING) &&
 93                 !__kthread_should_park(tsk);
 94 }
 95 
 96 /*
 97  * preempt_count and SOFTIRQ_OFFSET usage:
 98  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
 99  *   softirq processing.
100  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
101  *   on local_bh_disable or local_bh_enable.
102  * This lets us distinguish between whether we are currently processing
103  * softirq and whether we just have bh disabled.
104  */
105 
106 /*
107  * This one is for softirq.c-internal use,
108  * where hardirqs are disabled legitimately:
109  */
110 #ifdef CONFIG_TRACE_IRQFLAGS
111 void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
112 {
113         unsigned long flags;
114 
115         WARN_ON_ONCE(in_irq());
116 
117         raw_local_irq_save(flags);
118         /*
119          * The preempt tracer hooks into preempt_count_add and will break
120          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
121          * is set and before current->softirq_enabled is cleared.
122          * We must manually increment preempt_count here and manually
123          * call the trace_preempt_off later.
124          */
125         __preempt_count_add(cnt);
126         /*
127          * Were softirqs turned off above:
128          */
129         if (softirq_count() == (cnt & SOFTIRQ_MASK))
130                 trace_softirqs_off(ip);
131         raw_local_irq_restore(flags);
132 
133         if (preempt_count() == cnt) {
134 #ifdef CONFIG_DEBUG_PREEMPT
135                 current->preempt_disable_ip = get_lock_parent_ip();
136 #endif
137                 trace_preempt_off(CALLER_ADDR0, get_lock_parent_ip());
138         }
139 }
140 EXPORT_SYMBOL(__local_bh_disable_ip);
141 #endif /* CONFIG_TRACE_IRQFLAGS */
142 
143 static void __local_bh_enable(unsigned int cnt)
144 {
145         lockdep_assert_irqs_disabled();
146 
147         if (preempt_count() == cnt)
148                 trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
149 
150         if (softirq_count() == (cnt & SOFTIRQ_MASK))
151                 trace_softirqs_on(_RET_IP_);
152 
153         __preempt_count_sub(cnt);
154 }
155 
156 /*
157  * Special-case - softirqs can safely be enabled by __do_softirq(),
158  * without processing still-pending softirqs:
159  */
160 void _local_bh_enable(void)
161 {
162         WARN_ON_ONCE(in_irq());
163         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
164 }
165 EXPORT_SYMBOL(_local_bh_enable);
166 
167 void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
168 {
169         WARN_ON_ONCE(in_irq());
170         lockdep_assert_irqs_enabled();
171 #ifdef CONFIG_TRACE_IRQFLAGS
172         local_irq_disable();
173 #endif
174         /*
175          * Are softirqs going to be turned on now:
176          */
177         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
178                 trace_softirqs_on(ip);
179         /*
180          * Keep preemption disabled until we are done with
181          * softirq processing:
182          */
183         preempt_count_sub(cnt - 1);
184 
185         if (unlikely(!in_interrupt() && local_softirq_pending())) {
186                 /*
187                  * Run softirq if any pending. And do it in its own stack
188                  * as we may be calling this deep in a task call stack already.
189                  */
190                 do_softirq();
191         }
192 
193         preempt_count_dec();
194 #ifdef CONFIG_TRACE_IRQFLAGS
195         local_irq_enable();
196 #endif
197         preempt_check_resched();
198 }
199 EXPORT_SYMBOL(__local_bh_enable_ip);
200 
201 /*
202  * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
203  * but break the loop if need_resched() is set or after 2 ms.
204  * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
205  * certain cases, such as stop_machine(), jiffies may cease to
206  * increment and so we need the MAX_SOFTIRQ_RESTART limit as
207  * well to make sure we eventually return from this method.
208  *
209  * These limits have been established via experimentation.
210  * The two things to balance is latency against fairness -
211  * we want to handle softirqs as soon as possible, but they
212  * should not be able to lock up the box.
213  */
214 #define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
215 #define MAX_SOFTIRQ_RESTART 10
216 
217 #ifdef CONFIG_TRACE_IRQFLAGS
218 /*
219  * When we run softirqs from irq_exit() and thus on the hardirq stack we need
220  * to keep the lockdep irq context tracking as tight as possible in order to
221  * not miss-qualify lock contexts and miss possible deadlocks.
222  */
223 
224 static inline bool lockdep_softirq_start(void)
225 {
226         bool in_hardirq = false;
227 
228         if (trace_hardirq_context(current)) {
229                 in_hardirq = true;
230                 trace_hardirq_exit();
231         }
232 
233         lockdep_softirq_enter();
234 
235         return in_hardirq;
236 }
237 
238 static inline void lockdep_softirq_end(bool in_hardirq)
239 {
240         lockdep_softirq_exit();
241 
242         if (in_hardirq)
243                 trace_hardirq_enter();
244 }
245 #else
246 static inline bool lockdep_softirq_start(void) { return false; }
247 static inline void lockdep_softirq_end(bool in_hardirq) { }
248 #endif
249 
250 asmlinkage __visible void __softirq_entry __do_softirq(void)
251 {
252         unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
253         unsigned long old_flags = current->flags;
254         int max_restart = MAX_SOFTIRQ_RESTART;
255         struct softirq_action *h;
256         bool in_hardirq;
257         __u32 pending;
258         int softirq_bit;
259 
260         /*
261          * Mask out PF_MEMALLOC as the current task context is borrowed for the
262          * softirq. A softirq handled, such as network RX, might set PF_MEMALLOC
263          * again if the socket is related to swapping.
264          */
265         current->flags &= ~PF_MEMALLOC;
266 
267         pending = local_softirq_pending();
268         account_irq_enter_time(current);
269 
270         __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
271         in_hardirq = lockdep_softirq_start();
272 
273 restart:
274         /* Reset the pending bitmask before enabling irqs */
275         set_softirq_pending(0);
276 
277         local_irq_enable();
278 
279         h = softirq_vec;
280 
281         while ((softirq_bit = ffs(pending))) {
282                 unsigned int vec_nr;
283                 int prev_count;
284 
285                 h += softirq_bit - 1;
286 
287                 vec_nr = h - softirq_vec;
288                 prev_count = preempt_count();
289 
290                 kstat_incr_softirqs_this_cpu(vec_nr);
291 
292                 trace_softirq_entry(vec_nr);
293                 h->action(h);
294                 trace_softirq_exit(vec_nr);
295                 if (unlikely(prev_count != preempt_count())) {
296                         pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
297                                vec_nr, softirq_to_name[vec_nr], h->action,
298                                prev_count, preempt_count());
299                         preempt_count_set(prev_count);
300                 }
301                 h++;
302                 pending >>= softirq_bit;
303         }
304 
305         if (__this_cpu_read(ksoftirqd) == current)
306                 rcu_softirq_qs();
307         local_irq_disable();
308 
309         pending = local_softirq_pending();
310         if (pending) {
311                 if (time_before(jiffies, end) && !need_resched() &&
312                     --max_restart)
313                         goto restart;
314 
315                 wakeup_softirqd();
316         }
317 
318         lockdep_softirq_end(in_hardirq);
319         account_irq_exit_time(current);
320         __local_bh_enable(SOFTIRQ_OFFSET);
321         WARN_ON_ONCE(in_interrupt());
322         current_restore_flags(old_flags, PF_MEMALLOC);
323 }
324 
325 asmlinkage __visible void do_softirq(void)
326 {
327         __u32 pending;
328         unsigned long flags;
329 
330         if (in_interrupt())
331                 return;
332 
333         local_irq_save(flags);
334 
335         pending = local_softirq_pending();
336 
337         if (pending && !ksoftirqd_running(pending))
338                 do_softirq_own_stack();
339 
340         local_irq_restore(flags);
341 }
342 
343 /*
344  * Enter an interrupt context.
345  */
346 void irq_enter(void)
347 {
348         rcu_irq_enter();
349         if (is_idle_task(current) && !in_interrupt()) {
350                 /*
351                  * Prevent raise_softirq from needlessly waking up ksoftirqd
352                  * here, as softirq will be serviced on return from interrupt.
353                  */
354                 local_bh_disable();
355                 tick_irq_enter();
356                 _local_bh_enable();
357         }
358 
359         __irq_enter();
360 }
361 
362 static inline void invoke_softirq(void)
363 {
364         if (ksoftirqd_running(local_softirq_pending()))
365                 return;
366 
367         if (!force_irqthreads) {
368 #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
369                 /*
370                  * We can safely execute softirq on the current stack if
371                  * it is the irq stack, because it should be near empty
372                  * at this stage.
373                  */
374                 __do_softirq();
375 #else
376                 /*
377                  * Otherwise, irq_exit() is called on the task stack that can
378                  * be potentially deep already. So call softirq in its own stack
379                  * to prevent from any overrun.
380                  */
381                 do_softirq_own_stack();
382 #endif
383         } else {
384                 wakeup_softirqd();
385         }
386 }
387 
388 static inline void tick_irq_exit(void)
389 {
390 #ifdef CONFIG_NO_HZ_COMMON
391         int cpu = smp_processor_id();
392 
393         /* Make sure that timer wheel updates are propagated */
394         if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
395                 if (!in_irq())
396                         tick_nohz_irq_exit();
397         }
398 #endif
399 }
400 
401 /*
402  * Exit an interrupt context. Process softirqs if needed and possible:
403  */
404 void irq_exit(void)
405 {
406 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
407         local_irq_disable();
408 #else
409         lockdep_assert_irqs_disabled();
410 #endif
411         account_irq_exit_time(current);
412         preempt_count_sub(HARDIRQ_OFFSET);
413         if (!in_interrupt() && local_softirq_pending())
414                 invoke_softirq();
415 
416         tick_irq_exit();
417         rcu_irq_exit();
418         trace_hardirq_exit(); /* must be last! */
419 }
420 
421 /*
422  * This function must run with irqs disabled!
423  */
424 inline void raise_softirq_irqoff(unsigned int nr)
425 {
426         __raise_softirq_irqoff(nr);
427 
428         /*
429          * If we're in an interrupt or softirq, we're done
430          * (this also catches softirq-disabled code). We will
431          * actually run the softirq once we return from
432          * the irq or softirq.
433          *
434          * Otherwise we wake up ksoftirqd to make sure we
435          * schedule the softirq soon.
436          */
437         if (!in_interrupt())
438                 wakeup_softirqd();
439 }
440 
441 void raise_softirq(unsigned int nr)
442 {
443         unsigned long flags;
444 
445         local_irq_save(flags);
446         raise_softirq_irqoff(nr);
447         local_irq_restore(flags);
448 }
449 
450 void __raise_softirq_irqoff(unsigned int nr)
451 {
452         trace_softirq_raise(nr);
453         or_softirq_pending(1UL << nr);
454 }
455 
456 void open_softirq(int nr, void (*action)(struct softirq_action *))
457 {
458         softirq_vec[nr].action = action;
459 }
460 
461 /*
462  * Tasklets
463  */
464 struct tasklet_head {
465         struct tasklet_struct *head;
466         struct tasklet_struct **tail;
467 };
468 
469 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
470 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
471 
472 static void __tasklet_schedule_common(struct tasklet_struct *t,
473                                       struct tasklet_head __percpu *headp,
474                                       unsigned int softirq_nr)
475 {
476         struct tasklet_head *head;
477         unsigned long flags;
478 
479         local_irq_save(flags);
480         head = this_cpu_ptr(headp);
481         t->next = NULL;
482         *head->tail = t;
483         head->tail = &(t->next);
484         raise_softirq_irqoff(softirq_nr);
485         local_irq_restore(flags);
486 }
487 
488 void __tasklet_schedule(struct tasklet_struct *t)
489 {
490         __tasklet_schedule_common(t, &tasklet_vec,
491                                   TASKLET_SOFTIRQ);
492 }
493 EXPORT_SYMBOL(__tasklet_schedule);
494 
495 void __tasklet_hi_schedule(struct tasklet_struct *t)
496 {
497         __tasklet_schedule_common(t, &tasklet_hi_vec,
498                                   HI_SOFTIRQ);
499 }
500 EXPORT_SYMBOL(__tasklet_hi_schedule);
501 
502 static void tasklet_action_common(struct softirq_action *a,
503                                   struct tasklet_head *tl_head,
504                                   unsigned int softirq_nr)
505 {
506         struct tasklet_struct *list;
507 
508         local_irq_disable();
509         list = tl_head->head;
510         tl_head->head = NULL;
511         tl_head->tail = &tl_head->head;
512         local_irq_enable();
513 
514         while (list) {
515                 struct tasklet_struct *t = list;
516 
517                 list = list->next;
518 
519                 if (tasklet_trylock(t)) {
520                         if (!atomic_read(&t->count)) {
521                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED,
522                                                         &t->state))
523                                         BUG();
524                                 t->func(t->data);
525                                 tasklet_unlock(t);
526                                 continue;
527                         }
528                         tasklet_unlock(t);
529                 }
530 
531                 local_irq_disable();
532                 t->next = NULL;
533                 *tl_head->tail = t;
534                 tl_head->tail = &t->next;
535                 __raise_softirq_irqoff(softirq_nr);
536                 local_irq_enable();
537         }
538 }
539 
540 static __latent_entropy void tasklet_action(struct softirq_action *a)
541 {
542         tasklet_action_common(a, this_cpu_ptr(&tasklet_vec), TASKLET_SOFTIRQ);
543 }
544 
545 static __latent_entropy void tasklet_hi_action(struct softirq_action *a)
546 {
547         tasklet_action_common(a, this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ);
548 }
549 
550 void tasklet_init(struct tasklet_struct *t,
551                   void (*func)(unsigned long), unsigned long data)
552 {
553         t->next = NULL;
554         t->state = 0;
555         atomic_set(&t->count, 0);
556         t->func = func;
557         t->data = data;
558 }
559 EXPORT_SYMBOL(tasklet_init);
560 
561 void tasklet_kill(struct tasklet_struct *t)
562 {
563         if (in_interrupt())
564                 pr_notice("Attempt to kill tasklet from interrupt\n");
565 
566         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
567                 do {
568                         yield();
569                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
570         }
571         tasklet_unlock_wait(t);
572         clear_bit(TASKLET_STATE_SCHED, &t->state);
573 }
574 EXPORT_SYMBOL(tasklet_kill);
575 
576 /*
577  * tasklet_hrtimer
578  */
579 
580 /*
581  * The trampoline is called when the hrtimer expires. It schedules a tasklet
582  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
583  * hrtimer callback, but from softirq context.
584  */
585 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
586 {
587         struct tasklet_hrtimer *ttimer =
588                 container_of(timer, struct tasklet_hrtimer, timer);
589 
590         tasklet_hi_schedule(&ttimer->tasklet);
591         return HRTIMER_NORESTART;
592 }
593 
594 /*
595  * Helper function which calls the hrtimer callback from
596  * tasklet/softirq context
597  */
598 static void __tasklet_hrtimer_trampoline(unsigned long data)
599 {
600         struct tasklet_hrtimer *ttimer = (void *)data;
601         enum hrtimer_restart restart;
602 
603         restart = ttimer->function(&ttimer->timer);
604         if (restart != HRTIMER_NORESTART)
605                 hrtimer_restart(&ttimer->timer);
606 }
607 
608 /**
609  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
610  * @ttimer:      tasklet_hrtimer which is initialized
611  * @function:    hrtimer callback function which gets called from softirq context
612  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
613  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
614  */
615 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
616                           enum hrtimer_restart (*function)(struct hrtimer *),
617                           clockid_t which_clock, enum hrtimer_mode mode)
618 {
619         hrtimer_init(&ttimer->timer, which_clock, mode);
620         ttimer->timer.function = __hrtimer_tasklet_trampoline;
621         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
622                      (unsigned long)ttimer);
623         ttimer->function = function;
624 }
625 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
626 
627 void __init softirq_init(void)
628 {
629         int cpu;
630 
631         for_each_possible_cpu(cpu) {
632                 per_cpu(tasklet_vec, cpu).tail =
633                         &per_cpu(tasklet_vec, cpu).head;
634                 per_cpu(tasklet_hi_vec, cpu).tail =
635                         &per_cpu(tasklet_hi_vec, cpu).head;
636         }
637 
638         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
639         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
640 }
641 
642 static int ksoftirqd_should_run(unsigned int cpu)
643 {
644         return local_softirq_pending();
645 }
646 
647 static void run_ksoftirqd(unsigned int cpu)
648 {
649         local_irq_disable();
650         if (local_softirq_pending()) {
651                 /*
652                  * We can safely run softirq on inline stack, as we are not deep
653                  * in the task stack here.
654                  */
655                 __do_softirq();
656                 local_irq_enable();
657                 cond_resched();
658                 return;
659         }
660         local_irq_enable();
661 }
662 
663 #ifdef CONFIG_HOTPLUG_CPU
664 /*
665  * tasklet_kill_immediate is called to remove a tasklet which can already be
666  * scheduled for execution on @cpu.
667  *
668  * Unlike tasklet_kill, this function removes the tasklet
669  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
670  *
671  * When this function is called, @cpu must be in the CPU_DEAD state.
672  */
673 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
674 {
675         struct tasklet_struct **i;
676 
677         BUG_ON(cpu_online(cpu));
678         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
679 
680         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
681                 return;
682 
683         /* CPU is dead, so no lock needed. */
684         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
685                 if (*i == t) {
686                         *i = t->next;
687                         /* If this was the tail element, move the tail ptr */
688                         if (*i == NULL)
689                                 per_cpu(tasklet_vec, cpu).tail = i;
690                         return;
691                 }
692         }
693         BUG();
694 }
695 
696 static int takeover_tasklets(unsigned int cpu)
697 {
698         /* CPU is dead, so no lock needed. */
699         local_irq_disable();
700 
701         /* Find end, append list for that CPU. */
702         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
703                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
704                 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
705                 per_cpu(tasklet_vec, cpu).head = NULL;
706                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
707         }
708         raise_softirq_irqoff(TASKLET_SOFTIRQ);
709 
710         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
711                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
712                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
713                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
714                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
715         }
716         raise_softirq_irqoff(HI_SOFTIRQ);
717 
718         local_irq_enable();
719         return 0;
720 }
721 #else
722 #define takeover_tasklets       NULL
723 #endif /* CONFIG_HOTPLUG_CPU */
724 
725 static struct smp_hotplug_thread softirq_threads = {
726         .store                  = &ksoftirqd,
727         .thread_should_run      = ksoftirqd_should_run,
728         .thread_fn              = run_ksoftirqd,
729         .thread_comm            = "ksoftirqd/%u",
730 };
731 
732 static __init int spawn_ksoftirqd(void)
733 {
734         cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD, "softirq:dead", NULL,
735                                   takeover_tasklets);
736         BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
737 
738         return 0;
739 }
740 early_initcall(spawn_ksoftirqd);
741 
742 /*
743  * [ These __weak aliases are kept in a separate compilation unit, so that
744  *   GCC does not inline them incorrectly. ]
745  */
746 
747 int __init __weak early_irq_init(void)
748 {
749         return 0;
750 }
751 
752 int __init __weak arch_probe_nr_irqs(void)
753 {
754         return NR_IRQS_LEGACY;
755 }
756 
757 int __init __weak arch_early_irq_init(void)
758 {
759         return 0;
760 }
761 
762 unsigned int __weak arch_dynirq_lower_bound(unsigned int from)
763 {
764         return from;
765 }
766 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp