1 /*
2 * linux/kernel/softirq.c
3 *
4 * Copyright (C) 1992 Linus Torvalds
5 *
6 * Distribute under GPLv2.
7 *
8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9 */
10
11 #include <linux/export.h>
12 #include <linux/kernel_stat.h>
13 #include <linux/interrupt.h>
14 #include <linux/init.h>
15 #include <linux/mm.h>
16 #include <linux/notifier.h>
17 #include <linux/percpu.h>
18 #include <linux/cpu.h>
19 #include <linux/freezer.h>
20 #include <linux/kthread.h>
21 #include <linux/rcupdate.h>
22 #include <linux/ftrace.h>
23 #include <linux/smp.h>
24 #include <linux/smpboot.h>
25 #include <linux/tick.h>
26
27 #define CREATE_TRACE_POINTS
28 #include <trace/events/irq.h>
29
30 #include <asm/irq.h>
31 /*
32 - No shared variables, all the data are CPU local.
33 - If a softirq needs serialization, let it serialize itself
34 by its own spinlocks.
35 - Even if softirq is serialized, only local cpu is marked for
36 execution. Hence, we get something sort of weak cpu binding.
37 Though it is still not clear, will it result in better locality
38 or will not.
39
40 Examples:
41 - NET RX softirq. It is multithreaded and does not require
42 any global serialization.
43 - NET TX softirq. It kicks software netdevice queues, hence
44 it is logically serialized per device, but this serialization
45 is invisible to common code.
46 - Tasklets: serialized wrt itself.
47 */
48
49 #ifndef __ARCH_IRQ_STAT
50 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
51 EXPORT_SYMBOL(irq_stat);
52 #endif
53
54 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
55
56 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
57
58 char *softirq_to_name[NR_SOFTIRQS] = {
59 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
60 "TASKLET", "SCHED", "HRTIMER", "RCU"
61 };
62
63 /*
64 * we cannot loop indefinitely here to avoid userspace starvation,
65 * but we also don't want to introduce a worst case 1/HZ latency
66 * to the pending events, so lets the scheduler to balance
67 * the softirq load for us.
68 */
69 static void wakeup_softirqd(void)
70 {
71 /* Interrupts are disabled: no need to stop preemption */
72 struct task_struct *tsk = __this_cpu_read(ksoftirqd);
73
74 if (tsk && tsk->state != TASK_RUNNING)
75 wake_up_process(tsk);
76 }
77
78 /*
79 * preempt_count and SOFTIRQ_OFFSET usage:
80 * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
81 * softirq processing.
82 * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
83 * on local_bh_disable or local_bh_enable.
84 * This lets us distinguish between whether we are currently processing
85 * softirq and whether we just have bh disabled.
86 */
87
88 /*
89 * This one is for softirq.c-internal use,
90 * where hardirqs are disabled legitimately:
91 */
92 #ifdef CONFIG_TRACE_IRQFLAGS
93 static void __local_bh_disable(unsigned long ip, unsigned int cnt)
94 {
95 unsigned long flags;
96
97 WARN_ON_ONCE(in_irq());
98
99 raw_local_irq_save(flags);
100 /*
101 * The preempt tracer hooks into add_preempt_count and will break
102 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
103 * is set and before current->softirq_enabled is cleared.
104 * We must manually increment preempt_count here and manually
105 * call the trace_preempt_off later.
106 */
107 preempt_count() += cnt;
108 /*
109 * Were softirqs turned off above:
110 */
111 if (softirq_count() == cnt)
112 trace_softirqs_off(ip);
113 raw_local_irq_restore(flags);
114
115 if (preempt_count() == cnt)
116 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
117 }
118 #else /* !CONFIG_TRACE_IRQFLAGS */
119 static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
120 {
121 add_preempt_count(cnt);
122 barrier();
123 }
124 #endif /* CONFIG_TRACE_IRQFLAGS */
125
126 void local_bh_disable(void)
127 {
128 __local_bh_disable((unsigned long)__builtin_return_address(0),
129 SOFTIRQ_DISABLE_OFFSET);
130 }
131
132 EXPORT_SYMBOL(local_bh_disable);
133
134 static void __local_bh_enable(unsigned int cnt)
135 {
136 WARN_ON_ONCE(in_irq());
137 WARN_ON_ONCE(!irqs_disabled());
138
139 if (softirq_count() == cnt)
140 trace_softirqs_on((unsigned long)__builtin_return_address(0));
141 sub_preempt_count(cnt);
142 }
143
144 /*
145 * Special-case - softirqs can safely be enabled in
146 * cond_resched_softirq(), or by __do_softirq(),
147 * without processing still-pending softirqs:
148 */
149 void _local_bh_enable(void)
150 {
151 __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
152 }
153
154 EXPORT_SYMBOL(_local_bh_enable);
155
156 static inline void _local_bh_enable_ip(unsigned long ip)
157 {
158 WARN_ON_ONCE(in_irq() || irqs_disabled());
159 #ifdef CONFIG_TRACE_IRQFLAGS
160 local_irq_disable();
161 #endif
162 /*
163 * Are softirqs going to be turned on now:
164 */
165 if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
166 trace_softirqs_on(ip);
167 /*
168 * Keep preemption disabled until we are done with
169 * softirq processing:
170 */
171 sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
172
173 if (unlikely(!in_interrupt() && local_softirq_pending()))
174 do_softirq();
175
176 dec_preempt_count();
177 #ifdef CONFIG_TRACE_IRQFLAGS
178 local_irq_enable();
179 #endif
180 preempt_check_resched();
181 }
182
183 void local_bh_enable(void)
184 {
185 _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
186 }
187 EXPORT_SYMBOL(local_bh_enable);
188
189 void local_bh_enable_ip(unsigned long ip)
190 {
191 _local_bh_enable_ip(ip);
192 }
193 EXPORT_SYMBOL(local_bh_enable_ip);
194
195 /*
196 * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
197 * but break the loop if need_resched() is set or after 2 ms.
198 * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
199 * certain cases, such as stop_machine(), jiffies may cease to
200 * increment and so we need the MAX_SOFTIRQ_RESTART limit as
201 * well to make sure we eventually return from this method.
202 *
203 * These limits have been established via experimentation.
204 * The two things to balance is latency against fairness -
205 * we want to handle softirqs as soon as possible, but they
206 * should not be able to lock up the box.
207 */
208 #define MAX_SOFTIRQ_TIME msecs_to_jiffies(2)
209 #define MAX_SOFTIRQ_RESTART 10
210
211 asmlinkage void __do_softirq(void)
212 {
213 struct softirq_action *h;
214 __u32 pending;
215 unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
216 int cpu;
217 unsigned long old_flags = current->flags;
218 int max_restart = MAX_SOFTIRQ_RESTART;
219
220 /*
221 * Mask out PF_MEMALLOC s current task context is borrowed for the
222 * softirq. A softirq handled such as network RX might set PF_MEMALLOC
223 * again if the socket is related to swap
224 */
225 current->flags &= ~PF_MEMALLOC;
226
227 pending = local_softirq_pending();
228 account_irq_enter_time(current);
229
230 __local_bh_disable((unsigned long)__builtin_return_address(0),
231 SOFTIRQ_OFFSET);
232 lockdep_softirq_enter();
233
234 cpu = smp_processor_id();
235 restart:
236 /* Reset the pending bitmask before enabling irqs */
237 set_softirq_pending(0);
238
239 local_irq_enable();
240
241 h = softirq_vec;
242
243 do {
244 if (pending & 1) {
245 unsigned int vec_nr = h - softirq_vec;
246 int prev_count = preempt_count();
247
248 kstat_incr_softirqs_this_cpu(vec_nr);
249
250 trace_softirq_entry(vec_nr);
251 h->action(h);
252 trace_softirq_exit(vec_nr);
253 if (unlikely(prev_count != preempt_count())) {
254 printk(KERN_ERR "huh, entered softirq %u %s %p"
255 "with preempt_count %08x,"
256 " exited with %08x?\n", vec_nr,
257 softirq_to_name[vec_nr], h->action,
258 prev_count, preempt_count());
259 preempt_count() = prev_count;
260 }
261
262 rcu_bh_qs(cpu);
263 }
264 h++;
265 pending >>= 1;
266 } while (pending);
267
268 local_irq_disable();
269
270 pending = local_softirq_pending();
271 if (pending) {
272 if (time_before(jiffies, end) && !need_resched() &&
273 --max_restart)
274 goto restart;
275
276 wakeup_softirqd();
277 }
278
279 lockdep_softirq_exit();
280
281 account_irq_exit_time(current);
282 __local_bh_enable(SOFTIRQ_OFFSET);
283 tsk_restore_flags(current, old_flags, PF_MEMALLOC);
284 }
285
286 #ifndef __ARCH_HAS_DO_SOFTIRQ
287
288 asmlinkage void do_softirq(void)
289 {
290 __u32 pending;
291 unsigned long flags;
292
293 if (in_interrupt())
294 return;
295
296 local_irq_save(flags);
297
298 pending = local_softirq_pending();
299
300 if (pending)
301 __do_softirq();
302
303 local_irq_restore(flags);
304 }
305
306 #endif
307
308 /*
309 * Enter an interrupt context.
310 */
311 void irq_enter(void)
312 {
313 int cpu = smp_processor_id();
314
315 rcu_irq_enter();
316 if (is_idle_task(current) && !in_interrupt()) {
317 /*
318 * Prevent raise_softirq from needlessly waking up ksoftirqd
319 * here, as softirq will be serviced on return from interrupt.
320 */
321 local_bh_disable();
322 tick_check_idle(cpu);
323 _local_bh_enable();
324 }
325
326 __irq_enter();
327 }
328
329 static inline void invoke_softirq(void)
330 {
331 if (!force_irqthreads) {
332 /*
333 * We can safely execute softirq on the current stack if
334 * it is the irq stack, because it should be near empty
335 * at this stage. But we have no way to know if the arch
336 * calls irq_exit() on the irq stack. So call softirq
337 * in its own stack to prevent from any overrun on top
338 * of a potentially deep task stack.
339 */
340 do_softirq();
341 } else {
342 wakeup_softirqd();
343 }
344 }
345
346 static inline void tick_irq_exit(void)
347 {
348 #ifdef CONFIG_NO_HZ_COMMON
349 int cpu = smp_processor_id();
350
351 /* Make sure that timer wheel updates are propagated */
352 if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
353 if (!in_interrupt())
354 tick_nohz_irq_exit();
355 }
356 #endif
357 }
358
359 /*
360 * Exit an interrupt context. Process softirqs if needed and possible:
361 */
362 void irq_exit(void)
363 {
364 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
365 local_irq_disable();
366 #else
367 WARN_ON_ONCE(!irqs_disabled());
368 #endif
369
370 account_irq_exit_time(current);
371 trace_hardirq_exit();
372 sub_preempt_count(HARDIRQ_OFFSET);
373 if (!in_interrupt() && local_softirq_pending())
374 invoke_softirq();
375
376 tick_irq_exit();
377 rcu_irq_exit();
378 }
379
380 /*
381 * This function must run with irqs disabled!
382 */
383 inline void raise_softirq_irqoff(unsigned int nr)
384 {
385 __raise_softirq_irqoff(nr);
386
387 /*
388 * If we're in an interrupt or softirq, we're done
389 * (this also catches softirq-disabled code). We will
390 * actually run the softirq once we return from
391 * the irq or softirq.
392 *
393 * Otherwise we wake up ksoftirqd to make sure we
394 * schedule the softirq soon.
395 */
396 if (!in_interrupt())
397 wakeup_softirqd();
398 }
399
400 void raise_softirq(unsigned int nr)
401 {
402 unsigned long flags;
403
404 local_irq_save(flags);
405 raise_softirq_irqoff(nr);
406 local_irq_restore(flags);
407 }
408
409 void __raise_softirq_irqoff(unsigned int nr)
410 {
411 trace_softirq_raise(nr);
412 or_softirq_pending(1UL << nr);
413 }
414
415 void open_softirq(int nr, void (*action)(struct softirq_action *))
416 {
417 softirq_vec[nr].action = action;
418 }
419
420 /*
421 * Tasklets
422 */
423 struct tasklet_head
424 {
425 struct tasklet_struct *head;
426 struct tasklet_struct **tail;
427 };
428
429 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
430 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
431
432 void __tasklet_schedule(struct tasklet_struct *t)
433 {
434 unsigned long flags;
435
436 local_irq_save(flags);
437 t->next = NULL;
438 *__this_cpu_read(tasklet_vec.tail) = t;
439 __this_cpu_write(tasklet_vec.tail, &(t->next));
440 raise_softirq_irqoff(TASKLET_SOFTIRQ);
441 local_irq_restore(flags);
442 }
443
444 EXPORT_SYMBOL(__tasklet_schedule);
445
446 void __tasklet_hi_schedule(struct tasklet_struct *t)
447 {
448 unsigned long flags;
449
450 local_irq_save(flags);
451 t->next = NULL;
452 *__this_cpu_read(tasklet_hi_vec.tail) = t;
453 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
454 raise_softirq_irqoff(HI_SOFTIRQ);
455 local_irq_restore(flags);
456 }
457
458 EXPORT_SYMBOL(__tasklet_hi_schedule);
459
460 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
461 {
462 BUG_ON(!irqs_disabled());
463
464 t->next = __this_cpu_read(tasklet_hi_vec.head);
465 __this_cpu_write(tasklet_hi_vec.head, t);
466 __raise_softirq_irqoff(HI_SOFTIRQ);
467 }
468
469 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
470
471 static void tasklet_action(struct softirq_action *a)
472 {
473 struct tasklet_struct *list;
474
475 local_irq_disable();
476 list = __this_cpu_read(tasklet_vec.head);
477 __this_cpu_write(tasklet_vec.head, NULL);
478 __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
479 local_irq_enable();
480
481 while (list) {
482 struct tasklet_struct *t = list;
483
484 list = list->next;
485
486 if (tasklet_trylock(t)) {
487 if (!atomic_read(&t->count)) {
488 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
489 BUG();
490 t->func(t->data);
491 tasklet_unlock(t);
492 continue;
493 }
494 tasklet_unlock(t);
495 }
496
497 local_irq_disable();
498 t->next = NULL;
499 *__this_cpu_read(tasklet_vec.tail) = t;
500 __this_cpu_write(tasklet_vec.tail, &(t->next));
501 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
502 local_irq_enable();
503 }
504 }
505
506 static void tasklet_hi_action(struct softirq_action *a)
507 {
508 struct tasklet_struct *list;
509
510 local_irq_disable();
511 list = __this_cpu_read(tasklet_hi_vec.head);
512 __this_cpu_write(tasklet_hi_vec.head, NULL);
513 __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
514 local_irq_enable();
515
516 while (list) {
517 struct tasklet_struct *t = list;
518
519 list = list->next;
520
521 if (tasklet_trylock(t)) {
522 if (!atomic_read(&t->count)) {
523 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
524 BUG();
525 t->func(t->data);
526 tasklet_unlock(t);
527 continue;
528 }
529 tasklet_unlock(t);
530 }
531
532 local_irq_disable();
533 t->next = NULL;
534 *__this_cpu_read(tasklet_hi_vec.tail) = t;
535 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
536 __raise_softirq_irqoff(HI_SOFTIRQ);
537 local_irq_enable();
538 }
539 }
540
541
542 void tasklet_init(struct tasklet_struct *t,
543 void (*func)(unsigned long), unsigned long data)
544 {
545 t->next = NULL;
546 t->state = 0;
547 atomic_set(&t->count, 0);
548 t->func = func;
549 t->data = data;
550 }
551
552 EXPORT_SYMBOL(tasklet_init);
553
554 void tasklet_kill(struct tasklet_struct *t)
555 {
556 if (in_interrupt())
557 printk("Attempt to kill tasklet from interrupt\n");
558
559 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
560 do {
561 yield();
562 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
563 }
564 tasklet_unlock_wait(t);
565 clear_bit(TASKLET_STATE_SCHED, &t->state);
566 }
567
568 EXPORT_SYMBOL(tasklet_kill);
569
570 /*
571 * tasklet_hrtimer
572 */
573
574 /*
575 * The trampoline is called when the hrtimer expires. It schedules a tasklet
576 * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
577 * hrtimer callback, but from softirq context.
578 */
579 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
580 {
581 struct tasklet_hrtimer *ttimer =
582 container_of(timer, struct tasklet_hrtimer, timer);
583
584 tasklet_hi_schedule(&ttimer->tasklet);
585 return HRTIMER_NORESTART;
586 }
587
588 /*
589 * Helper function which calls the hrtimer callback from
590 * tasklet/softirq context
591 */
592 static void __tasklet_hrtimer_trampoline(unsigned long data)
593 {
594 struct tasklet_hrtimer *ttimer = (void *)data;
595 enum hrtimer_restart restart;
596
597 restart = ttimer->function(&ttimer->timer);
598 if (restart != HRTIMER_NORESTART)
599 hrtimer_restart(&ttimer->timer);
600 }
601
602 /**
603 * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
604 * @ttimer: tasklet_hrtimer which is initialized
605 * @function: hrtimer callback function which gets called from softirq context
606 * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
607 * @mode: hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
608 */
609 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
610 enum hrtimer_restart (*function)(struct hrtimer *),
611 clockid_t which_clock, enum hrtimer_mode mode)
612 {
613 hrtimer_init(&ttimer->timer, which_clock, mode);
614 ttimer->timer.function = __hrtimer_tasklet_trampoline;
615 tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
616 (unsigned long)ttimer);
617 ttimer->function = function;
618 }
619 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
620
621 void __init softirq_init(void)
622 {
623 int cpu;
624
625 for_each_possible_cpu(cpu) {
626 per_cpu(tasklet_vec, cpu).tail =
627 &per_cpu(tasklet_vec, cpu).head;
628 per_cpu(tasklet_hi_vec, cpu).tail =
629 &per_cpu(tasklet_hi_vec, cpu).head;
630 }
631
632 open_softirq(TASKLET_SOFTIRQ, tasklet_action);
633 open_softirq(HI_SOFTIRQ, tasklet_hi_action);
634 }
635
636 static int ksoftirqd_should_run(unsigned int cpu)
637 {
638 return local_softirq_pending();
639 }
640
641 static void run_ksoftirqd(unsigned int cpu)
642 {
643 local_irq_disable();
644 if (local_softirq_pending()) {
645 __do_softirq();
646 local_irq_enable();
647 cond_resched();
648
649 preempt_disable();
650 rcu_note_context_switch(cpu);
651 preempt_enable();
652
653 return;
654 }
655 local_irq_enable();
656 }
657
658 #ifdef CONFIG_HOTPLUG_CPU
659 /*
660 * tasklet_kill_immediate is called to remove a tasklet which can already be
661 * scheduled for execution on @cpu.
662 *
663 * Unlike tasklet_kill, this function removes the tasklet
664 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
665 *
666 * When this function is called, @cpu must be in the CPU_DEAD state.
667 */
668 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
669 {
670 struct tasklet_struct **i;
671
672 BUG_ON(cpu_online(cpu));
673 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
674
675 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
676 return;
677
678 /* CPU is dead, so no lock needed. */
679 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
680 if (*i == t) {
681 *i = t->next;
682 /* If this was the tail element, move the tail ptr */
683 if (*i == NULL)
684 per_cpu(tasklet_vec, cpu).tail = i;
685 return;
686 }
687 }
688 BUG();
689 }
690
691 static void takeover_tasklets(unsigned int cpu)
692 {
693 /* CPU is dead, so no lock needed. */
694 local_irq_disable();
695
696 /* Find end, append list for that CPU. */
697 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
698 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
699 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
700 per_cpu(tasklet_vec, cpu).head = NULL;
701 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
702 }
703 raise_softirq_irqoff(TASKLET_SOFTIRQ);
704
705 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
706 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
707 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
708 per_cpu(tasklet_hi_vec, cpu).head = NULL;
709 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
710 }
711 raise_softirq_irqoff(HI_SOFTIRQ);
712
713 local_irq_enable();
714 }
715 #endif /* CONFIG_HOTPLUG_CPU */
716
717 static int __cpuinit cpu_callback(struct notifier_block *nfb,
718 unsigned long action,
719 void *hcpu)
720 {
721 switch (action) {
722 #ifdef CONFIG_HOTPLUG_CPU
723 case CPU_DEAD:
724 case CPU_DEAD_FROZEN:
725 takeover_tasklets((unsigned long)hcpu);
726 break;
727 #endif /* CONFIG_HOTPLUG_CPU */
728 }
729 return NOTIFY_OK;
730 }
731
732 static struct notifier_block __cpuinitdata cpu_nfb = {
733 .notifier_call = cpu_callback
734 };
735
736 static struct smp_hotplug_thread softirq_threads = {
737 .store = &ksoftirqd,
738 .thread_should_run = ksoftirqd_should_run,
739 .thread_fn = run_ksoftirqd,
740 .thread_comm = "ksoftirqd/%u",
741 };
742
743 static __init int spawn_ksoftirqd(void)
744 {
745 register_cpu_notifier(&cpu_nfb);
746
747 BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
748
749 return 0;
750 }
751 early_initcall(spawn_ksoftirqd);
752
753 /*
754 * [ These __weak aliases are kept in a separate compilation unit, so that
755 * GCC does not inline them incorrectly. ]
756 */
757
758 int __init __weak early_irq_init(void)
759 {
760 return 0;
761 }
762
763 #ifdef CONFIG_GENERIC_HARDIRQS
764 int __init __weak arch_probe_nr_irqs(void)
765 {
766 return NR_IRQS_LEGACY;
767 }
768
769 int __init __weak arch_early_irq_init(void)
770 {
771 return 0;
772 }
773 #endif
774
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.