2 * linux/kernel/softirq.c
4 * Copyright (C) 1992 Linus Torvalds
6 * Distribute under GPLv2.
8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
13 #include <linux/export.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/interrupt.h>
16 #include <linux/init.h>
18 #include <linux/notifier.h>
19 #include <linux/percpu.h>
20 #include <linux/cpu.h>
21 #include <linux/freezer.h>
22 #include <linux/kthread.h>
23 #include <linux/rcupdate.h>
24 #include <linux/delay.h>
25 #include <linux/ftrace.h>
26 #include <linux/smp.h>
27 #include <linux/smpboot.h>
28 #include <linux/tick.h>
29 #include <linux/locallock.h>
30 #include <linux/irq.h>
32 #define CREATE_TRACE_POINTS
33 #include <trace/events/irq.h>
36 - No shared variables, all the data are CPU local.
37 - If a softirq needs serialization, let it serialize itself
39 - Even if softirq is serialized, only local cpu is marked for
40 execution. Hence, we get something sort of weak cpu binding.
41 Though it is still not clear, will it result in better locality
45 - NET RX softirq. It is multithreaded and does not require
46 any global serialization.
47 - NET TX softirq. It kicks software netdevice queues, hence
48 it is logically serialized per device, but this serialization
49 is invisible to common code.
50 - Tasklets: serialized wrt itself.
53 #ifndef __ARCH_IRQ_STAT
54 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
55 EXPORT_SYMBOL(irq_stat);
58 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
60 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
61 #ifdef CONFIG_PREEMPT_RT_FULL
62 #define TIMER_SOFTIRQS ((1 << TIMER_SOFTIRQ) | (1 << HRTIMER_SOFTIRQ))
63 DEFINE_PER_CPU(struct task_struct *, ktimer_softirqd);
66 const char * const softirq_to_name[NR_SOFTIRQS] = {
67 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
68 "TASKLET", "SCHED", "HRTIMER", "RCU"
71 #ifdef CONFIG_NO_HZ_COMMON
72 # ifdef CONFIG_PREEMPT_RT_FULL
74 struct softirq_runner {
75 struct task_struct *runner[NR_SOFTIRQS];
78 static DEFINE_PER_CPU(struct softirq_runner, softirq_runners);
80 static inline void softirq_set_runner(unsigned int sirq)
82 struct softirq_runner *sr = this_cpu_ptr(&softirq_runners);
84 sr->runner[sirq] = current;
87 static inline void softirq_clr_runner(unsigned int sirq)
89 struct softirq_runner *sr = this_cpu_ptr(&softirq_runners);
91 sr->runner[sirq] = NULL;
95 * On preempt-rt a softirq running context might be blocked on a
96 * lock. There might be no other runnable task on this CPU because the
97 * lock owner runs on some other CPU. So we have to go into idle with
98 * the pending bit set. Therefor we need to check this otherwise we
99 * warn about false positives which confuses users and defeats the
100 * whole purpose of this test.
102 * This code is called with interrupts disabled.
104 void softirq_check_pending_idle(void)
106 static int rate_limit;
107 struct softirq_runner *sr = this_cpu_ptr(&softirq_runners);
111 if (rate_limit >= 10)
114 warnpending = local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK;
115 for (i = 0; i < NR_SOFTIRQS; i++) {
116 struct task_struct *tsk = sr->runner[i];
119 * The wakeup code in rtmutex.c wakes up the task
120 * _before_ it sets pi_blocked_on to NULL under
121 * tsk->pi_lock. So we need to check for both: state
125 raw_spin_lock(&tsk->pi_lock);
126 if (tsk->pi_blocked_on || tsk->state == TASK_RUNNING) {
127 /* Clear all bits pending in that task */
128 warnpending &= ~(tsk->softirqs_raised);
129 warnpending &= ~(1 << i);
131 raw_spin_unlock(&tsk->pi_lock);
136 printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
143 * On !PREEMPT_RT we just printk rate limited:
145 void softirq_check_pending_idle(void)
147 static int rate_limit;
149 if (rate_limit < 10 &&
150 (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) {
151 printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
152 local_softirq_pending());
158 #else /* !CONFIG_NO_HZ_COMMON */
159 static inline void softirq_set_runner(unsigned int sirq) { }
160 static inline void softirq_clr_runner(unsigned int sirq) { }
164 * we cannot loop indefinitely here to avoid userspace starvation,
165 * but we also don't want to introduce a worst case 1/HZ latency
166 * to the pending events, so lets the scheduler to balance
167 * the softirq load for us.
169 static void wakeup_softirqd(void)
171 /* Interrupts are disabled: no need to stop preemption */
172 struct task_struct *tsk = __this_cpu_read(ksoftirqd);
174 if (tsk && tsk->state != TASK_RUNNING)
175 wake_up_process(tsk);
178 #ifdef CONFIG_PREEMPT_RT_FULL
179 static void wakeup_timer_softirqd(void)
181 /* Interrupts are disabled: no need to stop preemption */
182 struct task_struct *tsk = __this_cpu_read(ktimer_softirqd);
184 if (tsk && tsk->state != TASK_RUNNING)
185 wake_up_process(tsk);
189 static void handle_softirq(unsigned int vec_nr)
191 struct softirq_action *h = softirq_vec + vec_nr;
194 prev_count = preempt_count();
196 kstat_incr_softirqs_this_cpu(vec_nr);
198 trace_softirq_entry(vec_nr);
200 trace_softirq_exit(vec_nr);
201 if (unlikely(prev_count != preempt_count())) {
202 pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
203 vec_nr, softirq_to_name[vec_nr], h->action,
204 prev_count, preempt_count());
205 preempt_count_set(prev_count);
209 #ifndef CONFIG_PREEMPT_RT_FULL
210 static inline int ksoftirqd_softirq_pending(void)
212 return local_softirq_pending();
215 static void handle_pending_softirqs(u32 pending)
217 struct softirq_action *h = softirq_vec;
224 while ((softirq_bit = ffs(pending))) {
227 h += softirq_bit - 1;
228 vec_nr = h - softirq_vec;
229 handle_softirq(vec_nr);
232 pending >>= softirq_bit;
239 static void run_ksoftirqd(unsigned int cpu)
242 if (ksoftirqd_softirq_pending()) {
245 cond_resched_rcu_qs();
252 * preempt_count and SOFTIRQ_OFFSET usage:
253 * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
254 * softirq processing.
255 * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
256 * on local_bh_disable or local_bh_enable.
257 * This lets us distinguish between whether we are currently processing
258 * softirq and whether we just have bh disabled.
262 * This one is for softirq.c-internal use,
263 * where hardirqs are disabled legitimately:
265 #ifdef CONFIG_TRACE_IRQFLAGS
266 void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
270 WARN_ON_ONCE(in_irq());
272 raw_local_irq_save(flags);
274 * The preempt tracer hooks into preempt_count_add and will break
275 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
276 * is set and before current->softirq_enabled is cleared.
277 * We must manually increment preempt_count here and manually
278 * call the trace_preempt_off later.
280 __preempt_count_add(cnt);
282 * Were softirqs turned off above:
284 if (softirq_count() == (cnt & SOFTIRQ_MASK))
285 trace_softirqs_off(ip);
286 raw_local_irq_restore(flags);
288 if (preempt_count() == cnt) {
289 #ifdef CONFIG_DEBUG_PREEMPT
290 current->preempt_disable_ip = get_lock_parent_ip();
292 trace_preempt_off(CALLER_ADDR0, get_lock_parent_ip());
295 EXPORT_SYMBOL(__local_bh_disable_ip);
296 #endif /* CONFIG_TRACE_IRQFLAGS */
298 static void __local_bh_enable(unsigned int cnt)
300 WARN_ON_ONCE(!irqs_disabled());
302 if (softirq_count() == (cnt & SOFTIRQ_MASK))
303 trace_softirqs_on(_RET_IP_);
304 preempt_count_sub(cnt);
308 * Special-case - softirqs can safely be enabled in
309 * cond_resched_softirq(), or by __do_softirq(),
310 * without processing still-pending softirqs:
312 void _local_bh_enable(void)
314 WARN_ON_ONCE(in_irq());
315 __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
317 EXPORT_SYMBOL(_local_bh_enable);
319 void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
321 WARN_ON_ONCE(in_irq() || irqs_disabled());
322 #ifdef CONFIG_TRACE_IRQFLAGS
326 * Are softirqs going to be turned on now:
328 if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
329 trace_softirqs_on(ip);
331 * Keep preemption disabled until we are done with
332 * softirq processing:
334 preempt_count_sub(cnt - 1);
336 if (unlikely(!in_interrupt() && local_softirq_pending())) {
338 * Run softirq if any pending. And do it in its own stack
339 * as we may be calling this deep in a task call stack already.
345 #ifdef CONFIG_TRACE_IRQFLAGS
348 preempt_check_resched();
350 EXPORT_SYMBOL(__local_bh_enable_ip);
353 * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
354 * but break the loop if need_resched() is set or after 2 ms.
355 * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
356 * certain cases, such as stop_machine(), jiffies may cease to
357 * increment and so we need the MAX_SOFTIRQ_RESTART limit as
358 * well to make sure we eventually return from this method.
360 * These limits have been established via experimentation.
361 * The two things to balance is latency against fairness -
362 * we want to handle softirqs as soon as possible, but they
363 * should not be able to lock up the box.
365 #define MAX_SOFTIRQ_TIME msecs_to_jiffies(2)
366 #define MAX_SOFTIRQ_RESTART 10
368 #ifdef CONFIG_TRACE_IRQFLAGS
370 * When we run softirqs from irq_exit() and thus on the hardirq stack we need
371 * to keep the lockdep irq context tracking as tight as possible in order to
372 * not miss-qualify lock contexts and miss possible deadlocks.
375 static inline bool lockdep_softirq_start(void)
377 bool in_hardirq = false;
379 if (trace_hardirq_context(current)) {
381 trace_hardirq_exit();
384 lockdep_softirq_enter();
389 static inline void lockdep_softirq_end(bool in_hardirq)
391 lockdep_softirq_exit();
394 trace_hardirq_enter();
397 static inline bool lockdep_softirq_start(void) { return false; }
398 static inline void lockdep_softirq_end(bool in_hardirq) { }
401 asmlinkage __visible void __do_softirq(void)
403 unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
404 unsigned long old_flags = current->flags;
405 int max_restart = MAX_SOFTIRQ_RESTART;
410 * Mask out PF_MEMALLOC s current task context is borrowed for the
411 * softirq. A softirq handled such as network RX might set PF_MEMALLOC
412 * again if the socket is related to swap
414 current->flags &= ~PF_MEMALLOC;
416 pending = local_softirq_pending();
417 account_irq_enter_time(current);
419 __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
420 in_hardirq = lockdep_softirq_start();
423 /* Reset the pending bitmask before enabling irqs */
424 set_softirq_pending(0);
426 handle_pending_softirqs(pending);
428 pending = local_softirq_pending();
430 if (time_before(jiffies, end) && !need_resched() &&
437 lockdep_softirq_end(in_hardirq);
438 account_irq_exit_time(current);
439 __local_bh_enable(SOFTIRQ_OFFSET);
440 WARN_ON_ONCE(in_interrupt());
441 tsk_restore_flags(current, old_flags, PF_MEMALLOC);
444 asmlinkage __visible void do_softirq(void)
452 local_irq_save(flags);
454 pending = local_softirq_pending();
457 do_softirq_own_stack();
459 local_irq_restore(flags);
463 * This function must run with irqs disabled!
465 void raise_softirq_irqoff(unsigned int nr)
467 __raise_softirq_irqoff(nr);
470 * If we're in an interrupt or softirq, we're done
471 * (this also catches softirq-disabled code). We will
472 * actually run the softirq once we return from
473 * the irq or softirq.
475 * Otherwise we wake up ksoftirqd to make sure we
476 * schedule the softirq soon.
482 void __raise_softirq_irqoff(unsigned int nr)
484 trace_softirq_raise(nr);
485 or_softirq_pending(1UL << nr);
488 static inline void local_bh_disable_nort(void) { local_bh_disable(); }
489 static inline void _local_bh_enable_nort(void) { _local_bh_enable(); }
490 static void ksoftirqd_set_sched_params(unsigned int cpu) { }
492 #else /* !PREEMPT_RT_FULL */
495 * On RT we serialize softirq execution with a cpu local lock per softirq
497 static DEFINE_PER_CPU(struct local_irq_lock [NR_SOFTIRQS], local_softirq_locks);
499 void __init softirq_early_init(void)
503 for (i = 0; i < NR_SOFTIRQS; i++)
504 local_irq_lock_init(local_softirq_locks[i]);
507 static void lock_softirq(int which)
509 local_lock(local_softirq_locks[which]);
512 static void unlock_softirq(int which)
514 local_unlock(local_softirq_locks[which]);
517 static void do_single_softirq(int which)
519 unsigned long old_flags = current->flags;
521 current->flags &= ~PF_MEMALLOC;
522 vtime_account_irq_enter(current);
523 current->flags |= PF_IN_SOFTIRQ;
524 lockdep_softirq_enter();
526 handle_softirq(which);
528 lockdep_softirq_exit();
529 current->flags &= ~PF_IN_SOFTIRQ;
530 vtime_account_irq_enter(current);
531 tsk_restore_flags(current, old_flags, PF_MEMALLOC);
535 * Called with interrupts disabled. Process softirqs which were raised
536 * in current context (or on behalf of ksoftirqd).
538 static void do_current_softirqs(void)
540 while (current->softirqs_raised) {
541 int i = __ffs(current->softirqs_raised);
542 unsigned int pending, mask = (1U << i);
544 current->softirqs_raised &= ~mask;
548 * If the lock is contended, we boost the owner to
549 * process the softirq or leave the critical section
554 softirq_set_runner(i);
556 * Check with the local_softirq_pending() bits,
557 * whether we need to process this still or if someone
558 * else took care of it.
560 pending = local_softirq_pending();
561 if (pending & mask) {
562 set_softirq_pending(pending & ~mask);
563 do_single_softirq(i);
565 softirq_clr_runner(i);
566 WARN_ON(current->softirq_nestcnt != 1);
573 void __local_bh_disable(void)
575 if (++current->softirq_nestcnt == 1)
578 EXPORT_SYMBOL(__local_bh_disable);
580 void __local_bh_enable(void)
582 if (WARN_ON(current->softirq_nestcnt == 0))
586 if (current->softirq_nestcnt == 1 && current->softirqs_raised)
587 do_current_softirqs();
590 if (--current->softirq_nestcnt == 0)
593 EXPORT_SYMBOL(__local_bh_enable);
595 void _local_bh_enable(void)
597 if (WARN_ON(current->softirq_nestcnt == 0))
599 if (--current->softirq_nestcnt == 0)
602 EXPORT_SYMBOL(_local_bh_enable);
604 int in_serving_softirq(void)
606 return current->flags & PF_IN_SOFTIRQ;
608 EXPORT_SYMBOL(in_serving_softirq);
610 /* Called with preemption disabled */
611 static void run_ksoftirqd(unsigned int cpu)
614 current->softirq_nestcnt++;
616 do_current_softirqs();
617 current->softirq_nestcnt--;
619 cond_resched_rcu_qs();
623 * Called from netif_rx_ni(). Preemption enabled, but migration
624 * disabled. So the cpu can't go away under us.
626 void thread_do_softirq(void)
628 if (!in_serving_softirq() && current->softirqs_raised) {
629 current->softirq_nestcnt++;
630 do_current_softirqs();
631 current->softirq_nestcnt--;
635 static void do_raise_softirq_irqoff(unsigned int nr)
641 trace_softirq_raise(nr);
642 or_softirq_pending(mask);
645 * If we are not in a hard interrupt and inside a bh disabled
646 * region, we simply raise the flag on current. local_bh_enable()
647 * will make sure that the softirq is executed. Otherwise we
648 * delegate it to ksoftirqd.
650 if (!in_irq() && current->softirq_nestcnt)
651 current->softirqs_raised |= mask;
652 else if (!__this_cpu_read(ksoftirqd) || !__this_cpu_read(ktimer_softirqd))
655 if (mask & TIMER_SOFTIRQS)
656 __this_cpu_read(ktimer_softirqd)->softirqs_raised |= mask;
658 __this_cpu_read(ksoftirqd)->softirqs_raised |= mask;
661 static void wakeup_proper_softirq(unsigned int nr)
663 if ((1UL << nr) & TIMER_SOFTIRQS)
664 wakeup_timer_softirqd();
670 void __raise_softirq_irqoff(unsigned int nr)
672 do_raise_softirq_irqoff(nr);
673 if (!in_irq() && !current->softirq_nestcnt)
674 wakeup_proper_softirq(nr);
678 * Same as __raise_softirq_irqoff() but will process them in ksoftirqd
680 void __raise_softirq_irqoff_ksoft(unsigned int nr)
684 if (WARN_ON_ONCE(!__this_cpu_read(ksoftirqd) ||
685 !__this_cpu_read(ktimer_softirqd)))
689 trace_softirq_raise(nr);
690 or_softirq_pending(mask);
691 if (mask & TIMER_SOFTIRQS)
692 __this_cpu_read(ktimer_softirqd)->softirqs_raised |= mask;
694 __this_cpu_read(ksoftirqd)->softirqs_raised |= mask;
695 wakeup_proper_softirq(nr);
699 * This function must run with irqs disabled!
701 void raise_softirq_irqoff(unsigned int nr)
703 do_raise_softirq_irqoff(nr);
706 * If we're in an hard interrupt we let irq return code deal
707 * with the wakeup of ksoftirqd.
712 * If we are in thread context but outside of a bh disabled
713 * region, we need to wake ksoftirqd as well.
715 * CHECKME: Some of the places which do that could be wrapped
716 * into local_bh_disable/enable pairs. Though it's unclear
717 * whether this is worth the effort. To find those places just
718 * raise a WARN() if the condition is met.
720 if (!current->softirq_nestcnt)
721 wakeup_proper_softirq(nr);
724 static inline int ksoftirqd_softirq_pending(void)
726 return current->softirqs_raised;
729 static inline void local_bh_disable_nort(void) { }
730 static inline void _local_bh_enable_nort(void) { }
732 static inline void ksoftirqd_set_sched_params(unsigned int cpu)
734 /* Take over all but timer pending softirqs when starting */
736 current->softirqs_raised = local_softirq_pending() & ~TIMER_SOFTIRQS;
740 static inline void ktimer_softirqd_set_sched_params(unsigned int cpu)
742 struct sched_param param = { .sched_priority = 1 };
744 sched_setscheduler(current, SCHED_FIFO, ¶m);
746 /* Take over timer pending softirqs when starting */
748 current->softirqs_raised = local_softirq_pending() & TIMER_SOFTIRQS;
752 static inline void ktimer_softirqd_clr_sched_params(unsigned int cpu,
755 struct sched_param param = { .sched_priority = 0 };
757 sched_setscheduler(current, SCHED_NORMAL, ¶m);
760 static int ktimer_softirqd_should_run(unsigned int cpu)
762 return current->softirqs_raised;
765 #endif /* PREEMPT_RT_FULL */
767 * Enter an interrupt context.
772 if (is_idle_task(current) && !in_interrupt()) {
774 * Prevent raise_softirq from needlessly waking up ksoftirqd
775 * here, as softirq will be serviced on return from interrupt.
777 local_bh_disable_nort();
779 _local_bh_enable_nort();
785 static inline void invoke_softirq(void)
787 #ifndef CONFIG_PREEMPT_RT_FULL
788 if (!force_irqthreads) {
789 #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
791 * We can safely execute softirq on the current stack if
792 * it is the irq stack, because it should be near empty
798 * Otherwise, irq_exit() is called on the task stack that can
799 * be potentially deep already. So call softirq in its own stack
800 * to prevent from any overrun.
802 do_softirq_own_stack();
807 #else /* PREEMPT_RT_FULL */
810 local_irq_save(flags);
811 if (__this_cpu_read(ksoftirqd) &&
812 __this_cpu_read(ksoftirqd)->softirqs_raised)
814 if (__this_cpu_read(ktimer_softirqd) &&
815 __this_cpu_read(ktimer_softirqd)->softirqs_raised)
816 wakeup_timer_softirqd();
817 local_irq_restore(flags);
821 static inline void tick_irq_exit(void)
823 #ifdef CONFIG_NO_HZ_COMMON
824 int cpu = smp_processor_id();
826 /* Make sure that timer wheel updates are propagated */
827 if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
829 tick_nohz_irq_exit();
835 * Exit an interrupt context. Process softirqs if needed and possible:
839 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
842 WARN_ON_ONCE(!irqs_disabled());
845 account_irq_exit_time(current);
846 preempt_count_sub(HARDIRQ_OFFSET);
847 if (!in_interrupt() && local_softirq_pending())
852 trace_hardirq_exit(); /* must be last! */
855 void raise_softirq(unsigned int nr)
859 local_irq_save(flags);
860 raise_softirq_irqoff(nr);
861 local_irq_restore(flags);
864 void open_softirq(int nr, void (*action)(struct softirq_action *))
866 softirq_vec[nr].action = action;
872 struct tasklet_head {
873 struct tasklet_struct *head;
874 struct tasklet_struct **tail;
877 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
878 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
881 __tasklet_common_schedule(struct tasklet_struct *t, struct tasklet_head *head, unsigned int nr)
883 if (tasklet_trylock(t)) {
885 /* We may have been preempted before tasklet_trylock
886 * and __tasklet_action may have already run.
887 * So double check the sched bit while the takslet
888 * is locked before adding it to the list.
890 if (test_bit(TASKLET_STATE_SCHED, &t->state)) {
893 head->tail = &(t->next);
894 raise_softirq_irqoff(nr);
897 /* This is subtle. If we hit the corner case above
898 * It is possible that we get preempted right here,
899 * and another task has successfully called
900 * tasklet_schedule(), then this function, and
901 * failed on the trylock. Thus we must be sure
902 * before releasing the tasklet lock, that the
903 * SCHED_BIT is clear. Otherwise the tasklet
904 * may get its SCHED_BIT set, but not added to the
907 if (!tasklet_tryunlock(t))
913 void __tasklet_schedule(struct tasklet_struct *t)
917 local_irq_save(flags);
918 __tasklet_common_schedule(t, this_cpu_ptr(&tasklet_vec), TASKLET_SOFTIRQ);
919 local_irq_restore(flags);
921 EXPORT_SYMBOL(__tasklet_schedule);
923 void __tasklet_hi_schedule(struct tasklet_struct *t)
927 local_irq_save(flags);
928 __tasklet_common_schedule(t, this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ);
929 local_irq_restore(flags);
931 EXPORT_SYMBOL(__tasklet_hi_schedule);
933 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
935 BUG_ON(!irqs_disabled());
937 __tasklet_hi_schedule(t);
939 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
941 void tasklet_enable(struct tasklet_struct *t)
943 if (!atomic_dec_and_test(&t->count))
945 if (test_and_clear_bit(TASKLET_STATE_PENDING, &t->state))
948 EXPORT_SYMBOL(tasklet_enable);
950 static void __tasklet_action(struct softirq_action *a,
951 struct tasklet_struct *list)
956 struct tasklet_struct *t = list;
961 * Should always succeed - after a tasklist got on the
962 * list (after getting the SCHED bit set from 0 to 1),
963 * nothing but the tasklet softirq it got queued to can
966 if (!tasklet_trylock(t)) {
974 * If we cannot handle the tasklet because it's disabled,
975 * mark it as pending. tasklet_enable() will later
976 * re-schedule the tasklet.
978 if (unlikely(atomic_read(&t->count))) {
980 /* implicit unlock: */
982 t->state = TASKLET_STATEF_PENDING;
987 * After this point on the tasklet might be rescheduled
988 * on another CPU, but it can only be added to another
989 * CPU's tasklet list if we unlock the tasklet (which we
992 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
999 * Try to unlock the tasklet. We must use cmpxchg, because
1000 * another CPU might have scheduled or disabled the tasklet.
1001 * We only allow the STATE_RUN -> 0 transition here.
1003 while (!tasklet_tryunlock(t)) {
1005 * If it got disabled meanwhile, bail out:
1007 if (atomic_read(&t->count))
1010 * If it got scheduled meanwhile, re-execute
1011 * the tasklet function:
1013 if (test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
1016 printk("hm, tasklet state: %08lx\n", t->state);
1025 static void tasklet_action(struct softirq_action *a)
1027 struct tasklet_struct *list;
1029 local_irq_disable();
1031 list = __this_cpu_read(tasklet_vec.head);
1032 __this_cpu_write(tasklet_vec.head, NULL);
1033 __this_cpu_write(tasklet_vec.tail, this_cpu_ptr(&tasklet_vec.head));
1037 __tasklet_action(a, list);
1040 static void tasklet_hi_action(struct softirq_action *a)
1042 struct tasklet_struct *list;
1044 local_irq_disable();
1046 list = __this_cpu_read(tasklet_hi_vec.head);
1047 __this_cpu_write(tasklet_hi_vec.head, NULL);
1048 __this_cpu_write(tasklet_hi_vec.tail, this_cpu_ptr(&tasklet_hi_vec.head));
1052 __tasklet_action(a, list);
1055 void tasklet_init(struct tasklet_struct *t,
1056 void (*func)(unsigned long), unsigned long data)
1060 atomic_set(&t->count, 0);
1064 EXPORT_SYMBOL(tasklet_init);
1066 void tasklet_kill(struct tasklet_struct *t)
1069 pr_notice("Attempt to kill tasklet from interrupt\n");
1071 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
1074 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
1076 tasklet_unlock_wait(t);
1077 clear_bit(TASKLET_STATE_SCHED, &t->state);
1079 EXPORT_SYMBOL(tasklet_kill);
1086 * The trampoline is called when the hrtimer expires. It schedules a tasklet
1087 * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
1088 * hrtimer callback, but from softirq context.
1090 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
1092 struct tasklet_hrtimer *ttimer =
1093 container_of(timer, struct tasklet_hrtimer, timer);
1095 tasklet_hi_schedule(&ttimer->tasklet);
1096 return HRTIMER_NORESTART;
1100 * Helper function which calls the hrtimer callback from
1101 * tasklet/softirq context
1103 static void __tasklet_hrtimer_trampoline(unsigned long data)
1105 struct tasklet_hrtimer *ttimer = (void *)data;
1106 enum hrtimer_restart restart;
1108 restart = ttimer->function(&ttimer->timer);
1109 if (restart != HRTIMER_NORESTART)
1110 hrtimer_restart(&ttimer->timer);
1114 * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
1115 * @ttimer: tasklet_hrtimer which is initialized
1116 * @function: hrtimer callback function which gets called from softirq context
1117 * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
1118 * @mode: hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
1120 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
1121 enum hrtimer_restart (*function)(struct hrtimer *),
1122 clockid_t which_clock, enum hrtimer_mode mode)
1124 hrtimer_init(&ttimer->timer, which_clock, mode);
1125 ttimer->timer.function = __hrtimer_tasklet_trampoline;
1126 tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
1127 (unsigned long)ttimer);
1128 ttimer->function = function;
1130 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
1132 void __init softirq_init(void)
1136 for_each_possible_cpu(cpu) {
1137 per_cpu(tasklet_vec, cpu).tail =
1138 &per_cpu(tasklet_vec, cpu).head;
1139 per_cpu(tasklet_hi_vec, cpu).tail =
1140 &per_cpu(tasklet_hi_vec, cpu).head;
1143 open_softirq(TASKLET_SOFTIRQ, tasklet_action);
1144 open_softirq(HI_SOFTIRQ, tasklet_hi_action);
1147 #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL)
1148 void tasklet_unlock_wait(struct tasklet_struct *t)
1150 while (test_bit(TASKLET_STATE_RUN, &(t)->state)) {
1152 * Hack for now to avoid this busy-loop:
1154 #ifdef CONFIG_PREEMPT_RT_FULL
1161 EXPORT_SYMBOL(tasklet_unlock_wait);
1164 static int ksoftirqd_should_run(unsigned int cpu)
1166 return ksoftirqd_softirq_pending();
1169 #ifdef CONFIG_HOTPLUG_CPU
1171 * tasklet_kill_immediate is called to remove a tasklet which can already be
1172 * scheduled for execution on @cpu.
1174 * Unlike tasklet_kill, this function removes the tasklet
1175 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
1177 * When this function is called, @cpu must be in the CPU_DEAD state.
1179 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
1181 struct tasklet_struct **i;
1183 BUG_ON(cpu_online(cpu));
1184 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
1186 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
1189 /* CPU is dead, so no lock needed. */
1190 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
1193 /* If this was the tail element, move the tail ptr */
1195 per_cpu(tasklet_vec, cpu).tail = i;
1202 static void takeover_tasklets(unsigned int cpu)
1204 /* CPU is dead, so no lock needed. */
1205 local_irq_disable();
1207 /* Find end, append list for that CPU. */
1208 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
1209 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
1210 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
1211 per_cpu(tasklet_vec, cpu).head = NULL;
1212 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
1214 raise_softirq_irqoff(TASKLET_SOFTIRQ);
1216 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
1217 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
1218 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
1219 per_cpu(tasklet_hi_vec, cpu).head = NULL;
1220 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
1222 raise_softirq_irqoff(HI_SOFTIRQ);
1226 #endif /* CONFIG_HOTPLUG_CPU */
1228 static int cpu_callback(struct notifier_block *nfb, unsigned long action,
1232 #ifdef CONFIG_HOTPLUG_CPU
1234 case CPU_DEAD_FROZEN:
1235 takeover_tasklets((unsigned long)hcpu);
1237 #endif /* CONFIG_HOTPLUG_CPU */
1242 static struct notifier_block cpu_nfb = {
1243 .notifier_call = cpu_callback
1246 static struct smp_hotplug_thread softirq_threads = {
1247 .store = &ksoftirqd,
1248 .setup = ksoftirqd_set_sched_params,
1249 .thread_should_run = ksoftirqd_should_run,
1250 .thread_fn = run_ksoftirqd,
1251 .thread_comm = "ksoftirqd/%u",
1254 #ifdef CONFIG_PREEMPT_RT_FULL
1255 static struct smp_hotplug_thread softirq_timer_threads = {
1256 .store = &ktimer_softirqd,
1257 .setup = ktimer_softirqd_set_sched_params,
1258 .cleanup = ktimer_softirqd_clr_sched_params,
1259 .thread_should_run = ktimer_softirqd_should_run,
1260 .thread_fn = run_ksoftirqd,
1261 .thread_comm = "ktimersoftd/%u",
1265 static __init int spawn_ksoftirqd(void)
1267 register_cpu_notifier(&cpu_nfb);
1269 BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
1270 #ifdef CONFIG_PREEMPT_RT_FULL
1271 BUG_ON(smpboot_register_percpu_thread(&softirq_timer_threads));
1276 early_initcall(spawn_ksoftirqd);
1279 * [ These __weak aliases are kept in a separate compilation unit, so that
1280 * GCC does not inline them incorrectly. ]
1283 int __init __weak early_irq_init(void)
1288 int __init __weak arch_probe_nr_irqs(void)
1290 return NR_IRQS_LEGACY;
1293 int __init __weak arch_early_irq_init(void)
1298 unsigned int __weak arch_dynirq_lower_bound(unsigned int from)