kernel/kernel/irq/manage.c

   1 /*
   2  * linux/kernel/irq/manage.c
   3  *
   4  * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
   5  * Copyright (C) 2005-2006 Thomas Gleixner
   6  *
   7  * This file contains driver APIs to the irq subsystem.
   8  */
   9
  10 #define pr_fmt(fmt) "genirq: " fmt
  11
  12 #include <linux/irq.h>
  13 #include <linux/kthread.h>
  14 #include <linux/module.h>
  15 #include <linux/random.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/slab.h>
  18 #include <linux/sched.h>
  19 #include <linux/sched/rt.h>
  20 #include <linux/task_work.h>
  21
  22 #include "internals.h"
  23
  24 #ifdef CONFIG_IRQ_FORCED_THREADING
  25 # ifndef CONFIG_PREEMPT_RT_BASE
  26 __read_mostly bool force_irqthreads;
  27
  28 static int __init setup_forced_irqthreads(char *arg)
  29 {
  30         force_irqthreads = true;
  31         return 0;
  32 }
  33 early_param("threadirqs", setup_forced_irqthreads);
  34 # endif
  35 #endif
  36
  37 static void __synchronize_hardirq(struct irq_desc *desc)
  38 {
  39         bool inprogress;
  40
  41         do {
  42                 unsigned long flags;
  43
  44                 /*
  45                  * Wait until we're out of the critical section.  This might
  46                  * give the wrong answer due to the lack of memory barriers.
  47                  */
  48                 while (irqd_irq_inprogress(&desc->irq_data))
  49                         cpu_relax();
  50
  51                 /* Ok, that indicated we're done: double-check carefully. */
  52                 raw_spin_lock_irqsave(&desc->lock, flags);
  53                 inprogress = irqd_irq_inprogress(&desc->irq_data);
  54                 raw_spin_unlock_irqrestore(&desc->lock, flags);
  55
  56                 /* Oops, that failed? */
  57         } while (inprogress);
  58 }
  59
  60 /**
  61  *      synchronize_hardirq - wait for pending hard IRQ handlers (on other CPUs)
  62  *      @irq: interrupt number to wait for
  63  *
  64  *      This function waits for any pending hard IRQ handlers for this
  65  *      interrupt to complete before returning. If you use this
  66  *      function while holding a resource the IRQ handler may need you
  67  *      will deadlock. It does not take associated threaded handlers
  68  *      into account.
  69  *
  70  *      Do not use this for shutdown scenarios where you must be sure
  71  *      that all parts (hardirq and threaded handler) have completed.
  72  *
  73  *      Returns: false if a threaded handler is active.
  74  *
  75  *      This function may be called - with care - from IRQ context.
  76  */
  77 bool synchronize_hardirq(unsigned int irq)
  78 {
  79         struct irq_desc *desc = irq_to_desc(irq);
  80
  81         if (desc) {
  82                 __synchronize_hardirq(desc);
  83                 return !atomic_read(&desc->threads_active);
  84         }
  85
  86         return true;
  87 }
  88 EXPORT_SYMBOL(synchronize_hardirq);
  89
  90 /**
  91  *      synchronize_irq - wait for pending IRQ handlers (on other CPUs)
  92  *      @irq: interrupt number to wait for
  93  *
  94  *      This function waits for any pending IRQ handlers for this interrupt
  95  *      to complete before returning. If you use this function while
  96  *      holding a resource the IRQ handler may need you will deadlock.
  97  *
  98  *      This function may be called - with care - from IRQ context.
  99  */
 100 void synchronize_irq(unsigned int irq)
 101 {
 102         struct irq_desc *desc = irq_to_desc(irq);
 103
 104         if (desc) {
 105                 __synchronize_hardirq(desc);
 106                 /*
 107                  * We made sure that no hardirq handler is
 108                  * running. Now verify that no threaded handlers are
 109                  * active.
 110                  */
 111                 wait_event(desc->wait_for_threads,
 112                            !atomic_read(&desc->threads_active));
 113         }
 114 }
 115 EXPORT_SYMBOL(synchronize_irq);
 116
 117 #ifdef CONFIG_SMP
 118 cpumask_var_t irq_default_affinity;
 119
 120 /**
 121  *      irq_can_set_affinity - Check if the affinity of a given irq can be set
 122  *      @irq:           Interrupt to check
 123  *
 124  */
 125 int irq_can_set_affinity(unsigned int irq)
 126 {
 127         struct irq_desc *desc = irq_to_desc(irq);
 128
 129         if (!desc || !irqd_can_balance(&desc->irq_data) ||
 130             !desc->irq_data.chip || !desc->irq_data.chip->irq_set_affinity)
 131                 return 0;
 132
 133         return 1;
 134 }
 135
 136 /**
 137  *      irq_set_thread_affinity - Notify irq threads to adjust affinity
 138  *      @desc:          irq descriptor which has affitnity changed
 139  *
 140  *      We just set IRQTF_AFFINITY and delegate the affinity setting
 141  *      to the interrupt thread itself. We can not call
 142  *      set_cpus_allowed_ptr() here as we hold desc->lock and this
 143  *      code can be called from hard interrupt context.
 144  */
 145 void irq_set_thread_affinity(struct irq_desc *desc)
 146 {
 147         struct irqaction *action = desc->action;
 148
 149         while (action) {
 150                 if (action->thread)
 151                         set_bit(IRQTF_AFFINITY, &action->thread_flags);
 152                 action = action->next;
 153         }
 154 }
 155
 156 #ifdef CONFIG_GENERIC_PENDING_IRQ
 157 static inline bool irq_can_move_pcntxt(struct irq_data *data)
 158 {
 159         return irqd_can_move_in_process_context(data);
 160 }
 161 static inline bool irq_move_pending(struct irq_data *data)
 162 {
 163         return irqd_is_setaffinity_pending(data);
 164 }
 165 static inline void
 166 irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
 167 {
 168         cpumask_copy(desc->pending_mask, mask);
 169 }
 170 static inline void
 171 irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
 172 {
 173         cpumask_copy(mask, desc->pending_mask);
 174 }
 175 #else
 176 static inline bool irq_can_move_pcntxt(struct irq_data *data) { return true; }
 177 static inline bool irq_move_pending(struct irq_data *data) { return false; }
 178 static inline void
 179 irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask) { }
 180 static inline void
 181 irq_get_pending(struct cpumask *mask, struct irq_desc *desc) { }
 182 #endif
 183
 184 #ifdef CONFIG_PREEMPT_RT_FULL
 185 static void _irq_affinity_notify(struct irq_affinity_notify *notify);
 186 static struct task_struct *set_affinity_helper;
 187 static LIST_HEAD(affinity_list);
 188 static DEFINE_RAW_SPINLOCK(affinity_list_lock);
 189
 190 static int set_affinity_thread(void *unused)
 191 {
 192         while (1) {
 193                 struct irq_affinity_notify *notify;
 194                 int empty;
 195
 196                 set_current_state(TASK_INTERRUPTIBLE);
 197
 198                 raw_spin_lock_irq(&affinity_list_lock);
 199                 empty = list_empty(&affinity_list);
 200                 raw_spin_unlock_irq(&affinity_list_lock);
 201
 202                 if (empty)
 203                         schedule();
 204                 if (kthread_should_stop())
 205                         break;
 206                 set_current_state(TASK_RUNNING);
 207 try_next:
 208                 notify = NULL;
 209
 210                 raw_spin_lock_irq(&affinity_list_lock);
 211                 if (!list_empty(&affinity_list)) {
 212                         notify = list_first_entry(&affinity_list,
 213                                         struct irq_affinity_notify, list);
 214                         list_del_init(&notify->list);
 215                 }
 216                 raw_spin_unlock_irq(&affinity_list_lock);
 217
 218                 if (!notify)
 219                         continue;
 220                 _irq_affinity_notify(notify);
 221                 goto try_next;
 222         }
 223         return 0;
 224 }
 225
 226 static void init_helper_thread(void)
 227 {
 228         if (set_affinity_helper)
 229                 return;
 230         set_affinity_helper = kthread_run(set_affinity_thread, NULL,
 231                         "affinity-cb");
 232         WARN_ON(IS_ERR(set_affinity_helper));
 233 }
 234 #else
 235
 236 static inline void init_helper_thread(void) { }
 237
 238 #endif
 239
 240 int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
 241                         bool force)
 242 {
 243         struct irq_desc *desc = irq_data_to_desc(data);
 244         struct irq_chip *chip = irq_data_get_irq_chip(data);
 245         int ret;
 246
 247         ret = chip->irq_set_affinity(data, mask, force);
 248         switch (ret) {
 249         case IRQ_SET_MASK_OK:
 250         case IRQ_SET_MASK_OK_DONE:
 251                 cpumask_copy(data->affinity, mask);
 252         case IRQ_SET_MASK_OK_NOCOPY:
 253                 irq_set_thread_affinity(desc);
 254                 ret = 0;
 255         }
 256
 257         return ret;
 258 }
 259
 260 int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask,
 261                             bool force)
 262 {
 263         struct irq_chip *chip = irq_data_get_irq_chip(data);
 264         struct irq_desc *desc = irq_data_to_desc(data);
 265         int ret = 0;
 266
 267         if (!chip || !chip->irq_set_affinity)
 268                 return -EINVAL;
 269
 270         if (irq_can_move_pcntxt(data)) {
 271                 ret = irq_do_set_affinity(data, mask, force);
 272         } else {
 273                 irqd_set_move_pending(data);
 274                 irq_copy_pending(desc, mask);
 275         }
 276
 277         if (desc->affinity_notify) {
 278                 kref_get(&desc->affinity_notify->kref);
 279
 280 #ifdef CONFIG_PREEMPT_RT_FULL
 281                 raw_spin_lock(&affinity_list_lock);
 282                 if (list_empty(&desc->affinity_notify->list))
 283                         list_add_tail(&affinity_list,
 284                                         &desc->affinity_notify->list);
 285                 raw_spin_unlock(&affinity_list_lock);
 286                 wake_up_process(set_affinity_helper);
 287 #else
 288                 schedule_work(&desc->affinity_notify->work);
 289 #endif
 290         }
 291         irqd_set(data, IRQD_AFFINITY_SET);
 292
 293         return ret;
 294 }
 295
 296 int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, bool force)
 297 {
 298         struct irq_desc *desc = irq_to_desc(irq);
 299         unsigned long flags;
 300         int ret;
 301
 302         if (!desc)
 303                 return -EINVAL;
 304
 305         raw_spin_lock_irqsave(&desc->lock, flags);
 306         ret = irq_set_affinity_locked(irq_desc_get_irq_data(desc), mask, force);
 307         raw_spin_unlock_irqrestore(&desc->lock, flags);
 308         return ret;
 309 }
 310
 311 int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m)
 312 {
 313         unsigned long flags;
 314         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
 315
 316         if (!desc)
 317                 return -EINVAL;
 318         desc->affinity_hint = m;
 319         irq_put_desc_unlock(desc, flags);
 320         /* set the initial affinity to prevent every interrupt being on CPU0 */
 321         if (m)
 322                 __irq_set_affinity(irq, m, false);
 323         return 0;
 324 }
 325 EXPORT_SYMBOL_GPL(irq_set_affinity_hint);
 326
 327 static void _irq_affinity_notify(struct irq_affinity_notify *notify)
 328 {
 329         struct irq_desc *desc = irq_to_desc(notify->irq);
 330         cpumask_var_t cpumask;
 331         unsigned long flags;
 332
 333         if (!desc || !alloc_cpumask_var(&cpumask, GFP_KERNEL))
 334                 goto out;
 335
 336         raw_spin_lock_irqsave(&desc->lock, flags);
 337         if (irq_move_pending(&desc->irq_data))
 338                 irq_get_pending(cpumask, desc);
 339         else
 340                 cpumask_copy(cpumask, desc->irq_data.affinity);
 341         raw_spin_unlock_irqrestore(&desc->lock, flags);
 342
 343         notify->notify(notify, cpumask);
 344
 345         free_cpumask_var(cpumask);
 346 out:
 347         kref_put(&notify->kref, notify->release);
 348 }
 349
 350 static void irq_affinity_notify(struct work_struct *work)
 351 {
 352         struct irq_affinity_notify *notify =
 353                 container_of(work, struct irq_affinity_notify, work);
 354         _irq_affinity_notify(notify);
 355 }
 356
 357 /**
 358  *      irq_set_affinity_notifier - control notification of IRQ affinity changes
 359  *      @irq:           Interrupt for which to enable/disable notification
 360  *      @notify:        Context for notification, or %NULL to disable
 361  *                      notification.  Function pointers must be initialised;
 362  *                      the other fields will be initialised by this function.
 363  *
 364  *      Must be called in process context.  Notification may only be enabled
 365  *      after the IRQ is allocated and must be disabled before the IRQ is
 366  *      freed using free_irq().
 367  */
 368 int
 369 irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify)
 370 {
 371         struct irq_desc *desc = irq_to_desc(irq);
 372         struct irq_affinity_notify *old_notify;
 373         unsigned long flags;
 374
 375         /* The release function is promised process context */
 376         might_sleep();
 377
 378         if (!desc)
 379                 return -EINVAL;
 380
 381         /* Complete initialisation of *notify */
 382         if (notify) {
 383                 notify->irq = irq;
 384                 kref_init(&notify->kref);
 385                 INIT_WORK(&notify->work, irq_affinity_notify);
 386                 INIT_LIST_HEAD(&notify->list);
 387                 init_helper_thread();
 388         }
 389
 390         raw_spin_lock_irqsave(&desc->lock, flags);
 391         old_notify = desc->affinity_notify;
 392         desc->affinity_notify = notify;
 393         raw_spin_unlock_irqrestore(&desc->lock, flags);
 394
 395         if (old_notify)
 396                 kref_put(&old_notify->kref, old_notify->release);
 397
 398         return 0;
 399 }
 400 EXPORT_SYMBOL_GPL(irq_set_affinity_notifier);
 401
 402 #ifndef CONFIG_AUTO_IRQ_AFFINITY
 403 /*
 404  * Generic version of the affinity autoselector.
 405  */
 406 static int
 407 setup_affinity(unsigned int irq, struct irq_desc *desc, struct cpumask *mask)
 408 {
 409         struct cpumask *set = irq_default_affinity;
 410         int node = desc->irq_data.node;
 411
 412         /* Excludes PER_CPU and NO_BALANCE interrupts */
 413         if (!irq_can_set_affinity(irq))
 414                 return 0;
 415
 416         /*
 417          * Preserve an userspace affinity setup, but make sure that
 418          * one of the targets is online.
 419          */
 420         if (irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) {
 421                 if (cpumask_intersects(desc->irq_data.affinity,
 422                                        cpu_online_mask))
 423                         set = desc->irq_data.affinity;
 424                 else
 425                         irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET);
 426         }
 427
 428         cpumask_and(mask, cpu_online_mask, set);
 429         if (node != NUMA_NO_NODE) {
 430                 const struct cpumask *nodemask = cpumask_of_node(node);
 431
 432                 /* make sure at least one of the cpus in nodemask is online */
 433                 if (cpumask_intersects(mask, nodemask))
 434                         cpumask_and(mask, mask, nodemask);
 435         }
 436         irq_do_set_affinity(&desc->irq_data, mask, false);
 437         return 0;
 438 }
 439 #else
 440 static inline int
 441 setup_affinity(unsigned int irq, struct irq_desc *d, struct cpumask *mask)
 442 {
 443         return irq_select_affinity(irq);
 444 }
 445 #endif
 446
 447 /*
 448  * Called when affinity is set via /proc/irq
 449  */
 450 int irq_select_affinity_usr(unsigned int irq, struct cpumask *mask)
 451 {
 452         struct irq_desc *desc = irq_to_desc(irq);
 453         unsigned long flags;
 454         int ret;
 455
 456         raw_spin_lock_irqsave(&desc->lock, flags);
 457         ret = setup_affinity(irq, desc, mask);
 458         raw_spin_unlock_irqrestore(&desc->lock, flags);
 459         return ret;
 460 }
 461
 462 #else
 463 static inline int
 464 setup_affinity(unsigned int irq, struct irq_desc *desc, struct cpumask *mask)
 465 {
 466         return 0;
 467 }
 468 #endif
 469
 470 void __disable_irq(struct irq_desc *desc, unsigned int irq)
 471 {
 472         if (!desc->depth++)
 473                 irq_disable(desc);
 474 }
 475
 476 static int __disable_irq_nosync(unsigned int irq)
 477 {
 478         unsigned long flags;
 479         struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
 480
 481         if (!desc)
 482                 return -EINVAL;
 483         __disable_irq(desc, irq);
 484         irq_put_desc_busunlock(desc, flags);
 485         return 0;
 486 }
 487
 488 /**
 489  *      disable_irq_nosync - disable an irq without waiting
 490  *      @irq: Interrupt to disable
 491  *
 492  *      Disable the selected interrupt line.  Disables and Enables are
 493  *      nested.
 494  *      Unlike disable_irq(), this function does not ensure existing
 495  *      instances of the IRQ handler have completed before returning.
 496  *
 497  *      This function may be called from IRQ context.
 498  */
 499 void disable_irq_nosync(unsigned int irq)
 500 {
 501         __disable_irq_nosync(irq);
 502 }
 503 EXPORT_SYMBOL(disable_irq_nosync);
 504
 505 /**
 506  *      disable_irq - disable an irq and wait for completion
 507  *      @irq: Interrupt to disable
 508  *
 509  *      Disable the selected interrupt line.  Enables and Disables are
 510  *      nested.
 511  *      This function waits for any pending IRQ handlers for this interrupt
 512  *      to complete before returning. If you use this function while
 513  *      holding a resource the IRQ handler may need you will deadlock.
 514  *
 515  *      This function may be called - with care - from IRQ context.
 516  */
 517 void disable_irq(unsigned int irq)
 518 {
 519         if (!__disable_irq_nosync(irq))
 520                 synchronize_irq(irq);
 521 }
 522 EXPORT_SYMBOL(disable_irq);
 523
 524 /**
 525  *      disable_hardirq - disables an irq and waits for hardirq completion
 526  *      @irq: Interrupt to disable
 527  *
 528  *      Disable the selected interrupt line.  Enables and Disables are
 529  *      nested.
 530  *      This function waits for any pending hard IRQ handlers for this
 531  *      interrupt to complete before returning. If you use this function while
 532  *      holding a resource the hard IRQ handler may need you will deadlock.
 533  *
 534  *      When used to optimistically disable an interrupt from atomic context
 535  *      the return value must be checked.
 536  *
 537  *      Returns: false if a threaded handler is active.
 538  *
 539  *      This function may be called - with care - from IRQ context.
 540  */
 541 bool disable_hardirq(unsigned int irq)
 542 {
 543         if (!__disable_irq_nosync(irq))
 544                 return synchronize_hardirq(irq);
 545
 546         return false;
 547 }
 548 EXPORT_SYMBOL_GPL(disable_hardirq);
 549
 550 void __enable_irq(struct irq_desc *desc, unsigned int irq)
 551 {
 552         switch (desc->depth) {
 553         case 0:
 554  err_out:
 555                 WARN(1, KERN_WARNING "Unbalanced enable for IRQ %d\n", irq);
 556                 break;
 557         case 1: {
 558                 if (desc->istate & IRQS_SUSPENDED)
 559                         goto err_out;
 560                 /* Prevent probing on this irq: */
 561                 irq_settings_set_noprobe(desc);
 562                 irq_enable(desc);
 563                 check_irq_resend(desc, irq);
 564                 /* fall-through */
 565         }
 566         default:
 567                 desc->depth--;
 568         }
 569 }
 570
 571 /**
 572  *      enable_irq - enable handling of an irq
 573  *      @irq: Interrupt to enable
 574  *
 575  *      Undoes the effect of one call to disable_irq().  If this
 576  *      matches the last disable, processing of interrupts on this
 577  *      IRQ line is re-enabled.
 578  *
 579  *      This function may be called from IRQ context only when
 580  *      desc->irq_data.chip->bus_lock and desc->chip->bus_sync_unlock are NULL !
 581  */
 582 void enable_irq(unsigned int irq)
 583 {
 584         unsigned long flags;
 585         struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
 586
 587         if (!desc)
 588                 return;
 589         if (WARN(!desc->irq_data.chip,
 590                  KERN_ERR "enable_irq before setup/request_irq: irq %u\n", irq))
 591                 goto out;
 592
 593         __enable_irq(desc, irq);
 594 out:
 595         irq_put_desc_busunlock(desc, flags);
 596 }
 597 EXPORT_SYMBOL(enable_irq);
 598
 599 static int set_irq_wake_real(unsigned int irq, unsigned int on)
 600 {
 601         struct irq_desc *desc = irq_to_desc(irq);
 602         int ret = -ENXIO;
 603
 604         if (irq_desc_get_chip(desc)->flags &  IRQCHIP_SKIP_SET_WAKE)
 605                 return 0;
 606
 607         if (desc->irq_data.chip->irq_set_wake)
 608                 ret = desc->irq_data.chip->irq_set_wake(&desc->irq_data, on);
 609
 610         return ret;
 611 }
 612
 613 /**
 614  *      irq_set_irq_wake - control irq power management wakeup
 615  *      @irq:   interrupt to control
 616  *      @on:    enable/disable power management wakeup
 617  *
 618  *      Enable/disable power management wakeup mode, which is
 619  *      disabled by default.  Enables and disables must match,
 620  *      just as they match for non-wakeup mode support.
 621  *
 622  *      Wakeup mode lets this IRQ wake the system from sleep
 623  *      states like "suspend to RAM".
 624  */
 625 int irq_set_irq_wake(unsigned int irq, unsigned int on)
 626 {
 627         unsigned long flags;
 628         struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
 629         int ret = 0;
 630
 631         if (!desc)
 632                 return -EINVAL;
 633
 634         /* wakeup-capable irqs can be shared between drivers that
 635          * don't need to have the same sleep mode behaviors.
 636          */
 637         if (on) {
 638                 if (desc->wake_depth++ == 0) {
 639                         ret = set_irq_wake_real(irq, on);
 640                         if (ret)
 641                                 desc->wake_depth = 0;
 642                         else
 643                                 irqd_set(&desc->irq_data, IRQD_WAKEUP_STATE);
 644                 }
 645         } else {
 646                 if (desc->wake_depth == 0) {
 647                         WARN(1, "Unbalanced IRQ %d wake disable\n", irq);
 648                 } else if (--desc->wake_depth == 0) {
 649                         ret = set_irq_wake_real(irq, on);
 650                         if (ret)
 651                                 desc->wake_depth = 1;
 652                         else
 653                                 irqd_clear(&desc->irq_data, IRQD_WAKEUP_STATE);
 654                 }
 655         }
 656         irq_put_desc_busunlock(desc, flags);
 657         return ret;
 658 }
 659 EXPORT_SYMBOL(irq_set_irq_wake);
 660
 661 /*
 662  * Internal function that tells the architecture code whether a
 663  * particular irq has been exclusively allocated or is available
 664  * for driver use.
 665  */
 666 int can_request_irq(unsigned int irq, unsigned long irqflags)
 667 {
 668         unsigned long flags;
 669         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
 670         int canrequest = 0;
 671
 672         if (!desc)
 673                 return 0;
 674
 675         if (irq_settings_can_request(desc)) {
 676                 if (!desc->action ||
 677                     irqflags & desc->action->flags & IRQF_SHARED)
 678                         canrequest = 1;
 679         }
 680         irq_put_desc_unlock(desc, flags);
 681         return canrequest;
 682 }
 683
 684 int __irq_set_trigger(struct irq_desc *desc, unsigned int irq,
 685                       unsigned long flags)
 686 {
 687         struct irq_chip *chip = desc->irq_data.chip;
 688         int ret, unmask = 0;
 689
 690         if (!chip || !chip->irq_set_type) {
 691                 /*
 692                  * IRQF_TRIGGER_* but the PIC does not support multiple
 693                  * flow-types?
 694                  */
 695                 pr_debug("No set_type function for IRQ %d (%s)\n", irq,
 696                          chip ? (chip->name ? : "unknown") : "unknown");
 697                 return 0;
 698         }
 699
 700         flags &= IRQ_TYPE_SENSE_MASK;
 701
 702         if (chip->flags & IRQCHIP_SET_TYPE_MASKED) {
 703                 if (!irqd_irq_masked(&desc->irq_data))
 704                         mask_irq(desc);
 705                 if (!irqd_irq_disabled(&desc->irq_data))
 706                         unmask = 1;
 707         }
 708
 709         /* caller masked out all except trigger mode flags */
 710         ret = chip->irq_set_type(&desc->irq_data, flags);
 711
 712         switch (ret) {
 713         case IRQ_SET_MASK_OK:
 714         case IRQ_SET_MASK_OK_DONE:
 715                 irqd_clear(&desc->irq_data, IRQD_TRIGGER_MASK);
 716                 irqd_set(&desc->irq_data, flags);
 717
 718         case IRQ_SET_MASK_OK_NOCOPY:
 719                 flags = irqd_get_trigger_type(&desc->irq_data);
 720                 irq_settings_set_trigger_mask(desc, flags);
 721                 irqd_clear(&desc->irq_data, IRQD_LEVEL);
 722                 irq_settings_clr_level(desc);
 723                 if (flags & IRQ_TYPE_LEVEL_MASK) {
 724                         irq_settings_set_level(desc);
 725                         irqd_set(&desc->irq_data, IRQD_LEVEL);
 726                 }
 727
 728                 ret = 0;
 729                 break;
 730         default:
 731                 pr_err("Setting trigger mode %lu for irq %u failed (%pF)\n",
 732                        flags, irq, chip->irq_set_type);
 733         }
 734         if (unmask)
 735                 unmask_irq(desc);
 736         return ret;
 737 }
 738
 739 #ifdef CONFIG_HARDIRQS_SW_RESEND
 740 int irq_set_parent(int irq, int parent_irq)
 741 {
 742         unsigned long flags;
 743         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
 744
 745         if (!desc)
 746                 return -EINVAL;
 747
 748         desc->parent_irq = parent_irq;
 749
 750         irq_put_desc_unlock(desc, flags);
 751         return 0;
 752 }
 753 #endif
 754
 755 /*
 756  * Default primary interrupt handler for threaded interrupts. Is
 757  * assigned as primary handler when request_threaded_irq is called
 758  * with handler == NULL. Useful for oneshot interrupts.
 759  */
 760 static irqreturn_t irq_default_primary_handler(int irq, void *dev_id)
 761 {
 762         return IRQ_WAKE_THREAD;
 763 }
 764
 765 /*
 766  * Primary handler for nested threaded interrupts. Should never be
 767  * called.
 768  */
 769 static irqreturn_t irq_nested_primary_handler(int irq, void *dev_id)
 770 {
 771         WARN(1, "Primary handler called for nested irq %d\n", irq);
 772         return IRQ_NONE;
 773 }
 774
 775 static int irq_wait_for_interrupt(struct irqaction *action)
 776 {
 777         set_current_state(TASK_INTERRUPTIBLE);
 778
 779         while (!kthread_should_stop()) {
 780
 781                 if (test_and_clear_bit(IRQTF_RUNTHREAD,
 782                                        &action->thread_flags)) {
 783                         __set_current_state(TASK_RUNNING);
 784                         return 0;
 785                 }
 786                 schedule();
 787                 set_current_state(TASK_INTERRUPTIBLE);
 788         }
 789         __set_current_state(TASK_RUNNING);
 790         return -1;
 791 }
 792
 793 /*
 794  * Oneshot interrupts keep the irq line masked until the threaded
 795  * handler finished. unmask if the interrupt has not been disabled and
 796  * is marked MASKED.
 797  */
 798 static void irq_finalize_oneshot(struct irq_desc *desc,
 799                                  struct irqaction *action)
 800 {
 801         if (!(desc->istate & IRQS_ONESHOT))
 802                 return;
 803 again:
 804         chip_bus_lock(desc);
 805         raw_spin_lock_irq(&desc->lock);
 806
 807         /*
 808          * Implausible though it may be we need to protect us against
 809          * the following scenario:
 810          *
 811          * The thread is faster done than the hard interrupt handler
 812          * on the other CPU. If we unmask the irq line then the
 813          * interrupt can come in again and masks the line, leaves due
 814          * to IRQS_INPROGRESS and the irq line is masked forever.
 815          *
 816          * This also serializes the state of shared oneshot handlers
 817          * versus "desc->threads_onehsot |= action->thread_mask;" in
 818          * irq_wake_thread(). See the comment there which explains the
 819          * serialization.
 820          */
 821         if (unlikely(irqd_irq_inprogress(&desc->irq_data))) {
 822                 raw_spin_unlock_irq(&desc->lock);
 823                 chip_bus_sync_unlock(desc);
 824                 cpu_relax();
 825                 goto again;
 826         }
 827
 828         /*
 829          * Now check again, whether the thread should run. Otherwise
 830          * we would clear the threads_oneshot bit of this thread which
 831          * was just set.
 832          */
 833         if (test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
 834                 goto out_unlock;
 835
 836         desc->threads_oneshot &= ~action->thread_mask;
 837
 838         if (!desc->threads_oneshot && !irqd_irq_disabled(&desc->irq_data) &&
 839             irqd_irq_masked(&desc->irq_data))
 840                 unmask_threaded_irq(desc);
 841
 842 out_unlock:
 843         raw_spin_unlock_irq(&desc->lock);
 844         chip_bus_sync_unlock(desc);
 845 }
 846
 847 #ifdef CONFIG_SMP
 848 /*
 849  * Check whether we need to change the affinity of the interrupt thread.
 850  */
 851 static void
 852 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action)
 853 {
 854         cpumask_var_t mask;
 855         bool valid = true;
 856
 857         if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags))
 858                 return;
 859
 860         /*
 861          * In case we are out of memory we set IRQTF_AFFINITY again and
 862          * try again next time
 863          */
 864         if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
 865                 set_bit(IRQTF_AFFINITY, &action->thread_flags);
 866                 return;
 867         }
 868
 869         raw_spin_lock_irq(&desc->lock);
 870         /*
 871          * This code is triggered unconditionally. Check the affinity
 872          * mask pointer. For CPU_MASK_OFFSTACK=n this is optimized out.
 873          */
 874         if (desc->irq_data.affinity)
 875                 cpumask_copy(mask, desc->irq_data.affinity);
 876         else
 877                 valid = false;
 878         raw_spin_unlock_irq(&desc->lock);
 879
 880         if (valid)
 881                 set_cpus_allowed_ptr(current, mask);
 882         free_cpumask_var(mask);
 883 }
 884 #else
 885 static inline void
 886 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action) { }
 887 #endif
 888
 889 /*
 890  * Interrupts which are not explicitely requested as threaded
 891  * interrupts rely on the implicit bh/preempt disable of the hard irq
 892  * context. So we need to disable bh here to avoid deadlocks and other
 893  * side effects.
 894  */
 895 static irqreturn_t
 896 irq_forced_thread_fn(struct irq_desc *desc, struct irqaction *action)
 897 {
 898         irqreturn_t ret;
 899
 900         local_bh_disable();
 901         ret = action->thread_fn(action->irq, action->dev_id);
 902         irq_finalize_oneshot(desc, action);
 903         /*
 904          * Interrupts which have real time requirements can be set up
 905          * to avoid softirq processing in the thread handler. This is
 906          * safe as these interrupts do not raise soft interrupts.
 907          */
 908         if (irq_settings_no_softirq_call(desc))
 909                 _local_bh_enable();
 910         else
 911                 local_bh_enable();
 912         return ret;
 913 }
 914
 915 /*
 916  * Interrupts explicitly requested as threaded interrupts want to be
 917  * preemtible - many of them need to sleep and wait for slow busses to
 918  * complete.
 919  */
 920 static irqreturn_t irq_thread_fn(struct irq_desc *desc,
 921                 struct irqaction *action)
 922 {
 923         irqreturn_t ret;
 924
 925         ret = action->thread_fn(action->irq, action->dev_id);
 926         irq_finalize_oneshot(desc, action);
 927         return ret;
 928 }
 929
 930 static void wake_threads_waitq(struct irq_desc *desc)
 931 {
 932         if (atomic_dec_and_test(&desc->threads_active))
 933                 wake_up(&desc->wait_for_threads);
 934 }
 935
 936 static void irq_thread_dtor(struct callback_head *unused)
 937 {
 938         struct task_struct *tsk = current;
 939         struct irq_desc *desc;
 940         struct irqaction *action;
 941
 942         if (WARN_ON_ONCE(!(current->flags & PF_EXITING)))
 943                 return;
 944
 945         action = kthread_data(tsk);
 946
 947         pr_err("exiting task \"%s\" (%d) is an active IRQ thread (irq %d)\n",
 948                tsk->comm, tsk->pid, action->irq);
 949
 950
 951         desc = irq_to_desc(action->irq);
 952         /*
 953          * If IRQTF_RUNTHREAD is set, we need to decrement
 954          * desc->threads_active and wake possible waiters.
 955          */
 956         if (test_and_clear_bit(IRQTF_RUNTHREAD, &action->thread_flags))
 957                 wake_threads_waitq(desc);
 958
 959         /* Prevent a stale desc->threads_oneshot */
 960         irq_finalize_oneshot(desc, action);
 961 }
 962
 963 /*
 964  * Interrupt handler thread
 965  */
 966 static int irq_thread(void *data)
 967 {
 968         struct callback_head on_exit_work;
 969         struct irqaction *action = data;
 970         struct irq_desc *desc = irq_to_desc(action->irq);
 971         irqreturn_t (*handler_fn)(struct irq_desc *desc,
 972                         struct irqaction *action);
 973
 974         if (force_irqthreads && test_bit(IRQTF_FORCED_THREAD,
 975                                         &action->thread_flags))
 976                 handler_fn = irq_forced_thread_fn;
 977         else
 978                 handler_fn = irq_thread_fn;
 979
 980         init_task_work(&on_exit_work, irq_thread_dtor);
 981         task_work_add(current, &on_exit_work, false);
 982
 983         irq_thread_check_affinity(desc, action);
 984
 985         while (!irq_wait_for_interrupt(action)) {
 986                 irqreturn_t action_ret;
 987
 988                 irq_thread_check_affinity(desc, action);
 989
 990                 action_ret = handler_fn(desc, action);
 991                 if (action_ret == IRQ_HANDLED)
 992                         atomic_inc(&desc->threads_handled);
 993
 994 #ifdef CONFIG_PREEMPT_RT_FULL
 995                 migrate_disable();
 996                 add_interrupt_randomness(action->irq, 0,
 997                                  desc->random_ip ^ (unsigned long) action);
 998                 migrate_enable();
 999 #endif
1000                 wake_threads_waitq(desc);
1001         }
1002
1003         /*
1004          * This is the regular exit path. __free_irq() is stopping the
1005          * thread via kthread_stop() after calling
1006          * synchronize_irq(). So neither IRQTF_RUNTHREAD nor the
1007          * oneshot mask bit can be set. We cannot verify that as we
1008          * cannot touch the oneshot mask at this point anymore as
1009          * __setup_irq() might have given out currents thread_mask
1010          * again.
1011          */
1012         task_work_cancel(current, irq_thread_dtor);
1013         return 0;
1014 }
1015
1016 /**
1017  *      irq_wake_thread - wake the irq thread for the action identified by dev_id
1018  *      @irq:           Interrupt line
1019  *      @dev_id:        Device identity for which the thread should be woken
1020  *
1021  */
1022 void irq_wake_thread(unsigned int irq, void *dev_id)
1023 {
1024         struct irq_desc *desc = irq_to_desc(irq);
1025         struct irqaction *action;
1026         unsigned long flags;
1027
1028         if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1029                 return;
1030
1031         raw_spin_lock_irqsave(&desc->lock, flags);
1032         for (action = desc->action; action; action = action->next) {
1033                 if (action->dev_id == dev_id) {
1034                         if (action->thread)
1035                                 __irq_wake_thread(desc, action);
1036                         break;
1037                 }
1038         }
1039         raw_spin_unlock_irqrestore(&desc->lock, flags);
1040 }
1041 EXPORT_SYMBOL_GPL(irq_wake_thread);
1042
1043 static void irq_setup_forced_threading(struct irqaction *new)
1044 {
1045         if (!force_irqthreads)
1046                 return;
1047         if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))
1048                 return;
1049
1050         new->flags |= IRQF_ONESHOT;
1051
1052         if (!new->thread_fn) {
1053                 set_bit(IRQTF_FORCED_THREAD, &new->thread_flags);
1054                 new->thread_fn = new->handler;
1055                 new->handler = irq_default_primary_handler;
1056         }
1057 }
1058
1059 static int irq_request_resources(struct irq_desc *desc)
1060 {
1061         struct irq_data *d = &desc->irq_data;
1062         struct irq_chip *c = d->chip;
1063
1064         return c->irq_request_resources ? c->irq_request_resources(d) : 0;
1065 }
1066
1067 static void irq_release_resources(struct irq_desc *desc)
1068 {
1069         struct irq_data *d = &desc->irq_data;
1070         struct irq_chip *c = d->chip;
1071
1072         if (c->irq_release_resources)
1073                 c->irq_release_resources(d);
1074 }
1075
1076 /*
1077  * Internal function to register an irqaction - typically used to
1078  * allocate special interrupts that are part of the architecture.
1079  */
1080 static int
1081 __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
1082 {
1083         struct irqaction *old, **old_ptr;
1084         unsigned long flags, thread_mask = 0;
1085         int ret, nested, shared = 0;
1086         cpumask_var_t mask;
1087
1088         if (!desc)
1089                 return -EINVAL;
1090
1091         if (desc->irq_data.chip == &no_irq_chip)
1092                 return -ENOSYS;
1093         if (!try_module_get(desc->owner))
1094                 return -ENODEV;
1095
1096         /*
1097          * Check whether the interrupt nests into another interrupt
1098          * thread.
1099          */
1100         nested = irq_settings_is_nested_thread(desc);
1101         if (nested) {
1102                 if (!new->thread_fn) {
1103                         ret = -EINVAL;
1104                         goto out_mput;
1105                 }
1106                 /*
1107                  * Replace the primary handler which was provided from
1108                  * the driver for non nested interrupt handling by the
1109                  * dummy function which warns when called.
1110                  */
1111                 new->handler = irq_nested_primary_handler;
1112         } else {
1113                 if (irq_settings_can_thread(desc))
1114                         irq_setup_forced_threading(new);
1115         }
1116
1117         /*
1118          * Create a handler thread when a thread function is supplied
1119          * and the interrupt does not nest into another interrupt
1120          * thread.
1121          */
1122         if (new->thread_fn && !nested) {
1123                 struct task_struct *t;
1124                 static const struct sched_param param = {
1125                         .sched_priority = MAX_USER_RT_PRIO/2,
1126                 };
1127
1128                 t = kthread_create(irq_thread, new, "irq/%d-%s", irq,
1129                                    new->name);
1130                 if (IS_ERR(t)) {
1131                         ret = PTR_ERR(t);
1132                         goto out_mput;
1133                 }
1134
1135                 sched_setscheduler_nocheck(t, SCHED_FIFO, &param);
1136
1137                 /*
1138                  * We keep the reference to the task struct even if
1139                  * the thread dies to avoid that the interrupt code
1140                  * references an already freed task_struct.
1141                  */
1142                 get_task_struct(t);
1143                 new->thread = t;
1144                 /*
1145                  * Tell the thread to set its affinity. This is
1146                  * important for shared interrupt handlers as we do
1147                  * not invoke setup_affinity() for the secondary
1148                  * handlers as everything is already set up. Even for
1149                  * interrupts marked with IRQF_NO_BALANCE this is
1150                  * correct as we want the thread to move to the cpu(s)
1151                  * on which the requesting code placed the interrupt.
1152                  */
1153                 set_bit(IRQTF_AFFINITY, &new->thread_flags);
1154         }
1155
1156         if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
1157                 ret = -ENOMEM;
1158                 goto out_thread;
1159         }
1160
1161         /*
1162          * Drivers are often written to work w/o knowledge about the
1163          * underlying irq chip implementation, so a request for a
1164          * threaded irq without a primary hard irq context handler
1165          * requires the ONESHOT flag to be set. Some irq chips like
1166          * MSI based interrupts are per se one shot safe. Check the
1167          * chip flags, so we can avoid the unmask dance at the end of
1168          * the threaded handler for those.
1169          */
1170         if (desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)
1171                 new->flags &= ~IRQF_ONESHOT;
1172
1173         /*
1174          * The following block of code has to be executed atomically
1175          */
1176         raw_spin_lock_irqsave(&desc->lock, flags);
1177         old_ptr = &desc->action;
1178         old = *old_ptr;
1179         if (old) {
1180                 /*
1181                  * Can't share interrupts unless both agree to and are
1182                  * the same type (level, edge, polarity). So both flag
1183                  * fields must have IRQF_SHARED set and the bits which
1184                  * set the trigger type must match. Also all must
1185                  * agree on ONESHOT.
1186                  */
1187                 if (!((old->flags & new->flags) & IRQF_SHARED) ||
1188                     ((old->flags ^ new->flags) & IRQF_TRIGGER_MASK) ||
1189                     ((old->flags ^ new->flags) & IRQF_ONESHOT))
1190                         goto mismatch;
1191
1192                 /* All handlers must agree on per-cpuness */
1193                 if ((old->flags & IRQF_PERCPU) !=
1194                     (new->flags & IRQF_PERCPU))
1195                         goto mismatch;
1196
1197                 /* add new interrupt at end of irq queue */
1198                 do {
1199                         /*
1200                          * Or all existing action->thread_mask bits,
1201                          * so we can find the next zero bit for this
1202                          * new action.
1203                          */
1204                         thread_mask |= old->thread_mask;
1205                         old_ptr = &old->next;
1206                         old = *old_ptr;
1207                 } while (old);
1208                 shared = 1;
1209         }
1210
1211         /*
1212          * Setup the thread mask for this irqaction for ONESHOT. For
1213          * !ONESHOT irqs the thread mask is 0 so we can avoid a
1214          * conditional in irq_wake_thread().
1215          */
1216         if (new->flags & IRQF_ONESHOT) {
1217                 /*
1218                  * Unlikely to have 32 resp 64 irqs sharing one line,
1219                  * but who knows.
1220                  */
1221                 if (thread_mask == ~0UL) {
1222                         ret = -EBUSY;
1223                         goto out_mask;
1224                 }
1225                 /*
1226                  * The thread_mask for the action is or'ed to
1227                  * desc->thread_active to indicate that the
1228                  * IRQF_ONESHOT thread handler has been woken, but not
1229                  * yet finished. The bit is cleared when a thread
1230                  * completes. When all threads of a shared interrupt
1231                  * line have completed desc->threads_active becomes
1232                  * zero and the interrupt line is unmasked. See
1233                  * handle.c:irq_wake_thread() for further information.
1234                  *
1235                  * If no thread is woken by primary (hard irq context)
1236                  * interrupt handlers, then desc->threads_active is
1237                  * also checked for zero to unmask the irq line in the
1238                  * affected hard irq flow handlers
1239                  * (handle_[fasteoi|level]_irq).
1240                  *
1241                  * The new action gets the first zero bit of
1242                  * thread_mask assigned. See the loop above which or's
1243                  * all existing action->thread_mask bits.
1244                  */
1245                 new->thread_mask = 1 << ffz(thread_mask);
1246
1247         } else if (new->handler == irq_default_primary_handler &&
1248                    !(desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)) {
1249                 /*
1250                  * The interrupt was requested with handler = NULL, so
1251                  * we use the default primary handler for it. But it
1252                  * does not have the oneshot flag set. In combination
1253                  * with level interrupts this is deadly, because the
1254                  * default primary handler just wakes the thread, then
1255                  * the irq lines is reenabled, but the device still
1256                  * has the level irq asserted. Rinse and repeat....
1257                  *
1258                  * While this works for edge type interrupts, we play
1259                  * it safe and reject unconditionally because we can't
1260                  * say for sure which type this interrupt really
1261                  * has. The type flags are unreliable as the
1262                  * underlying chip implementation can override them.
1263                  */
1264                 pr_err("Threaded irq requested with handler=NULL and !ONESHOT for irq %d\n",
1265                        irq);
1266                 ret = -EINVAL;
1267                 goto out_mask;
1268         }
1269
1270         if (!shared) {
1271                 ret = irq_request_resources(desc);
1272                 if (ret) {
1273                         pr_err("Failed to request resources for %s (irq %d) on irqchip %s\n",
1274                                new->name, irq, desc->irq_data.chip->name);
1275                         goto out_mask;
1276                 }
1277
1278                 init_waitqueue_head(&desc->wait_for_threads);
1279
1280                 /* Setup the type (level, edge polarity) if configured: */
1281                 if (new->flags & IRQF_TRIGGER_MASK) {
1282                         ret = __irq_set_trigger(desc, irq,
1283                                         new->flags & IRQF_TRIGGER_MASK);
1284
1285                         if (ret)
1286                                 goto out_mask;
1287                 }
1288
1289                 desc->istate &= ~(IRQS_AUTODETECT | IRQS_SPURIOUS_DISABLED | \
1290                                   IRQS_ONESHOT | IRQS_WAITING);
1291                 irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
1292
1293                 if (new->flags & IRQF_PERCPU) {
1294                         irqd_set(&desc->irq_data, IRQD_PER_CPU);
1295                         irq_settings_set_per_cpu(desc);
1296                 }
1297
1298                 if (new->flags & IRQF_ONESHOT)
1299                         desc->istate |= IRQS_ONESHOT;
1300
1301                 if (irq_settings_can_autoenable(desc))
1302                         irq_startup(desc, true);
1303                 else
1304                         /* Undo nested disables: */
1305                         desc->depth = 1;
1306
1307                 /* Exclude IRQ from balancing if requested */
1308                 if (new->flags & IRQF_NOBALANCING) {
1309                         irq_settings_set_no_balancing(desc);
1310                         irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
1311                 }
1312
1313                 if (new->flags & IRQF_NO_SOFTIRQ_CALL)
1314                         irq_settings_set_no_softirq_call(desc);
1315
1316                 /* Set default affinity mask once everything is setup */
1317                 setup_affinity(irq, desc, mask);
1318
1319         } else if (new->flags & IRQF_TRIGGER_MASK) {
1320                 unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK;
1321                 unsigned int omsk = irq_settings_get_trigger_mask(desc);
1322
1323                 if (nmsk != omsk)
1324                         /* hope the handler works with current  trigger mode */
1325                         pr_warning("irq %d uses trigger mode %u; requested %u\n",
1326                                    irq, nmsk, omsk);
1327         }
1328
1329         new->irq = irq;
1330         *old_ptr = new;
1331
1332         irq_pm_install_action(desc, new);
1333
1334         /* Reset broken irq detection when installing new handler */
1335         desc->irq_count = 0;
1336         desc->irqs_unhandled = 0;
1337
1338         /*
1339          * Check whether we disabled the irq via the spurious handler
1340          * before. Reenable it and give it another chance.
1341          */
1342         if (shared && (desc->istate & IRQS_SPURIOUS_DISABLED)) {
1343                 desc->istate &= ~IRQS_SPURIOUS_DISABLED;
1344                 __enable_irq(desc, irq);
1345         }
1346
1347         raw_spin_unlock_irqrestore(&desc->lock, flags);
1348
1349         /*
1350          * Strictly no need to wake it up, but hung_task complains
1351          * when no hard interrupt wakes the thread up.
1352          */
1353         if (new->thread)
1354                 wake_up_process(new->thread);
1355
1356         register_irq_proc(irq, desc);
1357         new->dir = NULL;
1358         register_handler_proc(irq, new);
1359         free_cpumask_var(mask);
1360
1361         return 0;
1362
1363 mismatch:
1364         if (!(new->flags & IRQF_PROBE_SHARED)) {
1365                 pr_err("Flags mismatch irq %d. %08x (%s) vs. %08x (%s)\n",
1366                        irq, new->flags, new->name, old->flags, old->name);
1367 #ifdef CONFIG_DEBUG_SHIRQ
1368                 dump_stack();
1369 #endif
1370         }
1371         ret = -EBUSY;
1372
1373 out_mask:
1374         raw_spin_unlock_irqrestore(&desc->lock, flags);
1375         free_cpumask_var(mask);
1376
1377 out_thread:
1378         if (new->thread) {
1379                 struct task_struct *t = new->thread;
1380
1381                 new->thread = NULL;
1382                 kthread_stop(t);
1383                 put_task_struct(t);
1384         }
1385 out_mput:
1386         module_put(desc->owner);
1387         return ret;
1388 }
1389
1390 /**
1391  *      setup_irq - setup an interrupt
1392  *      @irq: Interrupt line to setup
1393  *      @act: irqaction for the interrupt
1394  *
1395  * Used to statically setup interrupts in the early boot process.
1396  */
1397 int setup_irq(unsigned int irq, struct irqaction *act)
1398 {
1399         int retval;
1400         struct irq_desc *desc = irq_to_desc(irq);
1401
1402         if (WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1403                 return -EINVAL;
1404         chip_bus_lock(desc);
1405         retval = __setup_irq(irq, desc, act);
1406         chip_bus_sync_unlock(desc);
1407
1408         return retval;
1409 }
1410 EXPORT_SYMBOL_GPL(setup_irq);
1411
1412 /*
1413  * Internal function to unregister an irqaction - used to free
1414  * regular and special interrupts that are part of the architecture.
1415  */
1416 static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
1417 {
1418         struct irq_desc *desc = irq_to_desc(irq);
1419         struct irqaction *action, **action_ptr;
1420         unsigned long flags;
1421
1422         WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
1423
1424         if (!desc)
1425                 return NULL;
1426
1427         raw_spin_lock_irqsave(&desc->lock, flags);
1428
1429         /*
1430          * There can be multiple actions per IRQ descriptor, find the right
1431          * one based on the dev_id:
1432          */
1433         action_ptr = &desc->action;
1434         for (;;) {
1435                 action = *action_ptr;
1436
1437                 if (!action) {
1438                         WARN(1, "Trying to free already-free IRQ %d\n", irq);
1439                         raw_spin_unlock_irqrestore(&desc->lock, flags);
1440
1441                         return NULL;
1442                 }
1443
1444                 if (action->dev_id == dev_id)
1445                         break;
1446                 action_ptr = &action->next;
1447         }
1448
1449         /* Found it - now remove it from the list of entries: */
1450         *action_ptr = action->next;
1451
1452         irq_pm_remove_action(desc, action);
1453
1454         /* If this was the last handler, shut down the IRQ line: */
1455         if (!desc->action) {
1456                 irq_shutdown(desc);
1457                 irq_release_resources(desc);
1458         }
1459
1460 #ifdef CONFIG_SMP
1461         /* make sure affinity_hint is cleaned up */
1462         if (WARN_ON_ONCE(desc->affinity_hint))
1463                 desc->affinity_hint = NULL;
1464 #endif
1465
1466         raw_spin_unlock_irqrestore(&desc->lock, flags);
1467
1468         unregister_handler_proc(irq, action);
1469
1470         /* Make sure it's not being used on another CPU: */
1471         synchronize_irq(irq);
1472
1473 #ifdef CONFIG_DEBUG_SHIRQ
1474         /*
1475          * It's a shared IRQ -- the driver ought to be prepared for an IRQ
1476          * event to happen even now it's being freed, so let's make sure that
1477          * is so by doing an extra call to the handler ....
1478          *
1479          * ( We do this after actually deregistering it, to make sure that a
1480          *   'real' IRQ doesn't run in * parallel with our fake. )
1481          */
1482         if (action->flags & IRQF_SHARED) {
1483                 local_irq_save(flags);
1484                 action->handler(irq, dev_id);
1485                 local_irq_restore(flags);
1486         }
1487 #endif
1488
1489         if (action->thread) {
1490                 kthread_stop(action->thread);
1491                 put_task_struct(action->thread);
1492         }
1493
1494         module_put(desc->owner);
1495         return action;
1496 }
1497
1498 /**
1499  *      remove_irq - free an interrupt
1500  *      @irq: Interrupt line to free
1501  *      @act: irqaction for the interrupt
1502  *
1503  * Used to remove interrupts statically setup by the early boot process.
1504  */
1505 void remove_irq(unsigned int irq, struct irqaction *act)
1506 {
1507         struct irq_desc *desc = irq_to_desc(irq);
1508
1509         if (desc && !WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1510             __free_irq(irq, act->dev_id);
1511 }
1512 EXPORT_SYMBOL_GPL(remove_irq);
1513
1514 /**
1515  *      free_irq - free an interrupt allocated with request_irq
1516  *      @irq: Interrupt line to free
1517  *      @dev_id: Device identity to free
1518  *
1519  *      Remove an interrupt handler. The handler is removed and if the
1520  *      interrupt line is no longer in use by any driver it is disabled.
1521  *      On a shared IRQ the caller must ensure the interrupt is disabled
1522  *      on the card it drives before calling this function. The function
1523  *      does not return until any executing interrupts for this IRQ
1524  *      have completed.
1525  *
1526  *      This function must not be called from interrupt context.
1527  */
1528 void free_irq(unsigned int irq, void *dev_id)
1529 {
1530         struct irq_desc *desc = irq_to_desc(irq);
1531
1532         if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1533                 return;
1534
1535 #ifdef CONFIG_SMP
1536         if (WARN_ON(desc->affinity_notify))
1537                 desc->affinity_notify = NULL;
1538 #endif
1539
1540         chip_bus_lock(desc);
1541         kfree(__free_irq(irq, dev_id));
1542         chip_bus_sync_unlock(desc);
1543 }
1544 EXPORT_SYMBOL(free_irq);
1545
1546 /**
1547  *      request_threaded_irq - allocate an interrupt line
1548  *      @irq: Interrupt line to allocate
1549  *      @handler: Function to be called when the IRQ occurs.
1550  *                Primary handler for threaded interrupts
1551  *                If NULL and thread_fn != NULL the default
1552  *                primary handler is installed
1553  *      @thread_fn: Function called from the irq handler thread
1554  *                  If NULL, no irq thread is created
1555  *      @irqflags: Interrupt type flags
1556  *      @devname: An ascii name for the claiming device
1557  *      @dev_id: A cookie passed back to the handler function
1558  *
1559  *      This call allocates interrupt resources and enables the
1560  *      interrupt line and IRQ handling. From the point this
1561  *      call is made your handler function may be invoked. Since
1562  *      your handler function must clear any interrupt the board
1563  *      raises, you must take care both to initialise your hardware
1564  *      and to set up the interrupt handler in the right order.
1565  *
1566  *      If you want to set up a threaded irq handler for your device
1567  *      then you need to supply @handler and @thread_fn. @handler is
1568  *      still called in hard interrupt context and has to check
1569  *      whether the interrupt originates from the device. If yes it
1570  *      needs to disable the interrupt on the device and return
1571  *      IRQ_WAKE_THREAD which will wake up the handler thread and run
1572  *      @thread_fn. This split handler design is necessary to support
1573  *      shared interrupts.
1574  *
1575  *      Dev_id must be globally unique. Normally the address of the
1576  *      device data structure is used as the cookie. Since the handler
1577  *      receives this value it makes sense to use it.
1578  *
1579  *      If your interrupt is shared you must pass a non NULL dev_id
1580  *      as this is required when freeing the interrupt.
1581  *
1582  *      Flags:
1583  *
1584  *      IRQF_SHARED             Interrupt is shared
1585  *      IRQF_TRIGGER_*          Specify active edge(s) or level
1586  *
1587  */
1588 int request_threaded_irq(unsigned int irq, irq_handler_t handler,
1589                          irq_handler_t thread_fn, unsigned long irqflags,
1590                          const char *devname, void *dev_id)
1591 {
1592         struct irqaction *action;
1593         struct irq_desc *desc;
1594         int retval;
1595
1596         /*
1597          * Sanity-check: shared interrupts must pass in a real dev-ID,
1598          * otherwise we'll have trouble later trying to figure out
1599          * which interrupt is which (messes up the interrupt freeing
1600          * logic etc).
1601          *
1602          * Also IRQF_COND_SUSPEND only makes sense for shared interrupts and
1603          * it cannot be set along with IRQF_NO_SUSPEND.
1604          */
1605         if (((irqflags & IRQF_SHARED) && !dev_id) ||
1606             (!(irqflags & IRQF_SHARED) && (irqflags & IRQF_COND_SUSPEND)) ||
1607             ((irqflags & IRQF_NO_SUSPEND) && (irqflags & IRQF_COND_SUSPEND)))
1608                 return -EINVAL;
1609
1610         desc = irq_to_desc(irq);
1611         if (!desc)
1612                 return -EINVAL;
1613
1614         if (!irq_settings_can_request(desc) ||
1615             WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1616                 return -EINVAL;
1617
1618         if (!handler) {
1619                 if (!thread_fn)
1620                         return -EINVAL;
1621                 handler = irq_default_primary_handler;
1622         }
1623
1624         action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
1625         if (!action)
1626                 return -ENOMEM;
1627
1628         action->handler = handler;
1629         action->thread_fn = thread_fn;
1630         action->flags = irqflags;
1631         action->name = devname;
1632         action->dev_id = dev_id;
1633
1634         chip_bus_lock(desc);
1635         retval = __setup_irq(irq, desc, action);
1636         chip_bus_sync_unlock(desc);
1637
1638         if (retval)
1639                 kfree(action);
1640
1641 #ifdef CONFIG_DEBUG_SHIRQ_FIXME
1642         if (!retval && (irqflags & IRQF_SHARED)) {
1643                 /*
1644                  * It's a shared IRQ -- the driver ought to be prepared for it
1645                  * to happen immediately, so let's make sure....
1646                  * We disable the irq to make sure that a 'real' IRQ doesn't
1647                  * run in parallel with our fake.
1648                  */
1649                 unsigned long flags;
1650
1651                 disable_irq(irq);
1652                 local_irq_save(flags);
1653
1654                 handler(irq, dev_id);
1655
1656                 local_irq_restore(flags);
1657                 enable_irq(irq);
1658         }
1659 #endif
1660         return retval;
1661 }
1662 EXPORT_SYMBOL(request_threaded_irq);
1663
1664 /**
1665  *      request_any_context_irq - allocate an interrupt line
1666  *      @irq: Interrupt line to allocate
1667  *      @handler: Function to be called when the IRQ occurs.
1668  *                Threaded handler for threaded interrupts.
1669  *      @flags: Interrupt type flags
1670  *      @name: An ascii name for the claiming device
1671  *      @dev_id: A cookie passed back to the handler function
1672  *
1673  *      This call allocates interrupt resources and enables the
1674  *      interrupt line and IRQ handling. It selects either a
1675  *      hardirq or threaded handling method depending on the
1676  *      context.
1677  *
1678  *      On failure, it returns a negative value. On success,
1679  *      it returns either IRQC_IS_HARDIRQ or IRQC_IS_NESTED.
1680  */
1681 int request_any_context_irq(unsigned int irq, irq_handler_t handler,
1682                             unsigned long flags, const char *name, void *dev_id)
1683 {
1684         struct irq_desc *desc = irq_to_desc(irq);
1685         int ret;
1686
1687         if (!desc)
1688                 return -EINVAL;
1689
1690         if (irq_settings_is_nested_thread(desc)) {
1691                 ret = request_threaded_irq(irq, NULL, handler,
1692                                            flags, name, dev_id);
1693                 return !ret ? IRQC_IS_NESTED : ret;
1694         }
1695
1696         ret = request_irq(irq, handler, flags, name, dev_id);
1697         return !ret ? IRQC_IS_HARDIRQ : ret;
1698 }
1699 EXPORT_SYMBOL_GPL(request_any_context_irq);
1700
1701 void enable_percpu_irq(unsigned int irq, unsigned int type)
1702 {
1703         unsigned int cpu = smp_processor_id();
1704         unsigned long flags;
1705         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
1706
1707         if (!desc)
1708                 return;
1709
1710         type &= IRQ_TYPE_SENSE_MASK;
1711         if (type != IRQ_TYPE_NONE) {
1712                 int ret;
1713
1714                 ret = __irq_set_trigger(desc, irq, type);
1715
1716                 if (ret) {
1717                         WARN(1, "failed to set type for IRQ%d\n", irq);
1718                         goto out;
1719                 }
1720         }
1721
1722         irq_percpu_enable(desc, cpu);
1723 out:
1724         irq_put_desc_unlock(desc, flags);
1725 }
1726 EXPORT_SYMBOL_GPL(enable_percpu_irq);
1727
1728 void disable_percpu_irq(unsigned int irq)
1729 {
1730         unsigned int cpu = smp_processor_id();
1731         unsigned long flags;
1732         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
1733
1734         if (!desc)
1735                 return;
1736
1737         irq_percpu_disable(desc, cpu);
1738         irq_put_desc_unlock(desc, flags);
1739 }
1740 EXPORT_SYMBOL_GPL(disable_percpu_irq);
1741
1742 /*
1743  * Internal function to unregister a percpu irqaction.
1744  */
1745 static struct irqaction *__free_percpu_irq(unsigned int irq, void __percpu *dev_id)
1746 {
1747         struct irq_desc *desc = irq_to_desc(irq);
1748         struct irqaction *action;
1749         unsigned long flags;
1750
1751         WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
1752
1753         if (!desc)
1754                 return NULL;
1755
1756         raw_spin_lock_irqsave(&desc->lock, flags);
1757
1758         action = desc->action;
1759         if (!action || action->percpu_dev_id != dev_id) {
1760                 WARN(1, "Trying to free already-free IRQ %d\n", irq);
1761                 goto bad;
1762         }
1763
1764         if (!cpumask_empty(desc->percpu_enabled)) {
1765                 WARN(1, "percpu IRQ %d still enabled on CPU%d!\n",
1766                      irq, cpumask_first(desc->percpu_enabled));
1767                 goto bad;
1768         }
1769
1770         /* Found it - now remove it from the list of entries: */
1771         desc->action = NULL;
1772
1773         raw_spin_unlock_irqrestore(&desc->lock, flags);
1774
1775         unregister_handler_proc(irq, action);
1776
1777         module_put(desc->owner);
1778         return action;
1779
1780 bad:
1781         raw_spin_unlock_irqrestore(&desc->lock, flags);
1782         return NULL;
1783 }
1784
1785 /**
1786  *      remove_percpu_irq - free a per-cpu interrupt
1787  *      @irq: Interrupt line to free
1788  *      @act: irqaction for the interrupt
1789  *
1790  * Used to remove interrupts statically setup by the early boot process.
1791  */
1792 void remove_percpu_irq(unsigned int irq, struct irqaction *act)
1793 {
1794         struct irq_desc *desc = irq_to_desc(irq);
1795
1796         if (desc && irq_settings_is_per_cpu_devid(desc))
1797             __free_percpu_irq(irq, act->percpu_dev_id);
1798 }
1799
1800 /**
1801  *      free_percpu_irq - free an interrupt allocated with request_percpu_irq
1802  *      @irq: Interrupt line to free
1803  *      @dev_id: Device identity to free
1804  *
1805  *      Remove a percpu interrupt handler. The handler is removed, but
1806  *      the interrupt line is not disabled. This must be done on each
1807  *      CPU before calling this function. The function does not return
1808  *      until any executing interrupts for this IRQ have completed.
1809  *
1810  *      This function must not be called from interrupt context.
1811  */
1812 void free_percpu_irq(unsigned int irq, void __percpu *dev_id)
1813 {
1814         struct irq_desc *desc = irq_to_desc(irq);
1815
1816         if (!desc || !irq_settings_is_per_cpu_devid(desc))
1817                 return;
1818
1819         chip_bus_lock(desc);
1820         kfree(__free_percpu_irq(irq, dev_id));
1821         chip_bus_sync_unlock(desc);
1822 }
1823
1824 /**
1825  *      setup_percpu_irq - setup a per-cpu interrupt
1826  *      @irq: Interrupt line to setup
1827  *      @act: irqaction for the interrupt
1828  *
1829  * Used to statically setup per-cpu interrupts in the early boot process.
1830  */
1831 int setup_percpu_irq(unsigned int irq, struct irqaction *act)
1832 {
1833         struct irq_desc *desc = irq_to_desc(irq);
1834         int retval;
1835
1836         if (!desc || !irq_settings_is_per_cpu_devid(desc))
1837                 return -EINVAL;
1838         chip_bus_lock(desc);
1839         retval = __setup_irq(irq, desc, act);
1840         chip_bus_sync_unlock(desc);
1841
1842         return retval;
1843 }
1844
1845 /**
1846  *      request_percpu_irq - allocate a percpu interrupt line
1847  *      @irq: Interrupt line to allocate
1848  *      @handler: Function to be called when the IRQ occurs.
1849  *      @devname: An ascii name for the claiming device
1850  *      @dev_id: A percpu cookie passed back to the handler function
1851  *
1852  *      This call allocates interrupt resources, but doesn't
1853  *      automatically enable the interrupt. It has to be done on each
1854  *      CPU using enable_percpu_irq().
1855  *
1856  *      Dev_id must be globally unique. It is a per-cpu variable, and
1857  *      the handler gets called with the interrupted CPU's instance of
1858  *      that variable.
1859  */
1860 int request_percpu_irq(unsigned int irq, irq_handler_t handler,
1861                        const char *devname, void __percpu *dev_id)
1862 {
1863         struct irqaction *action;
1864         struct irq_desc *desc;
1865         int retval;
1866
1867         if (!dev_id)
1868                 return -EINVAL;
1869
1870         desc = irq_to_desc(irq);
1871         if (!desc || !irq_settings_can_request(desc) ||
1872             !irq_settings_is_per_cpu_devid(desc))
1873                 return -EINVAL;
1874
1875         action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
1876         if (!action)
1877                 return -ENOMEM;
1878
1879         action->handler = handler;
1880         action->flags = IRQF_PERCPU | IRQF_NO_SUSPEND;
1881         action->name = devname;
1882         action->percpu_dev_id = dev_id;
1883
1884         chip_bus_lock(desc);
1885         retval = __setup_irq(irq, desc, action);
1886         chip_bus_sync_unlock(desc);
1887
1888         if (retval)
1889                 kfree(action);
1890
1891         return retval;
1892 }
1893
1894 /**
1895  *      irq_get_irqchip_state - returns the irqchip state of a interrupt.
1896  *      @irq: Interrupt line that is forwarded to a VM
1897  *      @which: One of IRQCHIP_STATE_* the caller wants to know about
1898  *      @state: a pointer to a boolean where the state is to be storeed
1899  *
1900  *      This call snapshots the internal irqchip state of an
1901  *      interrupt, returning into @state the bit corresponding to
1902  *      stage @which
1903  *
1904  *      This function should be called with preemption disabled if the
1905  *      interrupt controller has per-cpu registers.
1906  */
1907 int irq_get_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
1908                           bool *state)
1909 {
1910         struct irq_desc *desc;
1911         struct irq_data *data;
1912         struct irq_chip *chip;
1913         unsigned long flags;
1914         int err = -EINVAL;
1915
1916         desc = irq_get_desc_buslock(irq, &flags, 0);
1917         if (!desc)
1918                 return err;
1919
1920         data = irq_desc_get_irq_data(desc);
1921
1922         do {
1923                 chip = irq_data_get_irq_chip(data);
1924                 if (chip->irq_get_irqchip_state)
1925                         break;
1926 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
1927                 data = data->parent_data;
1928 #else
1929                 data = NULL;
1930 #endif
1931         } while (data);
1932
1933         if (data)
1934                 err = chip->irq_get_irqchip_state(data, which, state);
1935
1936         irq_put_desc_busunlock(desc, flags);
1937         return err;
1938 }
1939
1940 /**
1941  *      irq_set_irqchip_state - set the state of a forwarded interrupt.
1942  *      @irq: Interrupt line that is forwarded to a VM
1943  *      @which: State to be restored (one of IRQCHIP_STATE_*)
1944  *      @val: Value corresponding to @which
1945  *
1946  *      This call sets the internal irqchip state of an interrupt,
1947  *      depending on the value of @which.
1948  *
1949  *      This function should be called with preemption disabled if the
1950  *      interrupt controller has per-cpu registers.
1951  */
1952 int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
1953                           bool val)
1954 {
1955         struct irq_desc *desc;
1956         struct irq_data *data;
1957         struct irq_chip *chip;
1958         unsigned long flags;
1959         int err = -EINVAL;
1960
1961         desc = irq_get_desc_buslock(irq, &flags, 0);
1962         if (!desc)
1963                 return err;
1964
1965         data = irq_desc_get_irq_data(desc);
1966
1967         do {
1968                 chip = irq_data_get_irq_chip(data);
1969                 if (chip->irq_set_irqchip_state)
1970                         break;
1971 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
1972                 data = data->parent_data;
1973 #else
1974                 data = NULL;
1975 #endif
1976         } while (data);
1977
1978         if (data)
1979                 err = chip->irq_set_irqchip_state(data, which, val);
1980
1981         irq_put_desc_busunlock(desc, flags);
1982         return err;
1983 }