kernel/kernel/irq/manage.c

   1 /*
   2  * linux/kernel/irq/manage.c
   3  *
   4  * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
   5  * Copyright (C) 2005-2006 Thomas Gleixner
   6  *
   7  * This file contains driver APIs to the irq subsystem.
   8  */
   9
  10 #define pr_fmt(fmt) "genirq: " fmt
  11
  12 #include <linux/irq.h>
  13 #include <linux/kthread.h>
  14 #include <linux/module.h>
  15 #include <linux/random.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/slab.h>
  18 #include <linux/sched.h>
  19 #include <linux/sched/rt.h>
  20 #include <linux/task_work.h>
  21
  22 #include "internals.h"
  23
  24 #ifdef CONFIG_IRQ_FORCED_THREADING
  25 # ifndef CONFIG_PREEMPT_RT_BASE
  26 __read_mostly bool force_irqthreads;
  27
  28 static int __init setup_forced_irqthreads(char *arg)
  29 {
  30         force_irqthreads = true;
  31         return 0;
  32 }
  33 early_param("threadirqs", setup_forced_irqthreads);
  34 # endif
  35 #endif
  36
  37 static void __synchronize_hardirq(struct irq_desc *desc)
  38 {
  39         bool inprogress;
  40
  41         do {
  42                 unsigned long flags;
  43
  44                 /*
  45                  * Wait until we're out of the critical section.  This might
  46                  * give the wrong answer due to the lack of memory barriers.
  47                  */
  48                 while (irqd_irq_inprogress(&desc->irq_data))
  49                         cpu_relax();
  50
  51                 /* Ok, that indicated we're done: double-check carefully. */
  52                 raw_spin_lock_irqsave(&desc->lock, flags);
  53                 inprogress = irqd_irq_inprogress(&desc->irq_data);
  54                 raw_spin_unlock_irqrestore(&desc->lock, flags);
  55
  56                 /* Oops, that failed? */
  57         } while (inprogress);
  58 }
  59
  60 /**
  61  *      synchronize_hardirq - wait for pending hard IRQ handlers (on other CPUs)
  62  *      @irq: interrupt number to wait for
  63  *
  64  *      This function waits for any pending hard IRQ handlers for this
  65  *      interrupt to complete before returning. If you use this
  66  *      function while holding a resource the IRQ handler may need you
  67  *      will deadlock. It does not take associated threaded handlers
  68  *      into account.
  69  *
  70  *      Do not use this for shutdown scenarios where you must be sure
  71  *      that all parts (hardirq and threaded handler) have completed.
  72  *
  73  *      Returns: false if a threaded handler is active.
  74  *
  75  *      This function may be called - with care - from IRQ context.
  76  */
  77 bool synchronize_hardirq(unsigned int irq)
  78 {
  79         struct irq_desc *desc = irq_to_desc(irq);
  80
  81         if (desc) {
  82                 __synchronize_hardirq(desc);
  83                 return !atomic_read(&desc->threads_active);
  84         }
  85
  86         return true;
  87 }
  88 EXPORT_SYMBOL(synchronize_hardirq);
  89
  90 /**
  91  *      synchronize_irq - wait for pending IRQ handlers (on other CPUs)
  92  *      @irq: interrupt number to wait for
  93  *
  94  *      This function waits for any pending IRQ handlers for this interrupt
  95  *      to complete before returning. If you use this function while
  96  *      holding a resource the IRQ handler may need you will deadlock.
  97  *
  98  *      This function may be called - with care - from IRQ context.
  99  */
 100 void synchronize_irq(unsigned int irq)
 101 {
 102         struct irq_desc *desc = irq_to_desc(irq);
 103
 104         if (desc) {
 105                 __synchronize_hardirq(desc);
 106                 /*
 107                  * We made sure that no hardirq handler is
 108                  * running. Now verify that no threaded handlers are
 109                  * active.
 110                  */
 111                 wait_event(desc->wait_for_threads,
 112                            !atomic_read(&desc->threads_active));
 113         }
 114 }
 115 EXPORT_SYMBOL(synchronize_irq);
 116
 117 #ifdef CONFIG_SMP
 118 cpumask_var_t irq_default_affinity;
 119
 120 /**
 121  *      irq_can_set_affinity - Check if the affinity of a given irq can be set
 122  *      @irq:           Interrupt to check
 123  *
 124  */
 125 int irq_can_set_affinity(unsigned int irq)
 126 {
 127         struct irq_desc *desc = irq_to_desc(irq);
 128
 129         if (!desc || !irqd_can_balance(&desc->irq_data) ||
 130             !desc->irq_data.chip || !desc->irq_data.chip->irq_set_affinity)
 131                 return 0;
 132
 133         return 1;
 134 }
 135
 136 /**
 137  *      irq_set_thread_affinity - Notify irq threads to adjust affinity
 138  *      @desc:          irq descriptor which has affitnity changed
 139  *
 140  *      We just set IRQTF_AFFINITY and delegate the affinity setting
 141  *      to the interrupt thread itself. We can not call
 142  *      set_cpus_allowed_ptr() here as we hold desc->lock and this
 143  *      code can be called from hard interrupt context.
 144  */
 145 void irq_set_thread_affinity(struct irq_desc *desc)
 146 {
 147         struct irqaction *action = desc->action;
 148
 149         while (action) {
 150                 if (action->thread)
 151                         set_bit(IRQTF_AFFINITY, &action->thread_flags);
 152                 action = action->next;
 153         }
 154 }
 155
 156 #ifdef CONFIG_GENERIC_PENDING_IRQ
 157 static inline bool irq_can_move_pcntxt(struct irq_data *data)
 158 {
 159         return irqd_can_move_in_process_context(data);
 160 }
 161 static inline bool irq_move_pending(struct irq_data *data)
 162 {
 163         return irqd_is_setaffinity_pending(data);
 164 }
 165 static inline void
 166 irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
 167 {
 168         cpumask_copy(desc->pending_mask, mask);
 169 }
 170 static inline void
 171 irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
 172 {
 173         cpumask_copy(mask, desc->pending_mask);
 174 }
 175 #else
 176 static inline bool irq_can_move_pcntxt(struct irq_data *data) { return true; }
 177 static inline bool irq_move_pending(struct irq_data *data) { return false; }
 178 static inline void
 179 irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask) { }
 180 static inline void
 181 irq_get_pending(struct cpumask *mask, struct irq_desc *desc) { }
 182 #endif
 183
 184 #ifdef CONFIG_PREEMPT_RT_FULL
 185 static void _irq_affinity_notify(struct irq_affinity_notify *notify);
 186 static struct task_struct *set_affinity_helper;
 187 static LIST_HEAD(affinity_list);
 188 static DEFINE_RAW_SPINLOCK(affinity_list_lock);
 189
 190 static int set_affinity_thread(void *unused)
 191 {
 192         while (1) {
 193                 struct irq_affinity_notify *notify;
 194                 int empty;
 195
 196                 set_current_state(TASK_INTERRUPTIBLE);
 197
 198                 raw_spin_lock_irq(&affinity_list_lock);
 199                 empty = list_empty(&affinity_list);
 200                 raw_spin_unlock_irq(&affinity_list_lock);
 201
 202                 if (empty)
 203                         schedule();
 204                 if (kthread_should_stop())
 205                         break;
 206                 set_current_state(TASK_RUNNING);
 207 try_next:
 208                 notify = NULL;
 209
 210                 raw_spin_lock_irq(&affinity_list_lock);
 211                 if (!list_empty(&affinity_list)) {
 212                         notify = list_first_entry(&affinity_list,
 213                                         struct irq_affinity_notify, list);
 214                         list_del_init(&notify->list);
 215                 }
 216                 raw_spin_unlock_irq(&affinity_list_lock);
 217
 218                 if (!notify)
 219                         continue;
 220                 _irq_affinity_notify(notify);
 221                 goto try_next;
 222         }
 223         return 0;
 224 }
 225
 226 static void init_helper_thread(void)
 227 {
 228         if (set_affinity_helper)
 229                 return;
 230         set_affinity_helper = kthread_run(set_affinity_thread, NULL,
 231                         "affinity-cb");
 232         WARN_ON(IS_ERR(set_affinity_helper));
 233 }
 234 #else
 235
 236 static inline void init_helper_thread(void) { }
 237
 238 #endif
 239
 240 int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
 241                         bool force)
 242 {
 243         struct irq_desc *desc = irq_data_to_desc(data);
 244         struct irq_chip *chip = irq_data_get_irq_chip(data);
 245         int ret;
 246
 247         ret = chip->irq_set_affinity(data, mask, force);
 248         switch (ret) {
 249         case IRQ_SET_MASK_OK:
 250         case IRQ_SET_MASK_OK_DONE:
 251                 cpumask_copy(data->affinity, mask);
 252         case IRQ_SET_MASK_OK_NOCOPY:
 253                 irq_set_thread_affinity(desc);
 254                 ret = 0;
 255         }
 256
 257         return ret;
 258 }
 259
 260 int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask,
 261                             bool force)
 262 {
 263         struct irq_chip *chip = irq_data_get_irq_chip(data);
 264         struct irq_desc *desc = irq_data_to_desc(data);
 265         int ret = 0;
 266
 267         if (!chip || !chip->irq_set_affinity)
 268                 return -EINVAL;
 269
 270         if (irq_can_move_pcntxt(data)) {
 271                 ret = irq_do_set_affinity(data, mask, force);
 272         } else {
 273                 irqd_set_move_pending(data);
 274                 irq_copy_pending(desc, mask);
 275         }
 276
 277         if (desc->affinity_notify) {
 278                 kref_get(&desc->affinity_notify->kref);
 279
 280 #ifdef CONFIG_PREEMPT_RT_FULL
 281                 raw_spin_lock(&affinity_list_lock);
 282                 if (list_empty(&desc->affinity_notify->list))
 283                         list_add_tail(&affinity_list,
 284                                         &desc->affinity_notify->list);
 285                 raw_spin_unlock(&affinity_list_lock);
 286                 wake_up_process(set_affinity_helper);
 287 #else
 288                 schedule_work(&desc->affinity_notify->work);
 289 #endif
 290         }
 291         irqd_set(data, IRQD_AFFINITY_SET);
 292
 293         return ret;
 294 }
 295
 296 int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, bool force)
 297 {
 298         struct irq_desc *desc = irq_to_desc(irq);
 299         unsigned long flags;
 300         int ret;
 301
 302         if (!desc)
 303                 return -EINVAL;
 304
 305         raw_spin_lock_irqsave(&desc->lock, flags);
 306         ret = irq_set_affinity_locked(irq_desc_get_irq_data(desc), mask, force);
 307         raw_spin_unlock_irqrestore(&desc->lock, flags);
 308         return ret;
 309 }
 310
 311 int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m)
 312 {
 313         unsigned long flags;
 314         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
 315
 316         if (!desc)
 317                 return -EINVAL;
 318         desc->affinity_hint = m;
 319         irq_put_desc_unlock(desc, flags);
 320         /* set the initial affinity to prevent every interrupt being on CPU0 */
 321         if (m)
 322                 __irq_set_affinity(irq, m, false);
 323         return 0;
 324 }
 325 EXPORT_SYMBOL_GPL(irq_set_affinity_hint);
 326
 327 static void _irq_affinity_notify(struct irq_affinity_notify *notify)
 328 {
 329         struct irq_desc *desc = irq_to_desc(notify->irq);
 330         cpumask_var_t cpumask;
 331         unsigned long flags;
 332
 333         if (!desc || !alloc_cpumask_var(&cpumask, GFP_KERNEL))
 334                 goto out;
 335
 336         raw_spin_lock_irqsave(&desc->lock, flags);
 337         if (irq_move_pending(&desc->irq_data))
 338                 irq_get_pending(cpumask, desc);
 339         else
 340                 cpumask_copy(cpumask, desc->irq_data.affinity);
 341         raw_spin_unlock_irqrestore(&desc->lock, flags);
 342
 343         notify->notify(notify, cpumask);
 344
 345         free_cpumask_var(cpumask);
 346 out:
 347         kref_put(&notify->kref, notify->release);
 348 }
 349
 350 static void irq_affinity_notify(struct work_struct *work)
 351 {
 352         struct irq_affinity_notify *notify =
 353                 container_of(work, struct irq_affinity_notify, work);
 354         _irq_affinity_notify(notify);
 355 }
 356
 357 /**
 358  *      irq_set_affinity_notifier - control notification of IRQ affinity changes
 359  *      @irq:           Interrupt for which to enable/disable notification
 360  *      @notify:        Context for notification, or %NULL to disable
 361  *                      notification.  Function pointers must be initialised;
 362  *                      the other fields will be initialised by this function.
 363  *
 364  *      Must be called in process context.  Notification may only be enabled
 365  *      after the IRQ is allocated and must be disabled before the IRQ is
 366  *      freed using free_irq().
 367  */
 368 int
 369 irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify)
 370 {
 371         struct irq_desc *desc = irq_to_desc(irq);
 372         struct irq_affinity_notify *old_notify;
 373         unsigned long flags;
 374
 375         /* The release function is promised process context */
 376         might_sleep();
 377
 378         if (!desc)
 379                 return -EINVAL;
 380
 381         /* Complete initialisation of *notify */
 382         if (notify) {
 383                 notify->irq = irq;
 384                 kref_init(&notify->kref);
 385                 INIT_WORK(&notify->work, irq_affinity_notify);
 386                 INIT_LIST_HEAD(&notify->list);
 387                 init_helper_thread();
 388         }
 389
 390         raw_spin_lock_irqsave(&desc->lock, flags);
 391         old_notify = desc->affinity_notify;
 392         desc->affinity_notify = notify;
 393         raw_spin_unlock_irqrestore(&desc->lock, flags);
 394
 395         if (old_notify)
 396                 kref_put(&old_notify->kref, old_notify->release);
 397
 398         return 0;
 399 }
 400 EXPORT_SYMBOL_GPL(irq_set_affinity_notifier);
 401
 402 #ifndef CONFIG_AUTO_IRQ_AFFINITY
 403 /*
 404  * Generic version of the affinity autoselector.
 405  */
 406 static int
 407 setup_affinity(unsigned int irq, struct irq_desc *desc, struct cpumask *mask)
 408 {
 409         struct cpumask *set = irq_default_affinity;
 410         int node = desc->irq_data.node;
 411
 412         /* Excludes PER_CPU and NO_BALANCE interrupts */
 413         if (!irq_can_set_affinity(irq))
 414                 return 0;
 415
 416         /*
 417          * Preserve an userspace affinity setup, but make sure that
 418          * one of the targets is online.
 419          */
 420         if (irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) {
 421                 if (cpumask_intersects(desc->irq_data.affinity,
 422                                        cpu_online_mask))
 423                         set = desc->irq_data.affinity;
 424                 else
 425                         irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET);
 426         }
 427
 428         cpumask_and(mask, cpu_online_mask, set);
 429         if (node != NUMA_NO_NODE) {
 430                 const struct cpumask *nodemask = cpumask_of_node(node);
 431
 432                 /* make sure at least one of the cpus in nodemask is online */
 433                 if (cpumask_intersects(mask, nodemask))
 434                         cpumask_and(mask, mask, nodemask);
 435         }
 436         irq_do_set_affinity(&desc->irq_data, mask, false);
 437         return 0;
 438 }
 439 #else
 440 static inline int
 441 setup_affinity(unsigned int irq, struct irq_desc *d, struct cpumask *mask)
 442 {
 443         return irq_select_affinity(irq);
 444 }
 445 #endif
 446
 447 /*
 448  * Called when affinity is set via /proc/irq
 449  */
 450 int irq_select_affinity_usr(unsigned int irq, struct cpumask *mask)
 451 {
 452         struct irq_desc *desc = irq_to_desc(irq);
 453         unsigned long flags;
 454         int ret;
 455
 456         raw_spin_lock_irqsave(&desc->lock, flags);
 457         ret = setup_affinity(irq, desc, mask);
 458         raw_spin_unlock_irqrestore(&desc->lock, flags);
 459         return ret;
 460 }
 461
 462 #else
 463 static inline int
 464 setup_affinity(unsigned int irq, struct irq_desc *desc, struct cpumask *mask)
 465 {
 466         return 0;
 467 }
 468 #endif
 469
 470 void __disable_irq(struct irq_desc *desc, unsigned int irq)
 471 {
 472         if (!desc->depth++)
 473                 irq_disable(desc);
 474 }
 475
 476 static int __disable_irq_nosync(unsigned int irq)
 477 {
 478         unsigned long flags;
 479         struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
 480
 481         if (!desc)
 482                 return -EINVAL;
 483         __disable_irq(desc, irq);
 484         irq_put_desc_busunlock(desc, flags);
 485         return 0;
 486 }
 487
 488 /**
 489  *      disable_irq_nosync - disable an irq without waiting
 490  *      @irq: Interrupt to disable
 491  *
 492  *      Disable the selected interrupt line.  Disables and Enables are
 493  *      nested.
 494  *      Unlike disable_irq(), this function does not ensure existing
 495  *      instances of the IRQ handler have completed before returning.
 496  *
 497  *      This function may be called from IRQ context.
 498  */
 499 void disable_irq_nosync(unsigned int irq)
 500 {
 501         __disable_irq_nosync(irq);
 502 }
 503 EXPORT_SYMBOL(disable_irq_nosync);
 504
 505 /**
 506  *      disable_irq - disable an irq and wait for completion
 507  *      @irq: Interrupt to disable
 508  *
 509  *      Disable the selected interrupt line.  Enables and Disables are
 510  *      nested.
 511  *      This function waits for any pending IRQ handlers for this interrupt
 512  *      to complete before returning. If you use this function while
 513  *      holding a resource the IRQ handler may need you will deadlock.
 514  *
 515  *      This function may be called - with care - from IRQ context.
 516  */
 517 void disable_irq(unsigned int irq)
 518 {
 519         if (!__disable_irq_nosync(irq))
 520                 synchronize_irq(irq);
 521 }
 522 EXPORT_SYMBOL(disable_irq);
 523
 524 /**
 525  *      disable_hardirq - disables an irq and waits for hardirq completion
 526  *      @irq: Interrupt to disable
 527  *
 528  *      Disable the selected interrupt line.  Enables and Disables are
 529  *      nested.
 530  *      This function waits for any pending hard IRQ handlers for this
 531  *      interrupt to complete before returning. If you use this function while
 532  *      holding a resource the hard IRQ handler may need you will deadlock.
 533  *
 534  *      When used to optimistically disable an interrupt from atomic context
 535  *      the return value must be checked.
 536  *
 537  *      Returns: false if a threaded handler is active.
 538  *
 539  *      This function may be called - with care - from IRQ context.
 540  */
 541 bool disable_hardirq(unsigned int irq)
 542 {
 543         if (!__disable_irq_nosync(irq))
 544                 return synchronize_hardirq(irq);
 545
 546         return false;
 547 }
 548 EXPORT_SYMBOL_GPL(disable_hardirq);
 549
 550 void __enable_irq(struct irq_desc *desc, unsigned int irq)
 551 {
 552         switch (desc->depth) {
 553         case 0:
 554  err_out:
 555                 WARN(1, KERN_WARNING "Unbalanced enable for IRQ %d\n", irq);
 556                 break;
 557         case 1: {
 558                 if (desc->istate & IRQS_SUSPENDED)
 559                         goto err_out;
 560                 /* Prevent probing on this irq: */
 561                 irq_settings_set_noprobe(desc);
 562                 irq_enable(desc);
 563                 check_irq_resend(desc, irq);
 564                 /* fall-through */
 565         }
 566         default:
 567                 desc->depth--;
 568         }
 569 }
 570
 571 /**
 572  *      enable_irq - enable handling of an irq
 573  *      @irq: Interrupt to enable
 574  *
 575  *      Undoes the effect of one call to disable_irq().  If this
 576  *      matches the last disable, processing of interrupts on this
 577  *      IRQ line is re-enabled.
 578  *
 579  *      This function may be called from IRQ context only when
 580  *      desc->irq_data.chip->bus_lock and desc->chip->bus_sync_unlock are NULL !
 581  */
 582 void enable_irq(unsigned int irq)
 583 {
 584         unsigned long flags;
 585         struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
 586
 587         if (!desc)
 588                 return;
 589         if (WARN(!desc->irq_data.chip,
 590                  KERN_ERR "enable_irq before setup/request_irq: irq %u\n", irq))
 591                 goto out;
 592
 593         __enable_irq(desc, irq);
 594 out:
 595         irq_put_desc_busunlock(desc, flags);
 596 }
 597 EXPORT_SYMBOL(enable_irq);
 598
 599 static int set_irq_wake_real(unsigned int irq, unsigned int on)
 600 {
 601         struct irq_desc *desc = irq_to_desc(irq);
 602         int ret = -ENXIO;
 603
 604         if (irq_desc_get_chip(desc)->flags &  IRQCHIP_SKIP_SET_WAKE)
 605                 return 0;
 606
 607         if (desc->irq_data.chip->irq_set_wake)
 608                 ret = desc->irq_data.chip->irq_set_wake(&desc->irq_data, on);
 609
 610         return ret;
 611 }
 612
 613 /**
 614  *      irq_set_irq_wake - control irq power management wakeup
 615  *      @irq:   interrupt to control
 616  *      @on:    enable/disable power management wakeup
 617  *
 618  *      Enable/disable power management wakeup mode, which is
 619  *      disabled by default.  Enables and disables must match,
 620  *      just as they match for non-wakeup mode support.
 621  *
 622  *      Wakeup mode lets this IRQ wake the system from sleep
 623  *      states like "suspend to RAM".
 624  */
 625 int irq_set_irq_wake(unsigned int irq, unsigned int on)
 626 {
 627         unsigned long flags;
 628         struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
 629         int ret = 0;
 630
 631         if (!desc)
 632                 return -EINVAL;
 633
 634         /* wakeup-capable irqs can be shared between drivers that
 635          * don't need to have the same sleep mode behaviors.
 636          */
 637         if (on) {
 638                 if (desc->wake_depth++ == 0) {
 639                         ret = set_irq_wake_real(irq, on);
 640                         if (ret)
 641                                 desc->wake_depth = 0;
 642                         else
 643                                 irqd_set(&desc->irq_data, IRQD_WAKEUP_STATE);
 644                 }
 645         } else {
 646                 if (desc->wake_depth == 0) {
 647                         WARN(1, "Unbalanced IRQ %d wake disable\n", irq);
 648                 } else if (--desc->wake_depth == 0) {
 649                         ret = set_irq_wake_real(irq, on);
 650                         if (ret)
 651                                 desc->wake_depth = 1;
 652                         else
 653                                 irqd_clear(&desc->irq_data, IRQD_WAKEUP_STATE);
 654                 }
 655         }
 656         irq_put_desc_busunlock(desc, flags);
 657         return ret;
 658 }
 659 EXPORT_SYMBOL(irq_set_irq_wake);
 660
 661 /*
 662  * Internal function that tells the architecture code whether a
 663  * particular irq has been exclusively allocated or is available
 664  * for driver use.
 665  */
 666 int can_request_irq(unsigned int irq, unsigned long irqflags)
 667 {
 668         unsigned long flags;
 669         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
 670         int canrequest = 0;
 671
 672         if (!desc)
 673                 return 0;
 674
 675         if (irq_settings_can_request(desc)) {
 676                 if (!desc->action ||
 677                     irqflags & desc->action->flags & IRQF_SHARED)
 678                         canrequest = 1;
 679         }
 680         irq_put_desc_unlock(desc, flags);
 681         return canrequest;
 682 }
 683
 684 int __irq_set_trigger(struct irq_desc *desc, unsigned int irq,
 685                       unsigned long flags)
 686 {
 687         struct irq_chip *chip = desc->irq_data.chip;
 688         int ret, unmask = 0;
 689
 690         if (!chip || !chip->irq_set_type) {
 691                 /*
 692                  * IRQF_TRIGGER_* but the PIC does not support multiple
 693                  * flow-types?
 694                  */
 695                 pr_debug("No set_type function for IRQ %d (%s)\n", irq,
 696                          chip ? (chip->name ? : "unknown") : "unknown");
 697                 return 0;
 698         }
 699
 700         flags &= IRQ_TYPE_SENSE_MASK;
 701
 702         if (chip->flags & IRQCHIP_SET_TYPE_MASKED) {
 703                 if (!irqd_irq_masked(&desc->irq_data))
 704                         mask_irq(desc);
 705                 if (!irqd_irq_disabled(&desc->irq_data))
 706                         unmask = 1;
 707         }
 708
 709         /* caller masked out all except trigger mode flags */
 710         ret = chip->irq_set_type(&desc->irq_data, flags);
 711
 712         switch (ret) {
 713         case IRQ_SET_MASK_OK:
 714         case IRQ_SET_MASK_OK_DONE:
 715                 irqd_clear(&desc->irq_data, IRQD_TRIGGER_MASK);
 716                 irqd_set(&desc->irq_data, flags);
 717
 718         case IRQ_SET_MASK_OK_NOCOPY:
 719                 flags = irqd_get_trigger_type(&desc->irq_data);
 720                 irq_settings_set_trigger_mask(desc, flags);
 721                 irqd_clear(&desc->irq_data, IRQD_LEVEL);
 722                 irq_settings_clr_level(desc);
 723                 if (flags & IRQ_TYPE_LEVEL_MASK) {
 724                         irq_settings_set_level(desc);
 725                         irqd_set(&desc->irq_data, IRQD_LEVEL);
 726                 }
 727
 728                 ret = 0;
 729                 break;
 730         default:
 731                 pr_err("Setting trigger mode %lu for irq %u failed (%pF)\n",
 732                        flags, irq, chip->irq_set_type);
 733         }
 734         if (unmask)
 735                 unmask_irq(desc);
 736         return ret;
 737 }
 738
 739 #ifdef CONFIG_HARDIRQS_SW_RESEND
 740 int irq_set_parent(int irq, int parent_irq)
 741 {
 742         unsigned long flags;
 743         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
 744
 745         if (!desc)
 746                 return -EINVAL;
 747
 748         desc->parent_irq = parent_irq;
 749
 750         irq_put_desc_unlock(desc, flags);
 751         return 0;
 752 }
 753 #endif
 754
 755 /*
 756  * Default primary interrupt handler for threaded interrupts. Is
 757  * assigned as primary handler when request_threaded_irq is called
 758  * with handler == NULL. Useful for oneshot interrupts.
 759  */
 760 static irqreturn_t irq_default_primary_handler(int irq, void *dev_id)
 761 {
 762         return IRQ_WAKE_THREAD;
 763 }
 764
 765 /*
 766  * Primary handler for nested threaded interrupts. Should never be
 767  * called.
 768  */
 769 static irqreturn_t irq_nested_primary_handler(int irq, void *dev_id)
 770 {
 771         WARN(1, "Primary handler called for nested irq %d\n", irq);
 772         return IRQ_NONE;
 773 }
 774
 775 static irqreturn_t irq_forced_secondary_handler(int irq, void *dev_id)
 776 {
 777         WARN(1, "Secondary action handler called for irq %d\n", irq);
 778         return IRQ_NONE;
 779 }
 780
 781 static int irq_wait_for_interrupt(struct irqaction *action)
 782 {
 783         set_current_state(TASK_INTERRUPTIBLE);
 784
 785         while (!kthread_should_stop()) {
 786
 787                 if (test_and_clear_bit(IRQTF_RUNTHREAD,
 788                                        &action->thread_flags)) {
 789                         __set_current_state(TASK_RUNNING);
 790                         return 0;
 791                 }
 792                 schedule();
 793                 set_current_state(TASK_INTERRUPTIBLE);
 794         }
 795         __set_current_state(TASK_RUNNING);
 796         return -1;
 797 }
 798
 799 /*
 800  * Oneshot interrupts keep the irq line masked until the threaded
 801  * handler finished. unmask if the interrupt has not been disabled and
 802  * is marked MASKED.
 803  */
 804 static void irq_finalize_oneshot(struct irq_desc *desc,
 805                                  struct irqaction *action)
 806 {
 807         if (!(desc->istate & IRQS_ONESHOT) ||
 808             action->handler == irq_forced_secondary_handler)
 809                 return;
 810 again:
 811         chip_bus_lock(desc);
 812         raw_spin_lock_irq(&desc->lock);
 813
 814         /*
 815          * Implausible though it may be we need to protect us against
 816          * the following scenario:
 817          *
 818          * The thread is faster done than the hard interrupt handler
 819          * on the other CPU. If we unmask the irq line then the
 820          * interrupt can come in again and masks the line, leaves due
 821          * to IRQS_INPROGRESS and the irq line is masked forever.
 822          *
 823          * This also serializes the state of shared oneshot handlers
 824          * versus "desc->threads_onehsot |= action->thread_mask;" in
 825          * irq_wake_thread(). See the comment there which explains the
 826          * serialization.
 827          */
 828         if (unlikely(irqd_irq_inprogress(&desc->irq_data))) {
 829                 raw_spin_unlock_irq(&desc->lock);
 830                 chip_bus_sync_unlock(desc);
 831                 cpu_relax();
 832                 goto again;
 833         }
 834
 835         /*
 836          * Now check again, whether the thread should run. Otherwise
 837          * we would clear the threads_oneshot bit of this thread which
 838          * was just set.
 839          */
 840         if (test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
 841                 goto out_unlock;
 842
 843         desc->threads_oneshot &= ~action->thread_mask;
 844
 845         if (!desc->threads_oneshot && !irqd_irq_disabled(&desc->irq_data) &&
 846             irqd_irq_masked(&desc->irq_data))
 847                 unmask_threaded_irq(desc);
 848
 849 out_unlock:
 850         raw_spin_unlock_irq(&desc->lock);
 851         chip_bus_sync_unlock(desc);
 852 }
 853
 854 #ifdef CONFIG_SMP
 855 /*
 856  * Check whether we need to change the affinity of the interrupt thread.
 857  */
 858 static void
 859 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action)
 860 {
 861         cpumask_var_t mask;
 862         bool valid = true;
 863
 864         if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags))
 865                 return;
 866
 867         /*
 868          * In case we are out of memory we set IRQTF_AFFINITY again and
 869          * try again next time
 870          */
 871         if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
 872                 set_bit(IRQTF_AFFINITY, &action->thread_flags);
 873                 return;
 874         }
 875
 876         raw_spin_lock_irq(&desc->lock);
 877         /*
 878          * This code is triggered unconditionally. Check the affinity
 879          * mask pointer. For CPU_MASK_OFFSTACK=n this is optimized out.
 880          */
 881         if (desc->irq_data.affinity)
 882                 cpumask_copy(mask, desc->irq_data.affinity);
 883         else
 884                 valid = false;
 885         raw_spin_unlock_irq(&desc->lock);
 886
 887         if (valid)
 888                 set_cpus_allowed_ptr(current, mask);
 889         free_cpumask_var(mask);
 890 }
 891 #else
 892 static inline void
 893 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action) { }
 894 #endif
 895
 896 /*
 897  * Interrupts which are not explicitely requested as threaded
 898  * interrupts rely on the implicit bh/preempt disable of the hard irq
 899  * context. So we need to disable bh here to avoid deadlocks and other
 900  * side effects.
 901  */
 902 static irqreturn_t
 903 irq_forced_thread_fn(struct irq_desc *desc, struct irqaction *action)
 904 {
 905         irqreturn_t ret;
 906
 907         local_bh_disable();
 908         ret = action->thread_fn(action->irq, action->dev_id);
 909         irq_finalize_oneshot(desc, action);
 910         /*
 911          * Interrupts which have real time requirements can be set up
 912          * to avoid softirq processing in the thread handler. This is
 913          * safe as these interrupts do not raise soft interrupts.
 914          */
 915         if (irq_settings_no_softirq_call(desc))
 916                 _local_bh_enable();
 917         else
 918                 local_bh_enable();
 919         return ret;
 920 }
 921
 922 /*
 923  * Interrupts explicitly requested as threaded interrupts want to be
 924  * preemtible - many of them need to sleep and wait for slow busses to
 925  * complete.
 926  */
 927 static irqreturn_t irq_thread_fn(struct irq_desc *desc,
 928                 struct irqaction *action)
 929 {
 930         irqreturn_t ret;
 931
 932         ret = action->thread_fn(action->irq, action->dev_id);
 933         irq_finalize_oneshot(desc, action);
 934         return ret;
 935 }
 936
 937 static void wake_threads_waitq(struct irq_desc *desc)
 938 {
 939         if (atomic_dec_and_test(&desc->threads_active))
 940                 wake_up(&desc->wait_for_threads);
 941 }
 942
 943 static void irq_thread_dtor(struct callback_head *unused)
 944 {
 945         struct task_struct *tsk = current;
 946         struct irq_desc *desc;
 947         struct irqaction *action;
 948
 949         if (WARN_ON_ONCE(!(current->flags & PF_EXITING)))
 950                 return;
 951
 952         action = kthread_data(tsk);
 953
 954         pr_err("exiting task \"%s\" (%d) is an active IRQ thread (irq %d)\n",
 955                tsk->comm, tsk->pid, action->irq);
 956
 957
 958         desc = irq_to_desc(action->irq);
 959         /*
 960          * If IRQTF_RUNTHREAD is set, we need to decrement
 961          * desc->threads_active and wake possible waiters.
 962          */
 963         if (test_and_clear_bit(IRQTF_RUNTHREAD, &action->thread_flags))
 964                 wake_threads_waitq(desc);
 965
 966         /* Prevent a stale desc->threads_oneshot */
 967         irq_finalize_oneshot(desc, action);
 968 }
 969
 970 static void irq_wake_secondary(struct irq_desc *desc, struct irqaction *action)
 971 {
 972         struct irqaction *secondary = action->secondary;
 973
 974         if (WARN_ON_ONCE(!secondary))
 975                 return;
 976
 977         raw_spin_lock_irq(&desc->lock);
 978         __irq_wake_thread(desc, secondary);
 979         raw_spin_unlock_irq(&desc->lock);
 980 }
 981
 982 /*
 983  * Interrupt handler thread
 984  */
 985 static int irq_thread(void *data)
 986 {
 987         struct callback_head on_exit_work;
 988         struct irqaction *action = data;
 989         struct irq_desc *desc = irq_to_desc(action->irq);
 990         irqreturn_t (*handler_fn)(struct irq_desc *desc,
 991                         struct irqaction *action);
 992
 993         if (force_irqthreads && test_bit(IRQTF_FORCED_THREAD,
 994                                         &action->thread_flags))
 995                 handler_fn = irq_forced_thread_fn;
 996         else
 997                 handler_fn = irq_thread_fn;
 998
 999         init_task_work(&on_exit_work, irq_thread_dtor);
1000         task_work_add(current, &on_exit_work, false);
1001
1002         irq_thread_check_affinity(desc, action);
1003
1004         while (!irq_wait_for_interrupt(action)) {
1005                 irqreturn_t action_ret;
1006
1007                 irq_thread_check_affinity(desc, action);
1008
1009                 action_ret = handler_fn(desc, action);
1010                 if (action_ret == IRQ_HANDLED)
1011                         atomic_inc(&desc->threads_handled);
1012                 if (action_ret == IRQ_WAKE_THREAD)
1013                         irq_wake_secondary(desc, action);
1014
1015 #ifdef CONFIG_PREEMPT_RT_FULL
1016                 migrate_disable();
1017                 add_interrupt_randomness(action->irq, 0,
1018                                  desc->random_ip ^ (unsigned long) action);
1019                 migrate_enable();
1020 #endif
1021                 wake_threads_waitq(desc);
1022         }
1023
1024         /*
1025          * This is the regular exit path. __free_irq() is stopping the
1026          * thread via kthread_stop() after calling
1027          * synchronize_irq(). So neither IRQTF_RUNTHREAD nor the
1028          * oneshot mask bit can be set. We cannot verify that as we
1029          * cannot touch the oneshot mask at this point anymore as
1030          * __setup_irq() might have given out currents thread_mask
1031          * again.
1032          */
1033         task_work_cancel(current, irq_thread_dtor);
1034         return 0;
1035 }
1036
1037 /**
1038  *      irq_wake_thread - wake the irq thread for the action identified by dev_id
1039  *      @irq:           Interrupt line
1040  *      @dev_id:        Device identity for which the thread should be woken
1041  *
1042  */
1043 void irq_wake_thread(unsigned int irq, void *dev_id)
1044 {
1045         struct irq_desc *desc = irq_to_desc(irq);
1046         struct irqaction *action;
1047         unsigned long flags;
1048
1049         if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1050                 return;
1051
1052         raw_spin_lock_irqsave(&desc->lock, flags);
1053         for (action = desc->action; action; action = action->next) {
1054                 if (action->dev_id == dev_id) {
1055                         if (action->thread)
1056                                 __irq_wake_thread(desc, action);
1057                         break;
1058                 }
1059         }
1060         raw_spin_unlock_irqrestore(&desc->lock, flags);
1061 }
1062 EXPORT_SYMBOL_GPL(irq_wake_thread);
1063
1064 static int irq_setup_forced_threading(struct irqaction *new)
1065 {
1066         if (!force_irqthreads)
1067                 return 0;
1068         if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))
1069                 return 0;
1070
1071         new->flags |= IRQF_ONESHOT;
1072
1073         /*
1074          * Handle the case where we have a real primary handler and a
1075          * thread handler. We force thread them as well by creating a
1076          * secondary action.
1077          */
1078         if (new->handler != irq_default_primary_handler && new->thread_fn) {
1079                 /* Allocate the secondary action */
1080                 new->secondary = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
1081                 if (!new->secondary)
1082                         return -ENOMEM;
1083                 new->secondary->handler = irq_forced_secondary_handler;
1084                 new->secondary->thread_fn = new->thread_fn;
1085                 new->secondary->dev_id = new->dev_id;
1086                 new->secondary->irq = new->irq;
1087                 new->secondary->name = new->name;
1088         }
1089         /* Deal with the primary handler */
1090         set_bit(IRQTF_FORCED_THREAD, &new->thread_flags);
1091         new->thread_fn = new->handler;
1092         new->handler = irq_default_primary_handler;
1093         return 0;
1094 }
1095
1096 static int irq_request_resources(struct irq_desc *desc)
1097 {
1098         struct irq_data *d = &desc->irq_data;
1099         struct irq_chip *c = d->chip;
1100
1101         return c->irq_request_resources ? c->irq_request_resources(d) : 0;
1102 }
1103
1104 static void irq_release_resources(struct irq_desc *desc)
1105 {
1106         struct irq_data *d = &desc->irq_data;
1107         struct irq_chip *c = d->chip;
1108
1109         if (c->irq_release_resources)
1110                 c->irq_release_resources(d);
1111 }
1112
1113 static int
1114 setup_irq_thread(struct irqaction *new, unsigned int irq, bool secondary)
1115 {
1116         struct task_struct *t;
1117         struct sched_param param = {
1118                 .sched_priority = MAX_USER_RT_PRIO/2,
1119         };
1120
1121         if (!secondary) {
1122                 t = kthread_create(irq_thread, new, "irq/%d-%s", irq,
1123                                    new->name);
1124         } else {
1125                 t = kthread_create(irq_thread, new, "irq/%d-s-%s", irq,
1126                                    new->name);
1127                 param.sched_priority += 1;
1128         }
1129
1130         if (IS_ERR(t))
1131                 return PTR_ERR(t);
1132
1133         sched_setscheduler_nocheck(t, SCHED_FIFO, &param);
1134
1135         /*
1136          * We keep the reference to the task struct even if
1137          * the thread dies to avoid that the interrupt code
1138          * references an already freed task_struct.
1139          */
1140         get_task_struct(t);
1141         new->thread = t;
1142         /*
1143          * Tell the thread to set its affinity. This is
1144          * important for shared interrupt handlers as we do
1145          * not invoke setup_affinity() for the secondary
1146          * handlers as everything is already set up. Even for
1147          * interrupts marked with IRQF_NO_BALANCE this is
1148          * correct as we want the thread to move to the cpu(s)
1149          * on which the requesting code placed the interrupt.
1150          */
1151         set_bit(IRQTF_AFFINITY, &new->thread_flags);
1152         return 0;
1153 }
1154
1155 /*
1156  * Internal function to register an irqaction - typically used to
1157  * allocate special interrupts that are part of the architecture.
1158  */
1159 static int
1160 __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
1161 {
1162         struct irqaction *old, **old_ptr;
1163         unsigned long flags, thread_mask = 0;
1164         int ret, nested, shared = 0;
1165         cpumask_var_t mask;
1166
1167         if (!desc)
1168                 return -EINVAL;
1169
1170         if (desc->irq_data.chip == &no_irq_chip)
1171                 return -ENOSYS;
1172         if (!try_module_get(desc->owner))
1173                 return -ENODEV;
1174
1175         new->irq = irq;
1176
1177         /*
1178          * Check whether the interrupt nests into another interrupt
1179          * thread.
1180          */
1181         nested = irq_settings_is_nested_thread(desc);
1182         if (nested) {
1183                 if (!new->thread_fn) {
1184                         ret = -EINVAL;
1185                         goto out_mput;
1186                 }
1187                 /*
1188                  * Replace the primary handler which was provided from
1189                  * the driver for non nested interrupt handling by the
1190                  * dummy function which warns when called.
1191                  */
1192                 new->handler = irq_nested_primary_handler;
1193         } else {
1194                 if (irq_settings_can_thread(desc)) {
1195                         ret = irq_setup_forced_threading(new);
1196                         if (ret)
1197                                 goto out_mput;
1198                 }
1199         }
1200
1201         /*
1202          * Create a handler thread when a thread function is supplied
1203          * and the interrupt does not nest into another interrupt
1204          * thread.
1205          */
1206         if (new->thread_fn && !nested) {
1207                 ret = setup_irq_thread(new, irq, false);
1208                 if (ret)
1209                         goto out_mput;
1210                 if (new->secondary) {
1211                         ret = setup_irq_thread(new->secondary, irq, true);
1212                         if (ret)
1213                                 goto out_thread;
1214                 }
1215         }
1216
1217         if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
1218                 ret = -ENOMEM;
1219                 goto out_thread;
1220         }
1221
1222         /*
1223          * Drivers are often written to work w/o knowledge about the
1224          * underlying irq chip implementation, so a request for a
1225          * threaded irq without a primary hard irq context handler
1226          * requires the ONESHOT flag to be set. Some irq chips like
1227          * MSI based interrupts are per se one shot safe. Check the
1228          * chip flags, so we can avoid the unmask dance at the end of
1229          * the threaded handler for those.
1230          */
1231         if (desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)
1232                 new->flags &= ~IRQF_ONESHOT;
1233
1234         /*
1235          * The following block of code has to be executed atomically
1236          */
1237         raw_spin_lock_irqsave(&desc->lock, flags);
1238         old_ptr = &desc->action;
1239         old = *old_ptr;
1240         if (old) {
1241                 /*
1242                  * Can't share interrupts unless both agree to and are
1243                  * the same type (level, edge, polarity). So both flag
1244                  * fields must have IRQF_SHARED set and the bits which
1245                  * set the trigger type must match. Also all must
1246                  * agree on ONESHOT.
1247                  */
1248                 if (!((old->flags & new->flags) & IRQF_SHARED) ||
1249                     ((old->flags ^ new->flags) & IRQF_TRIGGER_MASK) ||
1250                     ((old->flags ^ new->flags) & IRQF_ONESHOT))
1251                         goto mismatch;
1252
1253                 /* All handlers must agree on per-cpuness */
1254                 if ((old->flags & IRQF_PERCPU) !=
1255                     (new->flags & IRQF_PERCPU))
1256                         goto mismatch;
1257
1258                 /* add new interrupt at end of irq queue */
1259                 do {
1260                         /*
1261                          * Or all existing action->thread_mask bits,
1262                          * so we can find the next zero bit for this
1263                          * new action.
1264                          */
1265                         thread_mask |= old->thread_mask;
1266                         old_ptr = &old->next;
1267                         old = *old_ptr;
1268                 } while (old);
1269                 shared = 1;
1270         }
1271
1272         /*
1273          * Setup the thread mask for this irqaction for ONESHOT. For
1274          * !ONESHOT irqs the thread mask is 0 so we can avoid a
1275          * conditional in irq_wake_thread().
1276          */
1277         if (new->flags & IRQF_ONESHOT) {
1278                 /*
1279                  * Unlikely to have 32 resp 64 irqs sharing one line,
1280                  * but who knows.
1281                  */
1282                 if (thread_mask == ~0UL) {
1283                         ret = -EBUSY;
1284                         goto out_mask;
1285                 }
1286                 /*
1287                  * The thread_mask for the action is or'ed to
1288                  * desc->thread_active to indicate that the
1289                  * IRQF_ONESHOT thread handler has been woken, but not
1290                  * yet finished. The bit is cleared when a thread
1291                  * completes. When all threads of a shared interrupt
1292                  * line have completed desc->threads_active becomes
1293                  * zero and the interrupt line is unmasked. See
1294                  * handle.c:irq_wake_thread() for further information.
1295                  *
1296                  * If no thread is woken by primary (hard irq context)
1297                  * interrupt handlers, then desc->threads_active is
1298                  * also checked for zero to unmask the irq line in the
1299                  * affected hard irq flow handlers
1300                  * (handle_[fasteoi|level]_irq).
1301                  *
1302                  * The new action gets the first zero bit of
1303                  * thread_mask assigned. See the loop above which or's
1304                  * all existing action->thread_mask bits.
1305                  */
1306                 new->thread_mask = 1 << ffz(thread_mask);
1307
1308         } else if (new->handler == irq_default_primary_handler &&
1309                    !(desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)) {
1310                 /*
1311                  * The interrupt was requested with handler = NULL, so
1312                  * we use the default primary handler for it. But it
1313                  * does not have the oneshot flag set. In combination
1314                  * with level interrupts this is deadly, because the
1315                  * default primary handler just wakes the thread, then
1316                  * the irq lines is reenabled, but the device still
1317                  * has the level irq asserted. Rinse and repeat....
1318                  *
1319                  * While this works for edge type interrupts, we play
1320                  * it safe and reject unconditionally because we can't
1321                  * say for sure which type this interrupt really
1322                  * has. The type flags are unreliable as the
1323                  * underlying chip implementation can override them.
1324                  */
1325                 pr_err("Threaded irq requested with handler=NULL and !ONESHOT for irq %d\n",
1326                        irq);
1327                 ret = -EINVAL;
1328                 goto out_mask;
1329         }
1330
1331         if (!shared) {
1332                 ret = irq_request_resources(desc);
1333                 if (ret) {
1334                         pr_err("Failed to request resources for %s (irq %d) on irqchip %s\n",
1335                                new->name, irq, desc->irq_data.chip->name);
1336                         goto out_mask;
1337                 }
1338
1339                 init_waitqueue_head(&desc->wait_for_threads);
1340
1341                 /* Setup the type (level, edge polarity) if configured: */
1342                 if (new->flags & IRQF_TRIGGER_MASK) {
1343                         ret = __irq_set_trigger(desc, irq,
1344                                         new->flags & IRQF_TRIGGER_MASK);
1345
1346                         if (ret)
1347                                 goto out_mask;
1348                 }
1349
1350                 desc->istate &= ~(IRQS_AUTODETECT | IRQS_SPURIOUS_DISABLED | \
1351                                   IRQS_ONESHOT | IRQS_WAITING);
1352                 irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
1353
1354                 if (new->flags & IRQF_PERCPU) {
1355                         irqd_set(&desc->irq_data, IRQD_PER_CPU);
1356                         irq_settings_set_per_cpu(desc);
1357                 }
1358
1359                 if (new->flags & IRQF_ONESHOT)
1360                         desc->istate |= IRQS_ONESHOT;
1361
1362                 if (irq_settings_can_autoenable(desc))
1363                         irq_startup(desc, true);
1364                 else
1365                         /* Undo nested disables: */
1366                         desc->depth = 1;
1367
1368                 /* Exclude IRQ from balancing if requested */
1369                 if (new->flags & IRQF_NOBALANCING) {
1370                         irq_settings_set_no_balancing(desc);
1371                         irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
1372                 }
1373
1374                 if (new->flags & IRQF_NO_SOFTIRQ_CALL)
1375                         irq_settings_set_no_softirq_call(desc);
1376
1377                 /* Set default affinity mask once everything is setup */
1378                 setup_affinity(irq, desc, mask);
1379
1380         } else if (new->flags & IRQF_TRIGGER_MASK) {
1381                 unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK;
1382                 unsigned int omsk = irq_settings_get_trigger_mask(desc);
1383
1384                 if (nmsk != omsk)
1385                         /* hope the handler works with current  trigger mode */
1386                         pr_warning("irq %d uses trigger mode %u; requested %u\n",
1387                                    irq, nmsk, omsk);
1388         }
1389
1390         *old_ptr = new;
1391
1392         irq_pm_install_action(desc, new);
1393
1394         /* Reset broken irq detection when installing new handler */
1395         desc->irq_count = 0;
1396         desc->irqs_unhandled = 0;
1397
1398         /*
1399          * Check whether we disabled the irq via the spurious handler
1400          * before. Reenable it and give it another chance.
1401          */
1402         if (shared && (desc->istate & IRQS_SPURIOUS_DISABLED)) {
1403                 desc->istate &= ~IRQS_SPURIOUS_DISABLED;
1404                 __enable_irq(desc, irq);
1405         }
1406
1407         raw_spin_unlock_irqrestore(&desc->lock, flags);
1408
1409         /*
1410          * Strictly no need to wake it up, but hung_task complains
1411          * when no hard interrupt wakes the thread up.
1412          */
1413         if (new->thread)
1414                 wake_up_process(new->thread);
1415         if (new->secondary)
1416                 wake_up_process(new->secondary->thread);
1417
1418         register_irq_proc(irq, desc);
1419         new->dir = NULL;
1420         register_handler_proc(irq, new);
1421         free_cpumask_var(mask);
1422
1423         return 0;
1424
1425 mismatch:
1426         if (!(new->flags & IRQF_PROBE_SHARED)) {
1427                 pr_err("Flags mismatch irq %d. %08x (%s) vs. %08x (%s)\n",
1428                        irq, new->flags, new->name, old->flags, old->name);
1429 #ifdef CONFIG_DEBUG_SHIRQ
1430                 dump_stack();
1431 #endif
1432         }
1433         ret = -EBUSY;
1434
1435 out_mask:
1436         raw_spin_unlock_irqrestore(&desc->lock, flags);
1437         free_cpumask_var(mask);
1438
1439 out_thread:
1440         if (new->thread) {
1441                 struct task_struct *t = new->thread;
1442
1443                 new->thread = NULL;
1444                 kthread_stop(t);
1445                 put_task_struct(t);
1446         }
1447         if (new->secondary && new->secondary->thread) {
1448                 struct task_struct *t = new->secondary->thread;
1449
1450                 new->secondary->thread = NULL;
1451                 kthread_stop(t);
1452                 put_task_struct(t);
1453         }
1454 out_mput:
1455         module_put(desc->owner);
1456         return ret;
1457 }
1458
1459 /**
1460  *      setup_irq - setup an interrupt
1461  *      @irq: Interrupt line to setup
1462  *      @act: irqaction for the interrupt
1463  *
1464  * Used to statically setup interrupts in the early boot process.
1465  */
1466 int setup_irq(unsigned int irq, struct irqaction *act)
1467 {
1468         int retval;
1469         struct irq_desc *desc = irq_to_desc(irq);
1470
1471         if (WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1472                 return -EINVAL;
1473         chip_bus_lock(desc);
1474         retval = __setup_irq(irq, desc, act);
1475         chip_bus_sync_unlock(desc);
1476
1477         return retval;
1478 }
1479 EXPORT_SYMBOL_GPL(setup_irq);
1480
1481 /*
1482  * Internal function to unregister an irqaction - used to free
1483  * regular and special interrupts that are part of the architecture.
1484  */
1485 static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
1486 {
1487         struct irq_desc *desc = irq_to_desc(irq);
1488         struct irqaction *action, **action_ptr;
1489         unsigned long flags;
1490
1491         WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
1492
1493         if (!desc)
1494                 return NULL;
1495
1496         raw_spin_lock_irqsave(&desc->lock, flags);
1497
1498         /*
1499          * There can be multiple actions per IRQ descriptor, find the right
1500          * one based on the dev_id:
1501          */
1502         action_ptr = &desc->action;
1503         for (;;) {
1504                 action = *action_ptr;
1505
1506                 if (!action) {
1507                         WARN(1, "Trying to free already-free IRQ %d\n", irq);
1508                         raw_spin_unlock_irqrestore(&desc->lock, flags);
1509
1510                         return NULL;
1511                 }
1512
1513                 if (action->dev_id == dev_id)
1514                         break;
1515                 action_ptr = &action->next;
1516         }
1517
1518         /* Found it - now remove it from the list of entries: */
1519         *action_ptr = action->next;
1520
1521         irq_pm_remove_action(desc, action);
1522
1523         /* If this was the last handler, shut down the IRQ line: */
1524         if (!desc->action) {
1525                 irq_shutdown(desc);
1526                 irq_release_resources(desc);
1527         }
1528
1529 #ifdef CONFIG_SMP
1530         /* make sure affinity_hint is cleaned up */
1531         if (WARN_ON_ONCE(desc->affinity_hint))
1532                 desc->affinity_hint = NULL;
1533 #endif
1534
1535         raw_spin_unlock_irqrestore(&desc->lock, flags);
1536
1537         unregister_handler_proc(irq, action);
1538
1539         /* Make sure it's not being used on another CPU: */
1540         synchronize_irq(irq);
1541
1542 #ifdef CONFIG_DEBUG_SHIRQ
1543         /*
1544          * It's a shared IRQ -- the driver ought to be prepared for an IRQ
1545          * event to happen even now it's being freed, so let's make sure that
1546          * is so by doing an extra call to the handler ....
1547          *
1548          * ( We do this after actually deregistering it, to make sure that a
1549          *   'real' IRQ doesn't run in * parallel with our fake. )
1550          */
1551         if (action->flags & IRQF_SHARED) {
1552                 local_irq_save(flags);
1553                 action->handler(irq, dev_id);
1554                 local_irq_restore(flags);
1555         }
1556 #endif
1557
1558         if (action->thread) {
1559                 kthread_stop(action->thread);
1560                 put_task_struct(action->thread);
1561                 if (action->secondary && action->secondary->thread) {
1562                         kthread_stop(action->secondary->thread);
1563                         put_task_struct(action->secondary->thread);
1564                 }
1565         }
1566
1567         module_put(desc->owner);
1568         kfree(action->secondary);
1569         return action;
1570 }
1571
1572 /**
1573  *      remove_irq - free an interrupt
1574  *      @irq: Interrupt line to free
1575  *      @act: irqaction for the interrupt
1576  *
1577  * Used to remove interrupts statically setup by the early boot process.
1578  */
1579 void remove_irq(unsigned int irq, struct irqaction *act)
1580 {
1581         struct irq_desc *desc = irq_to_desc(irq);
1582
1583         if (desc && !WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1584             __free_irq(irq, act->dev_id);
1585 }
1586 EXPORT_SYMBOL_GPL(remove_irq);
1587
1588 /**
1589  *      free_irq - free an interrupt allocated with request_irq
1590  *      @irq: Interrupt line to free
1591  *      @dev_id: Device identity to free
1592  *
1593  *      Remove an interrupt handler. The handler is removed and if the
1594  *      interrupt line is no longer in use by any driver it is disabled.
1595  *      On a shared IRQ the caller must ensure the interrupt is disabled
1596  *      on the card it drives before calling this function. The function
1597  *      does not return until any executing interrupts for this IRQ
1598  *      have completed.
1599  *
1600  *      This function must not be called from interrupt context.
1601  */
1602 void free_irq(unsigned int irq, void *dev_id)
1603 {
1604         struct irq_desc *desc = irq_to_desc(irq);
1605
1606         if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1607                 return;
1608
1609 #ifdef CONFIG_SMP
1610         if (WARN_ON(desc->affinity_notify))
1611                 desc->affinity_notify = NULL;
1612 #endif
1613
1614         chip_bus_lock(desc);
1615         kfree(__free_irq(irq, dev_id));
1616         chip_bus_sync_unlock(desc);
1617 }
1618 EXPORT_SYMBOL(free_irq);
1619
1620 /**
1621  *      request_threaded_irq - allocate an interrupt line
1622  *      @irq: Interrupt line to allocate
1623  *      @handler: Function to be called when the IRQ occurs.
1624  *                Primary handler for threaded interrupts
1625  *                If NULL and thread_fn != NULL the default
1626  *                primary handler is installed
1627  *      @thread_fn: Function called from the irq handler thread
1628  *                  If NULL, no irq thread is created
1629  *      @irqflags: Interrupt type flags
1630  *      @devname: An ascii name for the claiming device
1631  *      @dev_id: A cookie passed back to the handler function
1632  *
1633  *      This call allocates interrupt resources and enables the
1634  *      interrupt line and IRQ handling. From the point this
1635  *      call is made your handler function may be invoked. Since
1636  *      your handler function must clear any interrupt the board
1637  *      raises, you must take care both to initialise your hardware
1638  *      and to set up the interrupt handler in the right order.
1639  *
1640  *      If you want to set up a threaded irq handler for your device
1641  *      then you need to supply @handler and @thread_fn. @handler is
1642  *      still called in hard interrupt context and has to check
1643  *      whether the interrupt originates from the device. If yes it
1644  *      needs to disable the interrupt on the device and return
1645  *      IRQ_WAKE_THREAD which will wake up the handler thread and run
1646  *      @thread_fn. This split handler design is necessary to support
1647  *      shared interrupts.
1648  *
1649  *      Dev_id must be globally unique. Normally the address of the
1650  *      device data structure is used as the cookie. Since the handler
1651  *      receives this value it makes sense to use it.
1652  *
1653  *      If your interrupt is shared you must pass a non NULL dev_id
1654  *      as this is required when freeing the interrupt.
1655  *
1656  *      Flags:
1657  *
1658  *      IRQF_SHARED             Interrupt is shared
1659  *      IRQF_TRIGGER_*          Specify active edge(s) or level
1660  *
1661  */
1662 int request_threaded_irq(unsigned int irq, irq_handler_t handler,
1663                          irq_handler_t thread_fn, unsigned long irqflags,
1664                          const char *devname, void *dev_id)
1665 {
1666         struct irqaction *action;
1667         struct irq_desc *desc;
1668         int retval;
1669
1670         /*
1671          * Sanity-check: shared interrupts must pass in a real dev-ID,
1672          * otherwise we'll have trouble later trying to figure out
1673          * which interrupt is which (messes up the interrupt freeing
1674          * logic etc).
1675          *
1676          * Also IRQF_COND_SUSPEND only makes sense for shared interrupts and
1677          * it cannot be set along with IRQF_NO_SUSPEND.
1678          */
1679         if (((irqflags & IRQF_SHARED) && !dev_id) ||
1680             (!(irqflags & IRQF_SHARED) && (irqflags & IRQF_COND_SUSPEND)) ||
1681             ((irqflags & IRQF_NO_SUSPEND) && (irqflags & IRQF_COND_SUSPEND)))
1682                 return -EINVAL;
1683
1684         desc = irq_to_desc(irq);
1685         if (!desc)
1686                 return -EINVAL;
1687
1688         if (!irq_settings_can_request(desc) ||
1689             WARN_ON(irq_settings_is_per_cpu_devid(desc)))
1690                 return -EINVAL;
1691
1692         if (!handler) {
1693                 if (!thread_fn)
1694                         return -EINVAL;
1695                 handler = irq_default_primary_handler;
1696         }
1697
1698         action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
1699         if (!action)
1700                 return -ENOMEM;
1701
1702         action->handler = handler;
1703         action->thread_fn = thread_fn;
1704         action->flags = irqflags;
1705         action->name = devname;
1706         action->dev_id = dev_id;
1707
1708         chip_bus_lock(desc);
1709         retval = __setup_irq(irq, desc, action);
1710         chip_bus_sync_unlock(desc);
1711
1712         if (retval) {
1713                 kfree(action->secondary);
1714                 kfree(action);
1715         }
1716
1717 #ifdef CONFIG_DEBUG_SHIRQ_FIXME
1718         if (!retval && (irqflags & IRQF_SHARED)) {
1719                 /*
1720                  * It's a shared IRQ -- the driver ought to be prepared for it
1721                  * to happen immediately, so let's make sure....
1722                  * We disable the irq to make sure that a 'real' IRQ doesn't
1723                  * run in parallel with our fake.
1724                  */
1725                 unsigned long flags;
1726
1727                 disable_irq(irq);
1728                 local_irq_save(flags);
1729
1730                 handler(irq, dev_id);
1731
1732                 local_irq_restore(flags);
1733                 enable_irq(irq);
1734         }
1735 #endif
1736         return retval;
1737 }
1738 EXPORT_SYMBOL(request_threaded_irq);
1739
1740 /**
1741  *      request_any_context_irq - allocate an interrupt line
1742  *      @irq: Interrupt line to allocate
1743  *      @handler: Function to be called when the IRQ occurs.
1744  *                Threaded handler for threaded interrupts.
1745  *      @flags: Interrupt type flags
1746  *      @name: An ascii name for the claiming device
1747  *      @dev_id: A cookie passed back to the handler function
1748  *
1749  *      This call allocates interrupt resources and enables the
1750  *      interrupt line and IRQ handling. It selects either a
1751  *      hardirq or threaded handling method depending on the
1752  *      context.
1753  *
1754  *      On failure, it returns a negative value. On success,
1755  *      it returns either IRQC_IS_HARDIRQ or IRQC_IS_NESTED.
1756  */
1757 int request_any_context_irq(unsigned int irq, irq_handler_t handler,
1758                             unsigned long flags, const char *name, void *dev_id)
1759 {
1760         struct irq_desc *desc = irq_to_desc(irq);
1761         int ret;
1762
1763         if (!desc)
1764                 return -EINVAL;
1765
1766         if (irq_settings_is_nested_thread(desc)) {
1767                 ret = request_threaded_irq(irq, NULL, handler,
1768                                            flags, name, dev_id);
1769                 return !ret ? IRQC_IS_NESTED : ret;
1770         }
1771
1772         ret = request_irq(irq, handler, flags, name, dev_id);
1773         return !ret ? IRQC_IS_HARDIRQ : ret;
1774 }
1775 EXPORT_SYMBOL_GPL(request_any_context_irq);
1776
1777 void enable_percpu_irq(unsigned int irq, unsigned int type)
1778 {
1779         unsigned int cpu = smp_processor_id();
1780         unsigned long flags;
1781         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
1782
1783         if (!desc)
1784                 return;
1785
1786         type &= IRQ_TYPE_SENSE_MASK;
1787         if (type != IRQ_TYPE_NONE) {
1788                 int ret;
1789
1790                 ret = __irq_set_trigger(desc, irq, type);
1791
1792                 if (ret) {
1793                         WARN(1, "failed to set type for IRQ%d\n", irq);
1794                         goto out;
1795                 }
1796         }
1797
1798         irq_percpu_enable(desc, cpu);
1799 out:
1800         irq_put_desc_unlock(desc, flags);
1801 }
1802 EXPORT_SYMBOL_GPL(enable_percpu_irq);
1803
1804 void disable_percpu_irq(unsigned int irq)
1805 {
1806         unsigned int cpu = smp_processor_id();
1807         unsigned long flags;
1808         struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
1809
1810         if (!desc)
1811                 return;
1812
1813         irq_percpu_disable(desc, cpu);
1814         irq_put_desc_unlock(desc, flags);
1815 }
1816 EXPORT_SYMBOL_GPL(disable_percpu_irq);
1817
1818 /*
1819  * Internal function to unregister a percpu irqaction.
1820  */
1821 static struct irqaction *__free_percpu_irq(unsigned int irq, void __percpu *dev_id)
1822 {
1823         struct irq_desc *desc = irq_to_desc(irq);
1824         struct irqaction *action;
1825         unsigned long flags;
1826
1827         WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
1828
1829         if (!desc)
1830                 return NULL;
1831
1832         raw_spin_lock_irqsave(&desc->lock, flags);
1833
1834         action = desc->action;
1835         if (!action || action->percpu_dev_id != dev_id) {
1836                 WARN(1, "Trying to free already-free IRQ %d\n", irq);
1837                 goto bad;
1838         }
1839
1840         if (!cpumask_empty(desc->percpu_enabled)) {
1841                 WARN(1, "percpu IRQ %d still enabled on CPU%d!\n",
1842                      irq, cpumask_first(desc->percpu_enabled));
1843                 goto bad;
1844         }
1845
1846         /* Found it - now remove it from the list of entries: */
1847         desc->action = NULL;
1848
1849         raw_spin_unlock_irqrestore(&desc->lock, flags);
1850
1851         unregister_handler_proc(irq, action);
1852
1853         module_put(desc->owner);
1854         return action;
1855
1856 bad:
1857         raw_spin_unlock_irqrestore(&desc->lock, flags);
1858         return NULL;
1859 }
1860
1861 /**
1862  *      remove_percpu_irq - free a per-cpu interrupt
1863  *      @irq: Interrupt line to free
1864  *      @act: irqaction for the interrupt
1865  *
1866  * Used to remove interrupts statically setup by the early boot process.
1867  */
1868 void remove_percpu_irq(unsigned int irq, struct irqaction *act)
1869 {
1870         struct irq_desc *desc = irq_to_desc(irq);
1871
1872         if (desc && irq_settings_is_per_cpu_devid(desc))
1873             __free_percpu_irq(irq, act->percpu_dev_id);
1874 }
1875
1876 /**
1877  *      free_percpu_irq - free an interrupt allocated with request_percpu_irq
1878  *      @irq: Interrupt line to free
1879  *      @dev_id: Device identity to free
1880  *
1881  *      Remove a percpu interrupt handler. The handler is removed, but
1882  *      the interrupt line is not disabled. This must be done on each
1883  *      CPU before calling this function. The function does not return
1884  *      until any executing interrupts for this IRQ have completed.
1885  *
1886  *      This function must not be called from interrupt context.
1887  */
1888 void free_percpu_irq(unsigned int irq, void __percpu *dev_id)
1889 {
1890         struct irq_desc *desc = irq_to_desc(irq);
1891
1892         if (!desc || !irq_settings_is_per_cpu_devid(desc))
1893                 return;
1894
1895         chip_bus_lock(desc);
1896         kfree(__free_percpu_irq(irq, dev_id));
1897         chip_bus_sync_unlock(desc);
1898 }
1899
1900 /**
1901  *      setup_percpu_irq - setup a per-cpu interrupt
1902  *      @irq: Interrupt line to setup
1903  *      @act: irqaction for the interrupt
1904  *
1905  * Used to statically setup per-cpu interrupts in the early boot process.
1906  */
1907 int setup_percpu_irq(unsigned int irq, struct irqaction *act)
1908 {
1909         struct irq_desc *desc = irq_to_desc(irq);
1910         int retval;
1911
1912         if (!desc || !irq_settings_is_per_cpu_devid(desc))
1913                 return -EINVAL;
1914         chip_bus_lock(desc);
1915         retval = __setup_irq(irq, desc, act);
1916         chip_bus_sync_unlock(desc);
1917
1918         return retval;
1919 }
1920
1921 /**
1922  *      request_percpu_irq - allocate a percpu interrupt line
1923  *      @irq: Interrupt line to allocate
1924  *      @handler: Function to be called when the IRQ occurs.
1925  *      @devname: An ascii name for the claiming device
1926  *      @dev_id: A percpu cookie passed back to the handler function
1927  *
1928  *      This call allocates interrupt resources, but doesn't
1929  *      automatically enable the interrupt. It has to be done on each
1930  *      CPU using enable_percpu_irq().
1931  *
1932  *      Dev_id must be globally unique. It is a per-cpu variable, and
1933  *      the handler gets called with the interrupted CPU's instance of
1934  *      that variable.
1935  */
1936 int request_percpu_irq(unsigned int irq, irq_handler_t handler,
1937                        const char *devname, void __percpu *dev_id)
1938 {
1939         struct irqaction *action;
1940         struct irq_desc *desc;
1941         int retval;
1942
1943         if (!dev_id)
1944                 return -EINVAL;
1945
1946         desc = irq_to_desc(irq);
1947         if (!desc || !irq_settings_can_request(desc) ||
1948             !irq_settings_is_per_cpu_devid(desc))
1949                 return -EINVAL;
1950
1951         action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
1952         if (!action)
1953                 return -ENOMEM;
1954
1955         action->handler = handler;
1956         action->flags = IRQF_PERCPU | IRQF_NO_SUSPEND;
1957         action->name = devname;
1958         action->percpu_dev_id = dev_id;
1959
1960         chip_bus_lock(desc);
1961         retval = __setup_irq(irq, desc, action);
1962         chip_bus_sync_unlock(desc);
1963
1964         if (retval)
1965                 kfree(action);
1966
1967         return retval;
1968 }
1969
1970 /**
1971  *      irq_get_irqchip_state - returns the irqchip state of a interrupt.
1972  *      @irq: Interrupt line that is forwarded to a VM
1973  *      @which: One of IRQCHIP_STATE_* the caller wants to know about
1974  *      @state: a pointer to a boolean where the state is to be storeed
1975  *
1976  *      This call snapshots the internal irqchip state of an
1977  *      interrupt, returning into @state the bit corresponding to
1978  *      stage @which
1979  *
1980  *      This function should be called with preemption disabled if the
1981  *      interrupt controller has per-cpu registers.
1982  */
1983 int irq_get_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
1984                           bool *state)
1985 {
1986         struct irq_desc *desc;
1987         struct irq_data *data;
1988         struct irq_chip *chip;
1989         unsigned long flags;
1990         int err = -EINVAL;
1991
1992         desc = irq_get_desc_buslock(irq, &flags, 0);
1993         if (!desc)
1994                 return err;
1995
1996         data = irq_desc_get_irq_data(desc);
1997
1998         do {
1999                 chip = irq_data_get_irq_chip(data);
2000                 if (chip->irq_get_irqchip_state)
2001                         break;
2002 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
2003                 data = data->parent_data;
2004 #else
2005                 data = NULL;
2006 #endif
2007         } while (data);
2008
2009         if (data)
2010                 err = chip->irq_get_irqchip_state(data, which, state);
2011
2012         irq_put_desc_busunlock(desc, flags);
2013         return err;
2014 }
2015
2016 /**
2017  *      irq_set_irqchip_state - set the state of a forwarded interrupt.
2018  *      @irq: Interrupt line that is forwarded to a VM
2019  *      @which: State to be restored (one of IRQCHIP_STATE_*)
2020  *      @val: Value corresponding to @which
2021  *
2022  *      This call sets the internal irqchip state of an interrupt,
2023  *      depending on the value of @which.
2024  *
2025  *      This function should be called with preemption disabled if the
2026  *      interrupt controller has per-cpu registers.
2027  */
2028 int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which,
2029                           bool val)
2030 {
2031         struct irq_desc *desc;
2032         struct irq_data *data;
2033         struct irq_chip *chip;
2034         unsigned long flags;
2035         int err = -EINVAL;
2036
2037         desc = irq_get_desc_buslock(irq, &flags, 0);
2038         if (!desc)
2039                 return err;
2040
2041         data = irq_desc_get_irq_data(desc);
2042
2043         do {
2044                 chip = irq_data_get_irq_chip(data);
2045                 if (chip->irq_set_irqchip_state)
2046                         break;
2047 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
2048                 data = data->parent_data;
2049 #else
2050                 data = NULL;
2051 #endif
2052         } while (data);
2053
2054         if (data)
2055                 err = chip->irq_set_irqchip_state(data, which, val);
2056
2057         irq_put_desc_busunlock(desc, flags);
2058         return err;
2059 }