kernel/arch/x86/kernel/hpet.c

   1 #include <linux/clocksource.h>
   2 #include <linux/clockchips.h>
   3 #include <linux/interrupt.h>
   4 #include <linux/export.h>
   5 #include <linux/delay.h>
   6 #include <linux/errno.h>
   7 #include <linux/i8253.h>
   8 #include <linux/slab.h>
   9 #include <linux/hpet.h>
  10 #include <linux/init.h>
  11 #include <linux/cpu.h>
  12 #include <linux/pm.h>
  13 #include <linux/io.h>
  14
  15 #include <asm/fixmap.h>
  16 #include <asm/hpet.h>
  17 #include <asm/time.h>
  18
  19 #define HPET_MASK                       CLOCKSOURCE_MASK(32)
  20
  21 /* FSEC = 10^-15
  22    NSEC = 10^-9 */
  23 #define FSEC_PER_NSEC                   1000000L
  24
  25 #define HPET_DEV_USED_BIT               2
  26 #define HPET_DEV_USED                   (1 << HPET_DEV_USED_BIT)
  27 #define HPET_DEV_VALID                  0x8
  28 #define HPET_DEV_FSB_CAP                0x1000
  29 #define HPET_DEV_PERI_CAP               0x2000
  30
  31 #define HPET_MIN_CYCLES                 128
  32 #define HPET_MIN_PROG_DELTA             (HPET_MIN_CYCLES + (HPET_MIN_CYCLES >> 1))
  33
  34 /*
  35  * HPET address is set in acpi/boot.c, when an ACPI entry exists
  36  */
  37 unsigned long                           hpet_address;
  38 u8                                      hpet_blockid; /* OS timer block num */
  39 u8                                      hpet_msi_disable;
  40
  41 #ifdef CONFIG_PCI_MSI
  42 static unsigned long                    hpet_num_timers;
  43 #endif
  44 static void __iomem                     *hpet_virt_address;
  45
  46 struct hpet_dev {
  47         struct clock_event_device       evt;
  48         unsigned int                    num;
  49         int                             cpu;
  50         unsigned int                    irq;
  51         unsigned int                    flags;
  52         char                            name[10];
  53 };
  54
  55 inline struct hpet_dev *EVT_TO_HPET_DEV(struct clock_event_device *evtdev)
  56 {
  57         return container_of(evtdev, struct hpet_dev, evt);
  58 }
  59
  60 inline unsigned int hpet_readl(unsigned int a)
  61 {
  62         return readl(hpet_virt_address + a);
  63 }
  64
  65 static inline void hpet_writel(unsigned int d, unsigned int a)
  66 {
  67         writel(d, hpet_virt_address + a);
  68 }
  69
  70 #ifdef CONFIG_X86_64
  71 #include <asm/pgtable.h>
  72 #endif
  73
  74 static inline void hpet_set_mapping(void)
  75 {
  76         hpet_virt_address = ioremap_nocache(hpet_address, HPET_MMAP_SIZE);
  77 }
  78
  79 static inline void hpet_clear_mapping(void)
  80 {
  81         iounmap(hpet_virt_address);
  82         hpet_virt_address = NULL;
  83 }
  84
  85 /*
  86  * HPET command line enable / disable
  87  */
  88 int boot_hpet_disable;
  89 int hpet_force_user;
  90 static int hpet_verbose;
  91
  92 static int __init hpet_setup(char *str)
  93 {
  94         while (str) {
  95                 char *next = strchr(str, ',');
  96
  97                 if (next)
  98                         *next++ = 0;
  99                 if (!strncmp("disable", str, 7))
 100                         boot_hpet_disable = 1;
 101                 if (!strncmp("force", str, 5))
 102                         hpet_force_user = 1;
 103                 if (!strncmp("verbose", str, 7))
 104                         hpet_verbose = 1;
 105                 str = next;
 106         }
 107         return 1;
 108 }
 109 __setup("hpet=", hpet_setup);
 110
 111 static int __init disable_hpet(char *str)
 112 {
 113         boot_hpet_disable = 1;
 114         return 1;
 115 }
 116 __setup("nohpet", disable_hpet);
 117
 118 static inline int is_hpet_capable(void)
 119 {
 120         return !boot_hpet_disable && hpet_address;
 121 }
 122
 123 /*
 124  * HPET timer interrupt enable / disable
 125  */
 126 static int hpet_legacy_int_enabled;
 127
 128 /**
 129  * is_hpet_enabled - check whether the hpet timer interrupt is enabled
 130  */
 131 int is_hpet_enabled(void)
 132 {
 133         return is_hpet_capable() && hpet_legacy_int_enabled;
 134 }
 135 EXPORT_SYMBOL_GPL(is_hpet_enabled);
 136
 137 static void _hpet_print_config(const char *function, int line)
 138 {
 139         u32 i, timers, l, h;
 140         printk(KERN_INFO "hpet: %s(%d):\n", function, line);
 141         l = hpet_readl(HPET_ID);
 142         h = hpet_readl(HPET_PERIOD);
 143         timers = ((l & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT) + 1;
 144         printk(KERN_INFO "hpet: ID: 0x%x, PERIOD: 0x%x\n", l, h);
 145         l = hpet_readl(HPET_CFG);
 146         h = hpet_readl(HPET_STATUS);
 147         printk(KERN_INFO "hpet: CFG: 0x%x, STATUS: 0x%x\n", l, h);
 148         l = hpet_readl(HPET_COUNTER);
 149         h = hpet_readl(HPET_COUNTER+4);
 150         printk(KERN_INFO "hpet: COUNTER_l: 0x%x, COUNTER_h: 0x%x\n", l, h);
 151
 152         for (i = 0; i < timers; i++) {
 153                 l = hpet_readl(HPET_Tn_CFG(i));
 154                 h = hpet_readl(HPET_Tn_CFG(i)+4);
 155                 printk(KERN_INFO "hpet: T%d: CFG_l: 0x%x, CFG_h: 0x%x\n",
 156                        i, l, h);
 157                 l = hpet_readl(HPET_Tn_CMP(i));
 158                 h = hpet_readl(HPET_Tn_CMP(i)+4);
 159                 printk(KERN_INFO "hpet: T%d: CMP_l: 0x%x, CMP_h: 0x%x\n",
 160                        i, l, h);
 161                 l = hpet_readl(HPET_Tn_ROUTE(i));
 162                 h = hpet_readl(HPET_Tn_ROUTE(i)+4);
 163                 printk(KERN_INFO "hpet: T%d ROUTE_l: 0x%x, ROUTE_h: 0x%x\n",
 164                        i, l, h);
 165         }
 166 }
 167
 168 #define hpet_print_config()                                     \
 169 do {                                                            \
 170         if (hpet_verbose)                                       \
 171                 _hpet_print_config(__func__, __LINE__); \
 172 } while (0)
 173
 174 /*
 175  * When the hpet driver (/dev/hpet) is enabled, we need to reserve
 176  * timer 0 and timer 1 in case of RTC emulation.
 177  */
 178 #ifdef CONFIG_HPET
 179
 180 static void hpet_reserve_msi_timers(struct hpet_data *hd);
 181
 182 static void hpet_reserve_platform_timers(unsigned int id)
 183 {
 184         struct hpet __iomem *hpet = hpet_virt_address;
 185         struct hpet_timer __iomem *timer = &hpet->hpet_timers[2];
 186         unsigned int nrtimers, i;
 187         struct hpet_data hd;
 188
 189         nrtimers = ((id & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT) + 1;
 190
 191         memset(&hd, 0, sizeof(hd));
 192         hd.hd_phys_address      = hpet_address;
 193         hd.hd_address           = hpet;
 194         hd.hd_nirqs             = nrtimers;
 195         hpet_reserve_timer(&hd, 0);
 196
 197 #ifdef CONFIG_HPET_EMULATE_RTC
 198         hpet_reserve_timer(&hd, 1);
 199 #endif
 200
 201         /*
 202          * NOTE that hd_irq[] reflects IOAPIC input pins (LEGACY_8254
 203          * is wrong for i8259!) not the output IRQ.  Many BIOS writers
 204          * don't bother configuring *any* comparator interrupts.
 205          */
 206         hd.hd_irq[0] = HPET_LEGACY_8254;
 207         hd.hd_irq[1] = HPET_LEGACY_RTC;
 208
 209         for (i = 2; i < nrtimers; timer++, i++) {
 210                 hd.hd_irq[i] = (readl(&timer->hpet_config) &
 211                         Tn_INT_ROUTE_CNF_MASK) >> Tn_INT_ROUTE_CNF_SHIFT;
 212         }
 213
 214         hpet_reserve_msi_timers(&hd);
 215
 216         hpet_alloc(&hd);
 217
 218 }
 219 #else
 220 static void hpet_reserve_platform_timers(unsigned int id) { }
 221 #endif
 222
 223 /*
 224  * Common hpet info
 225  */
 226 static unsigned long hpet_freq;
 227
 228 static void hpet_legacy_set_mode(enum clock_event_mode mode,
 229                           struct clock_event_device *evt);
 230 static int hpet_legacy_next_event(unsigned long delta,
 231                            struct clock_event_device *evt);
 232
 233 /*
 234  * The hpet clock event device
 235  */
 236 static struct clock_event_device hpet_clockevent = {
 237         .name           = "hpet",
 238         .features       = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT,
 239         .set_mode       = hpet_legacy_set_mode,
 240         .set_next_event = hpet_legacy_next_event,
 241         .irq            = 0,
 242         .rating         = 50,
 243 };
 244
 245 static void hpet_stop_counter(void)
 246 {
 247         unsigned long cfg = hpet_readl(HPET_CFG);
 248         cfg &= ~HPET_CFG_ENABLE;
 249         hpet_writel(cfg, HPET_CFG);
 250 }
 251
 252 static void hpet_reset_counter(void)
 253 {
 254         hpet_writel(0, HPET_COUNTER);
 255         hpet_writel(0, HPET_COUNTER + 4);
 256 }
 257
 258 static void hpet_start_counter(void)
 259 {
 260         unsigned int cfg = hpet_readl(HPET_CFG);
 261         cfg |= HPET_CFG_ENABLE;
 262         hpet_writel(cfg, HPET_CFG);
 263 }
 264
 265 static void hpet_restart_counter(void)
 266 {
 267         hpet_stop_counter();
 268         hpet_reset_counter();
 269         hpet_start_counter();
 270 }
 271
 272 static void hpet_resume_device(void)
 273 {
 274         force_hpet_resume();
 275 }
 276
 277 static void hpet_resume_counter(struct clocksource *cs)
 278 {
 279         hpet_resume_device();
 280         hpet_restart_counter();
 281 }
 282
 283 static void hpet_enable_legacy_int(void)
 284 {
 285         unsigned int cfg = hpet_readl(HPET_CFG);
 286
 287         cfg |= HPET_CFG_LEGACY;
 288         hpet_writel(cfg, HPET_CFG);
 289         hpet_legacy_int_enabled = 1;
 290 }
 291
 292 static void hpet_legacy_clockevent_register(void)
 293 {
 294         /* Start HPET legacy interrupts */
 295         hpet_enable_legacy_int();
 296
 297         /*
 298          * Start hpet with the boot cpu mask and make it
 299          * global after the IO_APIC has been initialized.
 300          */
 301         hpet_clockevent.cpumask = cpumask_of(smp_processor_id());
 302         clockevents_config_and_register(&hpet_clockevent, hpet_freq,
 303                                         HPET_MIN_PROG_DELTA, 0x7FFFFFFF);
 304         global_clock_event = &hpet_clockevent;
 305         printk(KERN_DEBUG "hpet clockevent registered\n");
 306 }
 307
 308 static int hpet_setup_msi_irq(unsigned int irq);
 309
 310 static void hpet_set_mode(enum clock_event_mode mode,
 311                           struct clock_event_device *evt, int timer)
 312 {
 313         unsigned int cfg, cmp, now;
 314         uint64_t delta;
 315
 316         switch (mode) {
 317         case CLOCK_EVT_MODE_PERIODIC:
 318                 hpet_stop_counter();
 319                 delta = ((uint64_t)(NSEC_PER_SEC/HZ)) * evt->mult;
 320                 delta >>= evt->shift;
 321                 now = hpet_readl(HPET_COUNTER);
 322                 cmp = now + (unsigned int) delta;
 323                 cfg = hpet_readl(HPET_Tn_CFG(timer));
 324                 cfg |= HPET_TN_ENABLE | HPET_TN_PERIODIC |
 325                        HPET_TN_SETVAL | HPET_TN_32BIT;
 326                 hpet_writel(cfg, HPET_Tn_CFG(timer));
 327                 hpet_writel(cmp, HPET_Tn_CMP(timer));
 328                 udelay(1);
 329                 /*
 330                  * HPET on AMD 81xx needs a second write (with HPET_TN_SETVAL
 331                  * cleared) to T0_CMP to set the period. The HPET_TN_SETVAL
 332                  * bit is automatically cleared after the first write.
 333                  * (See AMD-8111 HyperTransport I/O Hub Data Sheet,
 334                  * Publication # 24674)
 335                  */
 336                 hpet_writel((unsigned int) delta, HPET_Tn_CMP(timer));
 337                 hpet_start_counter();
 338                 hpet_print_config();
 339                 break;
 340
 341         case CLOCK_EVT_MODE_ONESHOT:
 342                 cfg = hpet_readl(HPET_Tn_CFG(timer));
 343                 cfg &= ~HPET_TN_PERIODIC;
 344                 cfg |= HPET_TN_ENABLE | HPET_TN_32BIT;
 345                 hpet_writel(cfg, HPET_Tn_CFG(timer));
 346                 break;
 347
 348         case CLOCK_EVT_MODE_UNUSED:
 349         case CLOCK_EVT_MODE_SHUTDOWN:
 350                 cfg = hpet_readl(HPET_Tn_CFG(timer));
 351                 cfg &= ~HPET_TN_ENABLE;
 352                 hpet_writel(cfg, HPET_Tn_CFG(timer));
 353                 break;
 354
 355         case CLOCK_EVT_MODE_RESUME:
 356                 if (timer == 0) {
 357                         hpet_enable_legacy_int();
 358                 } else {
 359                         struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt);
 360                         hpet_setup_msi_irq(hdev->irq);
 361                         disable_irq(hdev->irq);
 362                         irq_set_affinity(hdev->irq, cpumask_of(hdev->cpu));
 363                         enable_irq(hdev->irq);
 364                 }
 365                 hpet_print_config();
 366                 break;
 367         }
 368 }
 369
 370 static int hpet_next_event(unsigned long delta,
 371                            struct clock_event_device *evt, int timer)
 372 {
 373         u32 cnt;
 374         s32 res;
 375
 376         cnt = hpet_readl(HPET_COUNTER);
 377         cnt += (u32) delta;
 378         hpet_writel(cnt, HPET_Tn_CMP(timer));
 379
 380         /*
 381          * HPETs are a complete disaster. The compare register is
 382          * based on a equal comparison and neither provides a less
 383          * than or equal functionality (which would require to take
 384          * the wraparound into account) nor a simple count down event
 385          * mode. Further the write to the comparator register is
 386          * delayed internally up to two HPET clock cycles in certain
 387          * chipsets (ATI, ICH9,10). Some newer AMD chipsets have even
 388          * longer delays. We worked around that by reading back the
 389          * compare register, but that required another workaround for
 390          * ICH9,10 chips where the first readout after write can
 391          * return the old stale value. We already had a minimum
 392          * programming delta of 5us enforced, but a NMI or SMI hitting
 393          * between the counter readout and the comparator write can
 394          * move us behind that point easily. Now instead of reading
 395          * the compare register back several times, we make the ETIME
 396          * decision based on the following: Return ETIME if the
 397          * counter value after the write is less than HPET_MIN_CYCLES
 398          * away from the event or if the counter is already ahead of
 399          * the event. The minimum programming delta for the generic
 400          * clockevents code is set to 1.5 * HPET_MIN_CYCLES.
 401          */
 402         res = (s32)(cnt - hpet_readl(HPET_COUNTER));
 403
 404         return res < HPET_MIN_CYCLES ? -ETIME : 0;
 405 }
 406
 407 static void hpet_legacy_set_mode(enum clock_event_mode mode,
 408                         struct clock_event_device *evt)
 409 {
 410         hpet_set_mode(mode, evt, 0);
 411 }
 412
 413 static int hpet_legacy_next_event(unsigned long delta,
 414                         struct clock_event_device *evt)
 415 {
 416         return hpet_next_event(delta, evt, 0);
 417 }
 418
 419 /*
 420  * HPET MSI Support
 421  */
 422 #ifdef CONFIG_PCI_MSI
 423
 424 static DEFINE_PER_CPU(struct hpet_dev *, cpu_hpet_dev);
 425 static struct hpet_dev  *hpet_devs;
 426
 427 void hpet_msi_unmask(struct irq_data *data)
 428 {
 429         struct hpet_dev *hdev = data->handler_data;
 430         unsigned int cfg;
 431
 432         /* unmask it */
 433         cfg = hpet_readl(HPET_Tn_CFG(hdev->num));
 434         cfg |= HPET_TN_ENABLE | HPET_TN_FSB;
 435         hpet_writel(cfg, HPET_Tn_CFG(hdev->num));
 436 }
 437
 438 void hpet_msi_mask(struct irq_data *data)
 439 {
 440         struct hpet_dev *hdev = data->handler_data;
 441         unsigned int cfg;
 442
 443         /* mask it */
 444         cfg = hpet_readl(HPET_Tn_CFG(hdev->num));
 445         cfg &= ~(HPET_TN_ENABLE | HPET_TN_FSB);
 446         hpet_writel(cfg, HPET_Tn_CFG(hdev->num));
 447 }
 448
 449 void hpet_msi_write(struct hpet_dev *hdev, struct msi_msg *msg)
 450 {
 451         hpet_writel(msg->data, HPET_Tn_ROUTE(hdev->num));
 452         hpet_writel(msg->address_lo, HPET_Tn_ROUTE(hdev->num) + 4);
 453 }
 454
 455 void hpet_msi_read(struct hpet_dev *hdev, struct msi_msg *msg)
 456 {
 457         msg->data = hpet_readl(HPET_Tn_ROUTE(hdev->num));
 458         msg->address_lo = hpet_readl(HPET_Tn_ROUTE(hdev->num) + 4);
 459         msg->address_hi = 0;
 460 }
 461
 462 static void hpet_msi_set_mode(enum clock_event_mode mode,
 463                                 struct clock_event_device *evt)
 464 {
 465         struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt);
 466         hpet_set_mode(mode, evt, hdev->num);
 467 }
 468
 469 static int hpet_msi_next_event(unsigned long delta,
 470                                 struct clock_event_device *evt)
 471 {
 472         struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt);
 473         return hpet_next_event(delta, evt, hdev->num);
 474 }
 475
 476 static int hpet_setup_msi_irq(unsigned int irq)
 477 {
 478         if (x86_msi.setup_hpet_msi(irq, hpet_blockid)) {
 479                 irq_free_hwirq(irq);
 480                 return -EINVAL;
 481         }
 482         return 0;
 483 }
 484
 485 static int hpet_assign_irq(struct hpet_dev *dev)
 486 {
 487         unsigned int irq = irq_alloc_hwirq(-1);
 488
 489         if (!irq)
 490                 return -EINVAL;
 491
 492         irq_set_handler_data(irq, dev);
 493
 494         if (hpet_setup_msi_irq(irq))
 495                 return -EINVAL;
 496
 497         dev->irq = irq;
 498         return 0;
 499 }
 500
 501 static irqreturn_t hpet_interrupt_handler(int irq, void *data)
 502 {
 503         struct hpet_dev *dev = (struct hpet_dev *)data;
 504         struct clock_event_device *hevt = &dev->evt;
 505
 506         if (!hevt->event_handler) {
 507                 printk(KERN_INFO "Spurious HPET timer interrupt on HPET timer %d\n",
 508                                 dev->num);
 509                 return IRQ_HANDLED;
 510         }
 511
 512         hevt->event_handler(hevt);
 513         return IRQ_HANDLED;
 514 }
 515
 516 static int hpet_setup_irq(struct hpet_dev *dev)
 517 {
 518
 519         if (request_irq(dev->irq, hpet_interrupt_handler,
 520                         IRQF_TIMER | IRQF_NOBALANCING,
 521                         dev->name, dev))
 522                 return -1;
 523
 524         disable_irq(dev->irq);
 525         irq_set_affinity(dev->irq, cpumask_of(dev->cpu));
 526         enable_irq(dev->irq);
 527
 528         printk(KERN_DEBUG "hpet: %s irq %d for MSI\n",
 529                          dev->name, dev->irq);
 530
 531         return 0;
 532 }
 533
 534 /* This should be called in specific @cpu */
 535 static void init_one_hpet_msi_clockevent(struct hpet_dev *hdev, int cpu)
 536 {
 537         struct clock_event_device *evt = &hdev->evt;
 538
 539         WARN_ON(cpu != smp_processor_id());
 540         if (!(hdev->flags & HPET_DEV_VALID))
 541                 return;
 542
 543         if (hpet_setup_msi_irq(hdev->irq))
 544                 return;
 545
 546         hdev->cpu = cpu;
 547         per_cpu(cpu_hpet_dev, cpu) = hdev;
 548         evt->name = hdev->name;
 549         hpet_setup_irq(hdev);
 550         evt->irq = hdev->irq;
 551
 552         evt->rating = 110;
 553         evt->features = CLOCK_EVT_FEAT_ONESHOT;
 554         if (hdev->flags & HPET_DEV_PERI_CAP)
 555                 evt->features |= CLOCK_EVT_FEAT_PERIODIC;
 556
 557         evt->set_mode = hpet_msi_set_mode;
 558         evt->set_next_event = hpet_msi_next_event;
 559         evt->cpumask = cpumask_of(hdev->cpu);
 560
 561         clockevents_config_and_register(evt, hpet_freq, HPET_MIN_PROG_DELTA,
 562                                         0x7FFFFFFF);
 563 }
 564
 565 #ifdef CONFIG_HPET
 566 /* Reserve at least one timer for userspace (/dev/hpet) */
 567 #define RESERVE_TIMERS 1
 568 #else
 569 #define RESERVE_TIMERS 0
 570 #endif
 571
 572 static void hpet_msi_capability_lookup(unsigned int start_timer)
 573 {
 574         unsigned int id;
 575         unsigned int num_timers;
 576         unsigned int num_timers_used = 0;
 577         int i;
 578
 579         if (hpet_msi_disable)
 580                 return;
 581
 582         if (boot_cpu_has(X86_FEATURE_ARAT))
 583                 return;
 584         id = hpet_readl(HPET_ID);
 585
 586         num_timers = ((id & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT);
 587         num_timers++; /* Value read out starts from 0 */
 588         hpet_print_config();
 589
 590         hpet_devs = kzalloc(sizeof(struct hpet_dev) * num_timers, GFP_KERNEL);
 591         if (!hpet_devs)
 592                 return;
 593
 594         hpet_num_timers = num_timers;
 595
 596         for (i = start_timer; i < num_timers - RESERVE_TIMERS; i++) {
 597                 struct hpet_dev *hdev = &hpet_devs[num_timers_used];
 598                 unsigned int cfg = hpet_readl(HPET_Tn_CFG(i));
 599
 600                 /* Only consider HPET timer with MSI support */
 601                 if (!(cfg & HPET_TN_FSB_CAP))
 602                         continue;
 603
 604                 hdev->flags = 0;
 605                 if (cfg & HPET_TN_PERIODIC_CAP)
 606                         hdev->flags |= HPET_DEV_PERI_CAP;
 607                 hdev->num = i;
 608
 609                 sprintf(hdev->name, "hpet%d", i);
 610                 if (hpet_assign_irq(hdev))
 611                         continue;
 612
 613                 hdev->flags |= HPET_DEV_FSB_CAP;
 614                 hdev->flags |= HPET_DEV_VALID;
 615                 num_timers_used++;
 616                 if (num_timers_used == num_possible_cpus())
 617                         break;
 618         }
 619
 620         printk(KERN_INFO "HPET: %d timers in total, %d timers will be used for per-cpu timer\n",
 621                 num_timers, num_timers_used);
 622 }
 623
 624 #ifdef CONFIG_HPET
 625 static void hpet_reserve_msi_timers(struct hpet_data *hd)
 626 {
 627         int i;
 628
 629         if (!hpet_devs)
 630                 return;
 631
 632         for (i = 0; i < hpet_num_timers; i++) {
 633                 struct hpet_dev *hdev = &hpet_devs[i];
 634
 635                 if (!(hdev->flags & HPET_DEV_VALID))
 636                         continue;
 637
 638                 hd->hd_irq[hdev->num] = hdev->irq;
 639                 hpet_reserve_timer(hd, hdev->num);
 640         }
 641 }
 642 #endif
 643
 644 static struct hpet_dev *hpet_get_unused_timer(void)
 645 {
 646         int i;
 647
 648         if (!hpet_devs)
 649                 return NULL;
 650
 651         for (i = 0; i < hpet_num_timers; i++) {
 652                 struct hpet_dev *hdev = &hpet_devs[i];
 653
 654                 if (!(hdev->flags & HPET_DEV_VALID))
 655                         continue;
 656                 if (test_and_set_bit(HPET_DEV_USED_BIT,
 657                         (unsigned long *)&hdev->flags))
 658                         continue;
 659                 return hdev;
 660         }
 661         return NULL;
 662 }
 663
 664 struct hpet_work_struct {
 665         struct delayed_work work;
 666         struct completion complete;
 667 };
 668
 669 static void hpet_work(struct work_struct *w)
 670 {
 671         struct hpet_dev *hdev;
 672         int cpu = smp_processor_id();
 673         struct hpet_work_struct *hpet_work;
 674
 675         hpet_work = container_of(w, struct hpet_work_struct, work.work);
 676
 677         hdev = hpet_get_unused_timer();
 678         if (hdev)
 679                 init_one_hpet_msi_clockevent(hdev, cpu);
 680
 681         complete(&hpet_work->complete);
 682 }
 683
 684 static int hpet_cpuhp_notify(struct notifier_block *n,
 685                 unsigned long action, void *hcpu)
 686 {
 687         unsigned long cpu = (unsigned long)hcpu;
 688         struct hpet_work_struct work;
 689         struct hpet_dev *hdev = per_cpu(cpu_hpet_dev, cpu);
 690
 691         switch (action & 0xf) {
 692         case CPU_ONLINE:
 693                 INIT_DELAYED_WORK_ONSTACK(&work.work, hpet_work);
 694                 init_completion(&work.complete);
 695                 /* FIXME: add schedule_work_on() */
 696                 schedule_delayed_work_on(cpu, &work.work, 0);
 697                 wait_for_completion(&work.complete);
 698                 destroy_delayed_work_on_stack(&work.work);
 699                 break;
 700         case CPU_DEAD:
 701                 if (hdev) {
 702                         free_irq(hdev->irq, hdev);
 703                         hdev->flags &= ~HPET_DEV_USED;
 704                         per_cpu(cpu_hpet_dev, cpu) = NULL;
 705                 }
 706                 break;
 707         }
 708         return NOTIFY_OK;
 709 }
 710 #else
 711
 712 static int hpet_setup_msi_irq(unsigned int irq)
 713 {
 714         return 0;
 715 }
 716 static void hpet_msi_capability_lookup(unsigned int start_timer)
 717 {
 718         return;
 719 }
 720
 721 #ifdef CONFIG_HPET
 722 static void hpet_reserve_msi_timers(struct hpet_data *hd)
 723 {
 724         return;
 725 }
 726 #endif
 727
 728 static int hpet_cpuhp_notify(struct notifier_block *n,
 729                 unsigned long action, void *hcpu)
 730 {
 731         return NOTIFY_OK;
 732 }
 733
 734 #endif
 735
 736 /*
 737  * Clock source related code
 738  */
 739 static cycle_t read_hpet(struct clocksource *cs)
 740 {
 741         return (cycle_t)hpet_readl(HPET_COUNTER);
 742 }
 743
 744 static struct clocksource clocksource_hpet = {
 745         .name           = "hpet",
 746         .rating         = 250,
 747         .read           = read_hpet,
 748         .mask           = HPET_MASK,
 749         .flags          = CLOCK_SOURCE_IS_CONTINUOUS,
 750         .resume         = hpet_resume_counter,
 751         .archdata       = { .vclock_mode = VCLOCK_HPET },
 752 };
 753
 754 static int hpet_clocksource_register(void)
 755 {
 756         u64 start, now;
 757         cycle_t t1;
 758
 759         /* Start the counter */
 760         hpet_restart_counter();
 761
 762         /* Verify whether hpet counter works */
 763         t1 = hpet_readl(HPET_COUNTER);
 764         rdtscll(start);
 765
 766         /*
 767          * We don't know the TSC frequency yet, but waiting for
 768          * 200000 TSC cycles is safe:
 769          * 4 GHz == 50us
 770          * 1 GHz == 200us
 771          */
 772         do {
 773                 rep_nop();
 774                 rdtscll(now);
 775         } while ((now - start) < 200000UL);
 776
 777         if (t1 == hpet_readl(HPET_COUNTER)) {
 778                 printk(KERN_WARNING
 779                        "HPET counter not counting. HPET disabled\n");
 780                 return -ENODEV;
 781         }
 782
 783         clocksource_register_hz(&clocksource_hpet, (u32)hpet_freq);
 784         return 0;
 785 }
 786
 787 static u32 *hpet_boot_cfg;
 788
 789 /**
 790  * hpet_enable - Try to setup the HPET timer. Returns 1 on success.
 791  */
 792 int __init hpet_enable(void)
 793 {
 794         u32 hpet_period, cfg, id;
 795         u64 freq;
 796         unsigned int i, last;
 797
 798         if (!is_hpet_capable())
 799                 return 0;
 800
 801         hpet_set_mapping();
 802
 803         /*
 804          * Read the period and check for a sane value:
 805          */
 806         hpet_period = hpet_readl(HPET_PERIOD);
 807
 808         /*
 809          * AMD SB700 based systems with spread spectrum enabled use a
 810          * SMM based HPET emulation to provide proper frequency
 811          * setting. The SMM code is initialized with the first HPET
 812          * register access and takes some time to complete. During
 813          * this time the config register reads 0xffffffff. We check
 814          * for max. 1000 loops whether the config register reads a non
 815          * 0xffffffff value to make sure that HPET is up and running
 816          * before we go further. A counting loop is safe, as the HPET
 817          * access takes thousands of CPU cycles. On non SB700 based
 818          * machines this check is only done once and has no side
 819          * effects.
 820          */
 821         for (i = 0; hpet_readl(HPET_CFG) == 0xFFFFFFFF; i++) {
 822                 if (i == 1000) {
 823                         printk(KERN_WARNING
 824                                "HPET config register value = 0xFFFFFFFF. "
 825                                "Disabling HPET\n");
 826                         goto out_nohpet;
 827                 }
 828         }
 829
 830         if (hpet_period < HPET_MIN_PERIOD || hpet_period > HPET_MAX_PERIOD)
 831                 goto out_nohpet;
 832
 833         /*
 834          * The period is a femto seconds value. Convert it to a
 835          * frequency.
 836          */
 837         freq = FSEC_PER_SEC;
 838         do_div(freq, hpet_period);
 839         hpet_freq = freq;
 840
 841         /*
 842          * Read the HPET ID register to retrieve the IRQ routing
 843          * information and the number of channels
 844          */
 845         id = hpet_readl(HPET_ID);
 846         hpet_print_config();
 847
 848         last = (id & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT;
 849
 850 #ifdef CONFIG_HPET_EMULATE_RTC
 851         /*
 852          * The legacy routing mode needs at least two channels, tick timer
 853          * and the rtc emulation channel.
 854          */
 855         if (!last)
 856                 goto out_nohpet;
 857 #endif
 858
 859         cfg = hpet_readl(HPET_CFG);
 860         hpet_boot_cfg = kmalloc((last + 2) * sizeof(*hpet_boot_cfg),
 861                                 GFP_KERNEL);
 862         if (hpet_boot_cfg)
 863                 *hpet_boot_cfg = cfg;
 864         else
 865                 pr_warn("HPET initial state will not be saved\n");
 866         cfg &= ~(HPET_CFG_ENABLE | HPET_CFG_LEGACY);
 867         hpet_writel(cfg, HPET_CFG);
 868         if (cfg)
 869                 pr_warn("HPET: Unrecognized bits %#x set in global cfg\n",
 870                         cfg);
 871
 872         for (i = 0; i <= last; ++i) {
 873                 cfg = hpet_readl(HPET_Tn_CFG(i));
 874                 if (hpet_boot_cfg)
 875                         hpet_boot_cfg[i + 1] = cfg;
 876                 cfg &= ~(HPET_TN_ENABLE | HPET_TN_LEVEL | HPET_TN_FSB);
 877                 hpet_writel(cfg, HPET_Tn_CFG(i));
 878                 cfg &= ~(HPET_TN_PERIODIC | HPET_TN_PERIODIC_CAP
 879                          | HPET_TN_64BIT_CAP | HPET_TN_32BIT | HPET_TN_ROUTE
 880                          | HPET_TN_FSB | HPET_TN_FSB_CAP);
 881                 if (cfg)
 882                         pr_warn("HPET: Unrecognized bits %#x set in cfg#%u\n",
 883                                 cfg, i);
 884         }
 885         hpet_print_config();
 886
 887         if (hpet_clocksource_register())
 888                 goto out_nohpet;
 889
 890         if (id & HPET_ID_LEGSUP) {
 891                 hpet_legacy_clockevent_register();
 892                 return 1;
 893         }
 894         return 0;
 895
 896 out_nohpet:
 897         hpet_clear_mapping();
 898         hpet_address = 0;
 899         return 0;
 900 }
 901
 902 /*
 903  * Needs to be late, as the reserve_timer code calls kalloc !
 904  *
 905  * Not a problem on i386 as hpet_enable is called from late_time_init,
 906  * but on x86_64 it is necessary !
 907  */
 908 static __init int hpet_late_init(void)
 909 {
 910         int cpu;
 911
 912         if (boot_hpet_disable)
 913                 return -ENODEV;
 914
 915         if (!hpet_address) {
 916                 if (!force_hpet_address)
 917                         return -ENODEV;
 918
 919                 hpet_address = force_hpet_address;
 920                 hpet_enable();
 921         }
 922
 923         if (!hpet_virt_address)
 924                 return -ENODEV;
 925
 926         if (hpet_readl(HPET_ID) & HPET_ID_LEGSUP)
 927                 hpet_msi_capability_lookup(2);
 928         else
 929                 hpet_msi_capability_lookup(0);
 930
 931         hpet_reserve_platform_timers(hpet_readl(HPET_ID));
 932         hpet_print_config();
 933
 934         if (hpet_msi_disable)
 935                 return 0;
 936
 937         if (boot_cpu_has(X86_FEATURE_ARAT))
 938                 return 0;
 939
 940         cpu_notifier_register_begin();
 941         for_each_online_cpu(cpu) {
 942                 hpet_cpuhp_notify(NULL, CPU_ONLINE, (void *)(long)cpu);
 943         }
 944
 945         /* This notifier should be called after workqueue is ready */
 946         __hotcpu_notifier(hpet_cpuhp_notify, -20);
 947         cpu_notifier_register_done();
 948
 949         return 0;
 950 }
 951 fs_initcall(hpet_late_init);
 952
 953 void hpet_disable(void)
 954 {
 955         if (is_hpet_capable() && hpet_virt_address) {
 956                 unsigned int cfg = hpet_readl(HPET_CFG), id, last;
 957
 958                 if (hpet_boot_cfg)
 959                         cfg = *hpet_boot_cfg;
 960                 else if (hpet_legacy_int_enabled) {
 961                         cfg &= ~HPET_CFG_LEGACY;
 962                         hpet_legacy_int_enabled = 0;
 963                 }
 964                 cfg &= ~HPET_CFG_ENABLE;
 965                 hpet_writel(cfg, HPET_CFG);
 966
 967                 if (!hpet_boot_cfg)
 968                         return;
 969
 970                 id = hpet_readl(HPET_ID);
 971                 last = ((id & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT);
 972
 973                 for (id = 0; id <= last; ++id)
 974                         hpet_writel(hpet_boot_cfg[id + 1], HPET_Tn_CFG(id));
 975
 976                 if (*hpet_boot_cfg & HPET_CFG_ENABLE)
 977                         hpet_writel(*hpet_boot_cfg, HPET_CFG);
 978         }
 979 }
 980
 981 #ifdef CONFIG_HPET_EMULATE_RTC
 982
 983 /* HPET in LegacyReplacement Mode eats up RTC interrupt line. When, HPET
 984  * is enabled, we support RTC interrupt functionality in software.
 985  * RTC has 3 kinds of interrupts:
 986  * 1) Update Interrupt - generate an interrupt, every sec, when RTC clock
 987  *    is updated
 988  * 2) Alarm Interrupt - generate an interrupt at a specific time of day
 989  * 3) Periodic Interrupt - generate periodic interrupt, with frequencies
 990  *    2Hz-8192Hz (2Hz-64Hz for non-root user) (all freqs in powers of 2)
 991  * (1) and (2) above are implemented using polling at a frequency of
 992  * 64 Hz. The exact frequency is a tradeoff between accuracy and interrupt
 993  * overhead. (DEFAULT_RTC_INT_FREQ)
 994  * For (3), we use interrupts at 64Hz or user specified periodic
 995  * frequency, whichever is higher.
 996  */
 997 #include <linux/mc146818rtc.h>
 998 #include <linux/rtc.h>
 999 #include <asm/rtc.h>
1000
1001 #define DEFAULT_RTC_INT_FREQ    64
1002 #define DEFAULT_RTC_SHIFT       6
1003 #define RTC_NUM_INTS            1
1004
1005 static unsigned long hpet_rtc_flags;
1006 static int hpet_prev_update_sec;
1007 static struct rtc_time hpet_alarm_time;
1008 static unsigned long hpet_pie_count;
1009 static u32 hpet_t1_cmp;
1010 static u32 hpet_default_delta;
1011 static u32 hpet_pie_delta;
1012 static unsigned long hpet_pie_limit;
1013
1014 static rtc_irq_handler irq_handler;
1015
1016 /*
1017  * Check that the hpet counter c1 is ahead of the c2
1018  */
1019 static inline int hpet_cnt_ahead(u32 c1, u32 c2)
1020 {
1021         return (s32)(c2 - c1) < 0;
1022 }
1023
1024 /*
1025  * Registers a IRQ handler.
1026  */
1027 int hpet_register_irq_handler(rtc_irq_handler handler)
1028 {
1029         if (!is_hpet_enabled())
1030                 return -ENODEV;
1031         if (irq_handler)
1032                 return -EBUSY;
1033
1034         irq_handler = handler;
1035
1036         return 0;
1037 }
1038 EXPORT_SYMBOL_GPL(hpet_register_irq_handler);
1039
1040 /*
1041  * Deregisters the IRQ handler registered with hpet_register_irq_handler()
1042  * and does cleanup.
1043  */
1044 void hpet_unregister_irq_handler(rtc_irq_handler handler)
1045 {
1046         if (!is_hpet_enabled())
1047                 return;
1048
1049         irq_handler = NULL;
1050         hpet_rtc_flags = 0;
1051 }
1052 EXPORT_SYMBOL_GPL(hpet_unregister_irq_handler);
1053
1054 /*
1055  * Timer 1 for RTC emulation. We use one shot mode, as periodic mode
1056  * is not supported by all HPET implementations for timer 1.
1057  *
1058  * hpet_rtc_timer_init() is called when the rtc is initialized.
1059  */
1060 int hpet_rtc_timer_init(void)
1061 {
1062         unsigned int cfg, cnt, delta;
1063         unsigned long flags;
1064
1065         if (!is_hpet_enabled())
1066                 return 0;
1067
1068         if (!hpet_default_delta) {
1069                 uint64_t clc;
1070
1071                 clc = (uint64_t) hpet_clockevent.mult * NSEC_PER_SEC;
1072                 clc >>= hpet_clockevent.shift + DEFAULT_RTC_SHIFT;
1073                 hpet_default_delta = clc;
1074         }
1075
1076         if (!(hpet_rtc_flags & RTC_PIE) || hpet_pie_limit)
1077                 delta = hpet_default_delta;
1078         else
1079                 delta = hpet_pie_delta;
1080
1081         local_irq_save(flags);
1082
1083         cnt = delta + hpet_readl(HPET_COUNTER);
1084         hpet_writel(cnt, HPET_T1_CMP);
1085         hpet_t1_cmp = cnt;
1086
1087         cfg = hpet_readl(HPET_T1_CFG);
1088         cfg &= ~HPET_TN_PERIODIC;
1089         cfg |= HPET_TN_ENABLE | HPET_TN_32BIT;
1090         hpet_writel(cfg, HPET_T1_CFG);
1091
1092         local_irq_restore(flags);
1093
1094         return 1;
1095 }
1096 EXPORT_SYMBOL_GPL(hpet_rtc_timer_init);
1097
1098 static void hpet_disable_rtc_channel(void)
1099 {
1100         unsigned long cfg;
1101         cfg = hpet_readl(HPET_T1_CFG);
1102         cfg &= ~HPET_TN_ENABLE;
1103         hpet_writel(cfg, HPET_T1_CFG);
1104 }
1105
1106 /*
1107  * The functions below are called from rtc driver.
1108  * Return 0 if HPET is not being used.
1109  * Otherwise do the necessary changes and return 1.
1110  */
1111 int hpet_mask_rtc_irq_bit(unsigned long bit_mask)
1112 {
1113         if (!is_hpet_enabled())
1114                 return 0;
1115
1116         hpet_rtc_flags &= ~bit_mask;
1117         if (unlikely(!hpet_rtc_flags))
1118                 hpet_disable_rtc_channel();
1119
1120         return 1;
1121 }
1122 EXPORT_SYMBOL_GPL(hpet_mask_rtc_irq_bit);
1123
1124 int hpet_set_rtc_irq_bit(unsigned long bit_mask)
1125 {
1126         unsigned long oldbits = hpet_rtc_flags;
1127
1128         if (!is_hpet_enabled())
1129                 return 0;
1130
1131         hpet_rtc_flags |= bit_mask;
1132
1133         if ((bit_mask & RTC_UIE) && !(oldbits & RTC_UIE))
1134                 hpet_prev_update_sec = -1;
1135
1136         if (!oldbits)
1137                 hpet_rtc_timer_init();
1138
1139         return 1;
1140 }
1141 EXPORT_SYMBOL_GPL(hpet_set_rtc_irq_bit);
1142
1143 int hpet_set_alarm_time(unsigned char hrs, unsigned char min,
1144                         unsigned char sec)
1145 {
1146         if (!is_hpet_enabled())
1147                 return 0;
1148
1149         hpet_alarm_time.tm_hour = hrs;
1150         hpet_alarm_time.tm_min = min;
1151         hpet_alarm_time.tm_sec = sec;
1152
1153         return 1;
1154 }
1155 EXPORT_SYMBOL_GPL(hpet_set_alarm_time);
1156
1157 int hpet_set_periodic_freq(unsigned long freq)
1158 {
1159         uint64_t clc;
1160
1161         if (!is_hpet_enabled())
1162                 return 0;
1163
1164         if (freq <= DEFAULT_RTC_INT_FREQ)
1165                 hpet_pie_limit = DEFAULT_RTC_INT_FREQ / freq;
1166         else {
1167                 clc = (uint64_t) hpet_clockevent.mult * NSEC_PER_SEC;
1168                 do_div(clc, freq);
1169                 clc >>= hpet_clockevent.shift;
1170                 hpet_pie_delta = clc;
1171                 hpet_pie_limit = 0;
1172         }
1173         return 1;
1174 }
1175 EXPORT_SYMBOL_GPL(hpet_set_periodic_freq);
1176
1177 int hpet_rtc_dropped_irq(void)
1178 {
1179         return is_hpet_enabled();
1180 }
1181 EXPORT_SYMBOL_GPL(hpet_rtc_dropped_irq);
1182
1183 static void hpet_rtc_timer_reinit(void)
1184 {
1185         unsigned int delta;
1186         int lost_ints = -1;
1187
1188         if (unlikely(!hpet_rtc_flags))
1189                 hpet_disable_rtc_channel();
1190
1191         if (!(hpet_rtc_flags & RTC_PIE) || hpet_pie_limit)
1192                 delta = hpet_default_delta;
1193         else
1194                 delta = hpet_pie_delta;
1195
1196         /*
1197          * Increment the comparator value until we are ahead of the
1198          * current count.
1199          */
1200         do {
1201                 hpet_t1_cmp += delta;
1202                 hpet_writel(hpet_t1_cmp, HPET_T1_CMP);
1203                 lost_ints++;
1204         } while (!hpet_cnt_ahead(hpet_t1_cmp, hpet_readl(HPET_COUNTER)));
1205
1206         if (lost_ints) {
1207                 if (hpet_rtc_flags & RTC_PIE)
1208                         hpet_pie_count += lost_ints;
1209                 if (printk_ratelimit())
1210                         printk(KERN_WARNING "hpet1: lost %d rtc interrupts\n",
1211                                 lost_ints);
1212         }
1213 }
1214
1215 irqreturn_t hpet_rtc_interrupt(int irq, void *dev_id)
1216 {
1217         struct rtc_time curr_time;
1218         unsigned long rtc_int_flag = 0;
1219
1220         hpet_rtc_timer_reinit();
1221         memset(&curr_time, 0, sizeof(struct rtc_time));
1222
1223         if (hpet_rtc_flags & (RTC_UIE | RTC_AIE))
1224                 get_rtc_time(&curr_time);
1225
1226         if (hpet_rtc_flags & RTC_UIE &&
1227             curr_time.tm_sec != hpet_prev_update_sec) {
1228                 if (hpet_prev_update_sec >= 0)
1229                         rtc_int_flag = RTC_UF;
1230                 hpet_prev_update_sec = curr_time.tm_sec;
1231         }
1232
1233         if (hpet_rtc_flags & RTC_PIE &&
1234             ++hpet_pie_count >= hpet_pie_limit) {
1235                 rtc_int_flag |= RTC_PF;
1236                 hpet_pie_count = 0;
1237         }
1238
1239         if (hpet_rtc_flags & RTC_AIE &&
1240             (curr_time.tm_sec == hpet_alarm_time.tm_sec) &&
1241             (curr_time.tm_min == hpet_alarm_time.tm_min) &&
1242             (curr_time.tm_hour == hpet_alarm_time.tm_hour))
1243                         rtc_int_flag |= RTC_AF;
1244
1245         if (rtc_int_flag) {
1246                 rtc_int_flag |= (RTC_IRQF | (RTC_NUM_INTS << 8));
1247                 if (irq_handler)
1248                         irq_handler(rtc_int_flag, dev_id);
1249         }
1250         return IRQ_HANDLED;
1251 }
1252 EXPORT_SYMBOL_GPL(hpet_rtc_interrupt);
1253 #endif