kernel/arch/powerpc/platforms/85xx/smp.c

   1 /*
   2  * Author: Andy Fleming <afleming@freescale.com>
   3  *         Kumar Gala <galak@kernel.crashing.org>
   4  *
   5  * Copyright 2006-2008, 2011-2012 Freescale Semiconductor Inc.
   6  *
   7  * This program is free software; you can redistribute  it and/or modify it
   8  * under  the terms of  the GNU General  Public License as published by the
   9  * Free Software Foundation;  either version 2 of the  License, or (at your
  10  * option) any later version.
  11  */
  12
  13 #include <linux/stddef.h>
  14 #include <linux/kernel.h>
  15 #include <linux/init.h>
  16 #include <linux/delay.h>
  17 #include <linux/of.h>
  18 #include <linux/of_address.h>
  19 #include <linux/kexec.h>
  20 #include <linux/highmem.h>
  21 #include <linux/cpu.h>
  22 #include <linux/fsl/guts.h>
  23
  24 #include <asm/machdep.h>
  25 #include <asm/pgtable.h>
  26 #include <asm/page.h>
  27 #include <asm/mpic.h>
  28 #include <asm/cacheflush.h>
  29 #include <asm/dbell.h>
  30 #include <asm/code-patching.h>
  31 #include <asm/cputhreads.h>
  32
  33 #include <sysdev/fsl_soc.h>
  34 #include <sysdev/mpic.h>
  35 #include "smp.h"
  36
  37 struct epapr_spin_table {
  38         u32     addr_h;
  39         u32     addr_l;
  40         u32     r3_h;
  41         u32     r3_l;
  42         u32     reserved;
  43         u32     pir;
  44 };
  45
  46 static struct ccsr_guts __iomem *guts;
  47 static u64 timebase;
  48 static int tb_req;
  49 static int tb_valid;
  50
  51 static void mpc85xx_timebase_freeze(int freeze)
  52 {
  53         uint32_t mask;
  54
  55         mask = CCSR_GUTS_DEVDISR_TB0 | CCSR_GUTS_DEVDISR_TB1;
  56         if (freeze)
  57                 setbits32(&guts->devdisr, mask);
  58         else
  59                 clrbits32(&guts->devdisr, mask);
  60
  61         in_be32(&guts->devdisr);
  62 }
  63
  64 static void mpc85xx_give_timebase(void)
  65 {
  66         unsigned long flags;
  67
  68         local_irq_save(flags);
  69
  70         while (!tb_req)
  71                 barrier();
  72         tb_req = 0;
  73
  74         mpc85xx_timebase_freeze(1);
  75 #ifdef CONFIG_PPC64
  76         /*
  77          * e5500/e6500 have a workaround for erratum A-006958 in place
  78          * that will reread the timebase until TBL is non-zero.
  79          * That would be a bad thing when the timebase is frozen.
  80          *
  81          * Thus, we read it manually, and instead of checking that
  82          * TBL is non-zero, we ensure that TB does not change.  We don't
  83          * do that for the main mftb implementation, because it requires
  84          * a scratch register
  85          */
  86         {
  87                 u64 prev;
  88
  89                 asm volatile("mfspr %0, %1" : "=r" (timebase) :
  90                              "i" (SPRN_TBRL));
  91
  92                 do {
  93                         prev = timebase;
  94                         asm volatile("mfspr %0, %1" : "=r" (timebase) :
  95                                      "i" (SPRN_TBRL));
  96                 } while (prev != timebase);
  97         }
  98 #else
  99         timebase = get_tb();
 100 #endif
 101         mb();
 102         tb_valid = 1;
 103
 104         while (tb_valid)
 105                 barrier();
 106
 107         mpc85xx_timebase_freeze(0);
 108
 109         local_irq_restore(flags);
 110 }
 111
 112 static void mpc85xx_take_timebase(void)
 113 {
 114         unsigned long flags;
 115
 116         local_irq_save(flags);
 117
 118         tb_req = 1;
 119         while (!tb_valid)
 120                 barrier();
 121
 122         set_tb(timebase >> 32, timebase & 0xffffffff);
 123         isync();
 124         tb_valid = 0;
 125
 126         local_irq_restore(flags);
 127 }
 128
 129 #ifdef CONFIG_HOTPLUG_CPU
 130 static void smp_85xx_mach_cpu_die(void)
 131 {
 132         unsigned int cpu = smp_processor_id();
 133         u32 tmp;
 134
 135         local_irq_disable();
 136         idle_task_exit();
 137         generic_set_cpu_dead(cpu);
 138         mb();
 139
 140         mtspr(SPRN_TCR, 0);
 141
 142         __flush_disable_L1();
 143         tmp = (mfspr(SPRN_HID0) & ~(HID0_DOZE|HID0_SLEEP)) | HID0_NAP;
 144         mtspr(SPRN_HID0, tmp);
 145         isync();
 146
 147         /* Enter NAP mode. */
 148         tmp = mfmsr();
 149         tmp |= MSR_WE;
 150         mb();
 151         mtmsr(tmp);
 152         isync();
 153
 154         while (1)
 155                 ;
 156 }
 157 #endif
 158
 159 static inline void flush_spin_table(void *spin_table)
 160 {
 161         flush_dcache_range((ulong)spin_table,
 162                 (ulong)spin_table + sizeof(struct epapr_spin_table));
 163 }
 164
 165 static inline u32 read_spin_table_addr_l(void *spin_table)
 166 {
 167         flush_dcache_range((ulong)spin_table,
 168                 (ulong)spin_table + sizeof(struct epapr_spin_table));
 169         return in_be32(&((struct epapr_spin_table *)spin_table)->addr_l);
 170 }
 171
 172 #ifdef CONFIG_PPC64
 173 static void wake_hw_thread(void *info)
 174 {
 175         void fsl_secondary_thread_init(void);
 176         unsigned long imsr, inia;
 177         int nr = *(const int *)info;
 178
 179         imsr = MSR_KERNEL;
 180         inia = *(unsigned long *)fsl_secondary_thread_init;
 181
 182         if (cpu_thread_in_core(nr) == 0) {
 183                 /* For when we boot on a secondary thread with kdump */
 184                 mttmr(TMRN_IMSR0, imsr);
 185                 mttmr(TMRN_INIA0, inia);
 186                 mtspr(SPRN_TENS, TEN_THREAD(0));
 187         } else {
 188                 mttmr(TMRN_IMSR1, imsr);
 189                 mttmr(TMRN_INIA1, inia);
 190                 mtspr(SPRN_TENS, TEN_THREAD(1));
 191         }
 192
 193         smp_generic_kick_cpu(nr);
 194 }
 195 #endif
 196
 197 static int smp_85xx_kick_cpu(int nr)
 198 {
 199         unsigned long flags;
 200         const u64 *cpu_rel_addr;
 201         __iomem struct epapr_spin_table *spin_table;
 202         struct device_node *np;
 203         int hw_cpu = get_hard_smp_processor_id(nr);
 204         int ioremappable;
 205         int ret = 0;
 206
 207         WARN_ON(nr < 0 || nr >= NR_CPUS);
 208         WARN_ON(hw_cpu < 0 || hw_cpu >= NR_CPUS);
 209
 210         pr_debug("smp_85xx_kick_cpu: kick CPU #%d\n", nr);
 211
 212 #ifdef CONFIG_PPC64
 213         /* Threads don't use the spin table */
 214         if (cpu_thread_in_core(nr) != 0) {
 215                 int primary = cpu_first_thread_sibling(nr);
 216
 217                 if (WARN_ON_ONCE(!cpu_has_feature(CPU_FTR_SMT)))
 218                         return -ENOENT;
 219
 220                 if (cpu_thread_in_core(nr) != 1) {
 221                         pr_err("%s: cpu %d: invalid hw thread %d\n",
 222                                __func__, nr, cpu_thread_in_core(nr));
 223                         return -ENOENT;
 224                 }
 225
 226                 if (!cpu_online(primary)) {
 227                         pr_err("%s: cpu %d: primary %d not online\n",
 228                                __func__, nr, primary);
 229                         return -ENOENT;
 230                 }
 231
 232                 smp_call_function_single(primary, wake_hw_thread, &nr, 0);
 233                 return 0;
 234         } else if (cpu_thread_in_core(boot_cpuid) != 0 &&
 235                    cpu_first_thread_sibling(boot_cpuid) == nr) {
 236                 if (WARN_ON_ONCE(!cpu_has_feature(CPU_FTR_SMT)))
 237                         return -ENOENT;
 238
 239                 smp_call_function_single(boot_cpuid, wake_hw_thread, &nr, 0);
 240         }
 241 #endif
 242
 243         np = of_get_cpu_node(nr, NULL);
 244         cpu_rel_addr = of_get_property(np, "cpu-release-addr", NULL);
 245
 246         if (cpu_rel_addr == NULL) {
 247                 printk(KERN_ERR "No cpu-release-addr for cpu %d\n", nr);
 248                 return -ENOENT;
 249         }
 250
 251         /*
 252          * A secondary core could be in a spinloop in the bootpage
 253          * (0xfffff000), somewhere in highmem, or somewhere in lowmem.
 254          * The bootpage and highmem can be accessed via ioremap(), but
 255          * we need to directly access the spinloop if its in lowmem.
 256          */
 257         ioremappable = *cpu_rel_addr > virt_to_phys(high_memory);
 258
 259         /* Map the spin table */
 260         if (ioremappable)
 261                 spin_table = ioremap_prot(*cpu_rel_addr,
 262                         sizeof(struct epapr_spin_table), _PAGE_COHERENT);
 263         else
 264                 spin_table = phys_to_virt(*cpu_rel_addr);
 265
 266         local_irq_save(flags);
 267 #ifdef CONFIG_PPC32
 268 #ifdef CONFIG_HOTPLUG_CPU
 269         /* Corresponding to generic_set_cpu_dead() */
 270         generic_set_cpu_up(nr);
 271
 272         if (system_state == SYSTEM_RUNNING) {
 273                 /*
 274                  * To keep it compatible with old boot program which uses
 275                  * cache-inhibit spin table, we need to flush the cache
 276                  * before accessing spin table to invalidate any staled data.
 277                  * We also need to flush the cache after writing to spin
 278                  * table to push data out.
 279                  */
 280                 flush_spin_table(spin_table);
 281                 out_be32(&spin_table->addr_l, 0);
 282                 flush_spin_table(spin_table);
 283
 284                 /*
 285                  * We don't set the BPTR register here since it already points
 286                  * to the boot page properly.
 287                  */
 288                 mpic_reset_core(nr);
 289
 290                 /*
 291                  * wait until core is ready...
 292                  * We need to invalidate the stale data, in case the boot
 293                  * loader uses a cache-inhibited spin table.
 294                  */
 295                 if (!spin_event_timeout(
 296                                 read_spin_table_addr_l(spin_table) == 1,
 297                                 10000, 100)) {
 298                         pr_err("%s: timeout waiting for core %d to reset\n",
 299                                                         __func__, hw_cpu);
 300                         ret = -ENOENT;
 301                         goto out;
 302                 }
 303
 304                 /*  clear the acknowledge status */
 305                 __secondary_hold_acknowledge = -1;
 306         }
 307 #endif
 308         flush_spin_table(spin_table);
 309         out_be32(&spin_table->pir, hw_cpu);
 310         out_be32(&spin_table->addr_l, __pa(__early_start));
 311         flush_spin_table(spin_table);
 312
 313         /* Wait a bit for the CPU to ack. */
 314         if (!spin_event_timeout(__secondary_hold_acknowledge == hw_cpu,
 315                                         10000, 100)) {
 316                 pr_err("%s: timeout waiting for core %d to ack\n",
 317                                                 __func__, hw_cpu);
 318                 ret = -ENOENT;
 319                 goto out;
 320         }
 321 out:
 322 #else
 323         smp_generic_kick_cpu(nr);
 324
 325         flush_spin_table(spin_table);
 326         out_be32(&spin_table->pir, hw_cpu);
 327         out_be64((u64 *)(&spin_table->addr_h),
 328                 __pa(ppc_function_entry(generic_secondary_smp_init)));
 329         flush_spin_table(spin_table);
 330 #endif
 331
 332         local_irq_restore(flags);
 333
 334         if (ioremappable)
 335                 iounmap(spin_table);
 336
 337         return ret;
 338 }
 339
 340 struct smp_ops_t smp_85xx_ops = {
 341         .kick_cpu = smp_85xx_kick_cpu,
 342         .cpu_bootable = smp_generic_cpu_bootable,
 343 #ifdef CONFIG_HOTPLUG_CPU
 344         .cpu_disable    = generic_cpu_disable,
 345         .cpu_die        = generic_cpu_die,
 346 #endif
 347 #if defined(CONFIG_KEXEC) && !defined(CONFIG_PPC64)
 348         .give_timebase  = smp_generic_give_timebase,
 349         .take_timebase  = smp_generic_take_timebase,
 350 #endif
 351 };
 352
 353 #ifdef CONFIG_KEXEC
 354 #ifdef CONFIG_PPC32
 355 atomic_t kexec_down_cpus = ATOMIC_INIT(0);
 356
 357 void mpc85xx_smp_kexec_cpu_down(int crash_shutdown, int secondary)
 358 {
 359         local_irq_disable();
 360
 361         if (secondary) {
 362                 __flush_disable_L1();
 363                 atomic_inc(&kexec_down_cpus);
 364                 /* loop forever */
 365                 while (1);
 366         }
 367 }
 368
 369 static void mpc85xx_smp_kexec_down(void *arg)
 370 {
 371         if (ppc_md.kexec_cpu_down)
 372                 ppc_md.kexec_cpu_down(0,1);
 373 }
 374 #else
 375 void mpc85xx_smp_kexec_cpu_down(int crash_shutdown, int secondary)
 376 {
 377         int cpu = smp_processor_id();
 378         int sibling = cpu_last_thread_sibling(cpu);
 379         bool notified = false;
 380         int disable_cpu;
 381         int disable_threadbit = 0;
 382         long start = mftb();
 383         long now;
 384
 385         local_irq_disable();
 386         hard_irq_disable();
 387         mpic_teardown_this_cpu(secondary);
 388
 389         if (cpu == crashing_cpu && cpu_thread_in_core(cpu) != 0) {
 390                 /*
 391                  * We enter the crash kernel on whatever cpu crashed,
 392                  * even if it's a secondary thread.  If that's the case,
 393                  * disable the corresponding primary thread.
 394                  */
 395                 disable_threadbit = 1;
 396                 disable_cpu = cpu_first_thread_sibling(cpu);
 397         } else if (sibling != crashing_cpu &&
 398                    cpu_thread_in_core(cpu) == 0 &&
 399                    cpu_thread_in_core(sibling) != 0) {
 400                 disable_threadbit = 2;
 401                 disable_cpu = sibling;
 402         }
 403
 404         if (disable_threadbit) {
 405                 while (paca[disable_cpu].kexec_state < KEXEC_STATE_REAL_MODE) {
 406                         barrier();
 407                         now = mftb();
 408                         if (!notified && now - start > 1000000) {
 409                                 pr_info("%s/%d: waiting for cpu %d to enter KEXEC_STATE_REAL_MODE (%d)\n",
 410                                         __func__, smp_processor_id(),
 411                                         disable_cpu,
 412                                         paca[disable_cpu].kexec_state);
 413                                 notified = true;
 414                         }
 415                 }
 416
 417                 if (notified) {
 418                         pr_info("%s: cpu %d done waiting\n",
 419                                 __func__, disable_cpu);
 420                 }
 421
 422                 mtspr(SPRN_TENC, disable_threadbit);
 423                 while (mfspr(SPRN_TENSR) & disable_threadbit)
 424                         cpu_relax();
 425         }
 426 }
 427 #endif
 428
 429 static void mpc85xx_smp_machine_kexec(struct kimage *image)
 430 {
 431 #ifdef CONFIG_PPC32
 432         int timeout = INT_MAX;
 433         int i, num_cpus = num_present_cpus();
 434
 435         if (image->type == KEXEC_TYPE_DEFAULT)
 436                 smp_call_function(mpc85xx_smp_kexec_down, NULL, 0);
 437
 438         while ( (atomic_read(&kexec_down_cpus) != (num_cpus - 1)) &&
 439                 ( timeout > 0 ) )
 440         {
 441                 timeout--;
 442         }
 443
 444         if ( !timeout )
 445                 printk(KERN_ERR "Unable to bring down secondary cpu(s)");
 446
 447         for_each_online_cpu(i)
 448         {
 449                 if ( i == smp_processor_id() ) continue;
 450                 mpic_reset_core(i);
 451         }
 452 #endif
 453
 454         default_machine_kexec(image);
 455 }
 456 #endif /* CONFIG_KEXEC */
 457
 458 static void smp_85xx_basic_setup(int cpu_nr)
 459 {
 460         if (cpu_has_feature(CPU_FTR_DBELL))
 461                 doorbell_setup_this_cpu();
 462 }
 463
 464 static void smp_85xx_setup_cpu(int cpu_nr)
 465 {
 466         mpic_setup_this_cpu();
 467         smp_85xx_basic_setup(cpu_nr);
 468 }
 469
 470 static const struct of_device_id mpc85xx_smp_guts_ids[] = {
 471         { .compatible = "fsl,mpc8572-guts", },
 472         { .compatible = "fsl,p1020-guts", },
 473         { .compatible = "fsl,p1021-guts", },
 474         { .compatible = "fsl,p1022-guts", },
 475         { .compatible = "fsl,p1023-guts", },
 476         { .compatible = "fsl,p2020-guts", },
 477         {},
 478 };
 479
 480 void __init mpc85xx_smp_init(void)
 481 {
 482         struct device_node *np;
 483
 484
 485         np = of_find_node_by_type(NULL, "open-pic");
 486         if (np) {
 487                 smp_85xx_ops.probe = smp_mpic_probe;
 488                 smp_85xx_ops.setup_cpu = smp_85xx_setup_cpu;
 489                 smp_85xx_ops.message_pass = smp_mpic_message_pass;
 490         } else
 491                 smp_85xx_ops.setup_cpu = smp_85xx_basic_setup;
 492
 493         if (cpu_has_feature(CPU_FTR_DBELL)) {
 494                 /*
 495                  * If left NULL, .message_pass defaults to
 496                  * smp_muxed_ipi_message_pass
 497                  */
 498                 smp_85xx_ops.message_pass = NULL;
 499                 smp_85xx_ops.cause_ipi = doorbell_cause_ipi;
 500                 smp_85xx_ops.probe = NULL;
 501         }
 502
 503         np = of_find_matching_node(NULL, mpc85xx_smp_guts_ids);
 504         if (np) {
 505                 guts = of_iomap(np, 0);
 506                 of_node_put(np);
 507                 if (!guts) {
 508                         pr_err("%s: Could not map guts node address\n",
 509                                                                 __func__);
 510                         return;
 511                 }
 512                 smp_85xx_ops.give_timebase = mpc85xx_give_timebase;
 513                 smp_85xx_ops.take_timebase = mpc85xx_take_timebase;
 514 #ifdef CONFIG_HOTPLUG_CPU
 515                 ppc_md.cpu_die = smp_85xx_mach_cpu_die;
 516 #endif
 517         }
 518
 519         smp_ops = &smp_85xx_ops;
 520
 521 #ifdef CONFIG_KEXEC
 522         ppc_md.kexec_cpu_down = mpc85xx_smp_kexec_cpu_down;
 523         ppc_md.machine_kexec = mpc85xx_smp_machine_kexec;
 524 #endif
 525 }