kernel/arch/xtensa/kernel/smp.c

   1 /*
   2  * Xtensa SMP support functions.
   3  *
   4  * This file is subject to the terms and conditions of the GNU General Public
   5  * License.  See the file "COPYING" in the main directory of this archive
   6  * for more details.
   7  *
   8  * Copyright (C) 2008 - 2013 Tensilica Inc.
   9  *
  10  * Chris Zankel <chris@zankel.net>
  11  * Joe Taylor <joe@tensilica.com>
  12  * Pete Delaney <piet@tensilica.com
  13  */
  14
  15 #include <linux/cpu.h>
  16 #include <linux/cpumask.h>
  17 #include <linux/delay.h>
  18 #include <linux/init.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/irqdomain.h>
  21 #include <linux/irq.h>
  22 #include <linux/kdebug.h>
  23 #include <linux/module.h>
  24 #include <linux/reboot.h>
  25 #include <linux/seq_file.h>
  26 #include <linux/smp.h>
  27 #include <linux/thread_info.h>
  28
  29 #include <asm/cacheflush.h>
  30 #include <asm/kdebug.h>
  31 #include <asm/mmu_context.h>
  32 #include <asm/mxregs.h>
  33 #include <asm/platform.h>
  34 #include <asm/tlbflush.h>
  35 #include <asm/traps.h>
  36
  37 #ifdef CONFIG_SMP
  38 # if XCHAL_HAVE_S32C1I == 0
  39 #  error "The S32C1I option is required for SMP."
  40 # endif
  41 #endif
  42
  43 static void system_invalidate_dcache_range(unsigned long start,
  44                 unsigned long size);
  45 static void system_flush_invalidate_dcache_range(unsigned long start,
  46                 unsigned long size);
  47
  48 /* IPI (Inter Process Interrupt) */
  49
  50 #define IPI_IRQ 0
  51
  52 static irqreturn_t ipi_interrupt(int irq, void *dev_id);
  53 static struct irqaction ipi_irqaction = {
  54         .handler =      ipi_interrupt,
  55         .flags =        IRQF_PERCPU,
  56         .name =         "ipi",
  57 };
  58
  59 void ipi_init(void)
  60 {
  61         unsigned irq = irq_create_mapping(NULL, IPI_IRQ);
  62         setup_irq(irq, &ipi_irqaction);
  63 }
  64
  65 static inline unsigned int get_core_count(void)
  66 {
  67         /* Bits 18..21 of SYSCFGID contain the core count minus 1. */
  68         unsigned int syscfgid = get_er(SYSCFGID);
  69         return ((syscfgid >> 18) & 0xf) + 1;
  70 }
  71
  72 static inline int get_core_id(void)
  73 {
  74         /* Bits 0...18 of SYSCFGID contain the core id  */
  75         unsigned int core_id = get_er(SYSCFGID);
  76         return core_id & 0x3fff;
  77 }
  78
  79 void __init smp_prepare_cpus(unsigned int max_cpus)
  80 {
  81         unsigned i;
  82
  83         for (i = 0; i < max_cpus; ++i)
  84                 set_cpu_present(i, true);
  85 }
  86
  87 void __init smp_init_cpus(void)
  88 {
  89         unsigned i;
  90         unsigned int ncpus = get_core_count();
  91         unsigned int core_id = get_core_id();
  92
  93         pr_info("%s: Core Count = %d\n", __func__, ncpus);
  94         pr_info("%s: Core Id = %d\n", __func__, core_id);
  95
  96         for (i = 0; i < ncpus; ++i)
  97                 set_cpu_possible(i, true);
  98 }
  99
 100 void __init smp_prepare_boot_cpu(void)
 101 {
 102         unsigned int cpu = smp_processor_id();
 103         BUG_ON(cpu != 0);
 104         cpu_asid_cache(cpu) = ASID_USER_FIRST;
 105 }
 106
 107 void __init smp_cpus_done(unsigned int max_cpus)
 108 {
 109 }
 110
 111 static int boot_secondary_processors = 1; /* Set with xt-gdb via .xt-gdb */
 112 static DECLARE_COMPLETION(cpu_running);
 113
 114 void secondary_start_kernel(void)
 115 {
 116         struct mm_struct *mm = &init_mm;
 117         unsigned int cpu = smp_processor_id();
 118
 119         init_mmu();
 120
 121 #ifdef CONFIG_DEBUG_KERNEL
 122         if (boot_secondary_processors == 0) {
 123                 pr_debug("%s: boot_secondary_processors:%d; Hanging cpu:%d\n",
 124                         __func__, boot_secondary_processors, cpu);
 125                 for (;;)
 126                         __asm__ __volatile__ ("waiti " __stringify(LOCKLEVEL));
 127         }
 128
 129         pr_debug("%s: boot_secondary_processors:%d; Booting cpu:%d\n",
 130                 __func__, boot_secondary_processors, cpu);
 131 #endif
 132         /* Init EXCSAVE1 */
 133
 134         secondary_trap_init();
 135
 136         /* All kernel threads share the same mm context. */
 137
 138         atomic_inc(&mm->mm_users);
 139         atomic_inc(&mm->mm_count);
 140         current->active_mm = mm;
 141         cpumask_set_cpu(cpu, mm_cpumask(mm));
 142         enter_lazy_tlb(mm, current);
 143
 144         preempt_disable();
 145         trace_hardirqs_off();
 146
 147         calibrate_delay();
 148
 149         notify_cpu_starting(cpu);
 150
 151         secondary_init_irq();
 152         local_timer_setup(cpu);
 153
 154         set_cpu_online(cpu, true);
 155
 156         local_irq_enable();
 157
 158         complete(&cpu_running);
 159
 160         cpu_startup_entry(CPUHP_ONLINE);
 161 }
 162
 163 static void mx_cpu_start(void *p)
 164 {
 165         unsigned cpu = (unsigned)p;
 166         unsigned long run_stall_mask = get_er(MPSCORE);
 167
 168         set_er(run_stall_mask & ~(1u << cpu), MPSCORE);
 169         pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
 170                         __func__, cpu, run_stall_mask, get_er(MPSCORE));
 171 }
 172
 173 static void mx_cpu_stop(void *p)
 174 {
 175         unsigned cpu = (unsigned)p;
 176         unsigned long run_stall_mask = get_er(MPSCORE);
 177
 178         set_er(run_stall_mask | (1u << cpu), MPSCORE);
 179         pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
 180                         __func__, cpu, run_stall_mask, get_er(MPSCORE));
 181 }
 182
 183 #ifdef CONFIG_HOTPLUG_CPU
 184 unsigned long cpu_start_id __cacheline_aligned;
 185 #endif
 186 unsigned long cpu_start_ccount;
 187
 188 static int boot_secondary(unsigned int cpu, struct task_struct *ts)
 189 {
 190         unsigned long timeout = jiffies + msecs_to_jiffies(1000);
 191         unsigned long ccount;
 192         int i;
 193
 194 #ifdef CONFIG_HOTPLUG_CPU
 195         cpu_start_id = cpu;
 196         system_flush_invalidate_dcache_range(
 197                         (unsigned long)&cpu_start_id, sizeof(cpu_start_id));
 198 #endif
 199         smp_call_function_single(0, mx_cpu_start, (void *)cpu, 1);
 200
 201         for (i = 0; i < 2; ++i) {
 202                 do
 203                         ccount = get_ccount();
 204                 while (!ccount);
 205
 206                 cpu_start_ccount = ccount;
 207
 208                 while (time_before(jiffies, timeout)) {
 209                         mb();
 210                         if (!cpu_start_ccount)
 211                                 break;
 212                 }
 213
 214                 if (cpu_start_ccount) {
 215                         smp_call_function_single(0, mx_cpu_stop,
 216                                         (void *)cpu, 1);
 217                         cpu_start_ccount = 0;
 218                         return -EIO;
 219                 }
 220         }
 221         return 0;
 222 }
 223
 224 int __cpu_up(unsigned int cpu, struct task_struct *idle)
 225 {
 226         int ret = 0;
 227
 228         if (cpu_asid_cache(cpu) == 0)
 229                 cpu_asid_cache(cpu) = ASID_USER_FIRST;
 230
 231         start_info.stack = (unsigned long)task_pt_regs(idle);
 232         wmb();
 233
 234         pr_debug("%s: Calling wakeup_secondary(cpu:%d, idle:%p, sp: %08lx)\n",
 235                         __func__, cpu, idle, start_info.stack);
 236
 237         ret = boot_secondary(cpu, idle);
 238         if (ret == 0) {
 239                 wait_for_completion_timeout(&cpu_running,
 240                                 msecs_to_jiffies(1000));
 241                 if (!cpu_online(cpu))
 242                         ret = -EIO;
 243         }
 244
 245         if (ret)
 246                 pr_err("CPU %u failed to boot\n", cpu);
 247
 248         return ret;
 249 }
 250
 251 #ifdef CONFIG_HOTPLUG_CPU
 252
 253 /*
 254  * __cpu_disable runs on the processor to be shutdown.
 255  */
 256 int __cpu_disable(void)
 257 {
 258         unsigned int cpu = smp_processor_id();
 259
 260         /*
 261          * Take this CPU offline.  Once we clear this, we can't return,
 262          * and we must not schedule until we're ready to give up the cpu.
 263          */
 264         set_cpu_online(cpu, false);
 265
 266         /*
 267          * OK - migrate IRQs away from this CPU
 268          */
 269         migrate_irqs();
 270
 271         /*
 272          * Flush user cache and TLB mappings, and then remove this CPU
 273          * from the vm mask set of all processes.
 274          */
 275         local_flush_cache_all();
 276         local_flush_tlb_all();
 277         invalidate_page_directory();
 278
 279         clear_tasks_mm_cpumask(cpu);
 280
 281         return 0;
 282 }
 283
 284 static void platform_cpu_kill(unsigned int cpu)
 285 {
 286         smp_call_function_single(0, mx_cpu_stop, (void *)cpu, true);
 287 }
 288
 289 /*
 290  * called on the thread which is asking for a CPU to be shutdown -
 291  * waits until shutdown has completed, or it is timed out.
 292  */
 293 void __cpu_die(unsigned int cpu)
 294 {
 295         unsigned long timeout = jiffies + msecs_to_jiffies(1000);
 296         while (time_before(jiffies, timeout)) {
 297                 system_invalidate_dcache_range((unsigned long)&cpu_start_id,
 298                                 sizeof(cpu_start_id));
 299                 if (cpu_start_id == -cpu) {
 300                         platform_cpu_kill(cpu);
 301                         return;
 302                 }
 303         }
 304         pr_err("CPU%u: unable to kill\n", cpu);
 305 }
 306
 307 void arch_cpu_idle_dead(void)
 308 {
 309         cpu_die();
 310 }
 311 /*
 312  * Called from the idle thread for the CPU which has been shutdown.
 313  *
 314  * Note that we disable IRQs here, but do not re-enable them
 315  * before returning to the caller. This is also the behaviour
 316  * of the other hotplug-cpu capable cores, so presumably coming
 317  * out of idle fixes this.
 318  */
 319 void __ref cpu_die(void)
 320 {
 321         idle_task_exit();
 322         local_irq_disable();
 323         __asm__ __volatile__(
 324                         "       movi    a2, cpu_restart\n"
 325                         "       jx      a2\n");
 326 }
 327
 328 #endif /* CONFIG_HOTPLUG_CPU */
 329
 330 enum ipi_msg_type {
 331         IPI_RESCHEDULE = 0,
 332         IPI_CALL_FUNC,
 333         IPI_CPU_STOP,
 334         IPI_MAX
 335 };
 336
 337 static const struct {
 338         const char *short_text;
 339         const char *long_text;
 340 } ipi_text[] = {
 341         { .short_text = "RES", .long_text = "Rescheduling interrupts" },
 342         { .short_text = "CAL", .long_text = "Function call interrupts" },
 343         { .short_text = "DIE", .long_text = "CPU shutdown interrupts" },
 344 };
 345
 346 struct ipi_data {
 347         unsigned long ipi_count[IPI_MAX];
 348 };
 349
 350 static DEFINE_PER_CPU(struct ipi_data, ipi_data);
 351
 352 static void send_ipi_message(const struct cpumask *callmask,
 353                 enum ipi_msg_type msg_id)
 354 {
 355         int index;
 356         unsigned long mask = 0;
 357
 358         for_each_cpu(index, callmask)
 359                 if (index != smp_processor_id())
 360                         mask |= 1 << index;
 361
 362         set_er(mask, MIPISET(msg_id));
 363 }
 364
 365 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
 366 {
 367         send_ipi_message(mask, IPI_CALL_FUNC);
 368 }
 369
 370 void arch_send_call_function_single_ipi(int cpu)
 371 {
 372         send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
 373 }
 374
 375 void smp_send_reschedule(int cpu)
 376 {
 377         send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
 378 }
 379
 380 void smp_send_stop(void)
 381 {
 382         struct cpumask targets;
 383
 384         cpumask_copy(&targets, cpu_online_mask);
 385         cpumask_clear_cpu(smp_processor_id(), &targets);
 386         send_ipi_message(&targets, IPI_CPU_STOP);
 387 }
 388
 389 static void ipi_cpu_stop(unsigned int cpu)
 390 {
 391         set_cpu_online(cpu, false);
 392         machine_halt();
 393 }
 394
 395 irqreturn_t ipi_interrupt(int irq, void *dev_id)
 396 {
 397         unsigned int cpu = smp_processor_id();
 398         struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
 399         unsigned int msg;
 400         unsigned i;
 401
 402         msg = get_er(MIPICAUSE(cpu));
 403         for (i = 0; i < IPI_MAX; i++)
 404                 if (msg & (1 << i)) {
 405                         set_er(1 << i, MIPICAUSE(cpu));
 406                         ++ipi->ipi_count[i];
 407                 }
 408
 409         if (msg & (1 << IPI_RESCHEDULE))
 410                 scheduler_ipi();
 411         if (msg & (1 << IPI_CALL_FUNC))
 412                 generic_smp_call_function_interrupt();
 413         if (msg & (1 << IPI_CPU_STOP))
 414                 ipi_cpu_stop(cpu);
 415
 416         return IRQ_HANDLED;
 417 }
 418
 419 void show_ipi_list(struct seq_file *p, int prec)
 420 {
 421         unsigned int cpu;
 422         unsigned i;
 423
 424         for (i = 0; i < IPI_MAX; ++i) {
 425                 seq_printf(p, "%*s:", prec, ipi_text[i].short_text);
 426                 for_each_online_cpu(cpu)
 427                         seq_printf(p, " %10lu",
 428                                         per_cpu(ipi_data, cpu).ipi_count[i]);
 429                 seq_printf(p, "   %s\n", ipi_text[i].long_text);
 430         }
 431 }
 432
 433 int setup_profiling_timer(unsigned int multiplier)
 434 {
 435         pr_debug("setup_profiling_timer %d\n", multiplier);
 436         return 0;
 437 }
 438
 439 /* TLB flush functions */
 440
 441 struct flush_data {
 442         struct vm_area_struct *vma;
 443         unsigned long addr1;
 444         unsigned long addr2;
 445 };
 446
 447 static void ipi_flush_tlb_all(void *arg)
 448 {
 449         local_flush_tlb_all();
 450 }
 451
 452 void flush_tlb_all(void)
 453 {
 454         on_each_cpu(ipi_flush_tlb_all, NULL, 1);
 455 }
 456
 457 static void ipi_flush_tlb_mm(void *arg)
 458 {
 459         local_flush_tlb_mm(arg);
 460 }
 461
 462 void flush_tlb_mm(struct mm_struct *mm)
 463 {
 464         on_each_cpu(ipi_flush_tlb_mm, mm, 1);
 465 }
 466
 467 static void ipi_flush_tlb_page(void *arg)
 468 {
 469         struct flush_data *fd = arg;
 470         local_flush_tlb_page(fd->vma, fd->addr1);
 471 }
 472
 473 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
 474 {
 475         struct flush_data fd = {
 476                 .vma = vma,
 477                 .addr1 = addr,
 478         };
 479         on_each_cpu(ipi_flush_tlb_page, &fd, 1);
 480 }
 481
 482 static void ipi_flush_tlb_range(void *arg)
 483 {
 484         struct flush_data *fd = arg;
 485         local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
 486 }
 487
 488 void flush_tlb_range(struct vm_area_struct *vma,
 489                      unsigned long start, unsigned long end)
 490 {
 491         struct flush_data fd = {
 492                 .vma = vma,
 493                 .addr1 = start,
 494                 .addr2 = end,
 495         };
 496         on_each_cpu(ipi_flush_tlb_range, &fd, 1);
 497 }
 498
 499 static void ipi_flush_tlb_kernel_range(void *arg)
 500 {
 501         struct flush_data *fd = arg;
 502         local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
 503 }
 504
 505 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 506 {
 507         struct flush_data fd = {
 508                 .addr1 = start,
 509                 .addr2 = end,
 510         };
 511         on_each_cpu(ipi_flush_tlb_kernel_range, &fd, 1);
 512 }
 513
 514 /* Cache flush functions */
 515
 516 static void ipi_flush_cache_all(void *arg)
 517 {
 518         local_flush_cache_all();
 519 }
 520
 521 void flush_cache_all(void)
 522 {
 523         on_each_cpu(ipi_flush_cache_all, NULL, 1);
 524 }
 525
 526 static void ipi_flush_cache_page(void *arg)
 527 {
 528         struct flush_data *fd = arg;
 529         local_flush_cache_page(fd->vma, fd->addr1, fd->addr2);
 530 }
 531
 532 void flush_cache_page(struct vm_area_struct *vma,
 533                      unsigned long address, unsigned long pfn)
 534 {
 535         struct flush_data fd = {
 536                 .vma = vma,
 537                 .addr1 = address,
 538                 .addr2 = pfn,
 539         };
 540         on_each_cpu(ipi_flush_cache_page, &fd, 1);
 541 }
 542
 543 static void ipi_flush_cache_range(void *arg)
 544 {
 545         struct flush_data *fd = arg;
 546         local_flush_cache_range(fd->vma, fd->addr1, fd->addr2);
 547 }
 548
 549 void flush_cache_range(struct vm_area_struct *vma,
 550                      unsigned long start, unsigned long end)
 551 {
 552         struct flush_data fd = {
 553                 .vma = vma,
 554                 .addr1 = start,
 555                 .addr2 = end,
 556         };
 557         on_each_cpu(ipi_flush_cache_range, &fd, 1);
 558 }
 559
 560 static void ipi_flush_icache_range(void *arg)
 561 {
 562         struct flush_data *fd = arg;
 563         local_flush_icache_range(fd->addr1, fd->addr2);
 564 }
 565
 566 void flush_icache_range(unsigned long start, unsigned long end)
 567 {
 568         struct flush_data fd = {
 569                 .addr1 = start,
 570                 .addr2 = end,
 571         };
 572         on_each_cpu(ipi_flush_icache_range, &fd, 1);
 573 }
 574 EXPORT_SYMBOL(flush_icache_range);
 575
 576 /* ------------------------------------------------------------------------- */
 577
 578 static void ipi_invalidate_dcache_range(void *arg)
 579 {
 580         struct flush_data *fd = arg;
 581         __invalidate_dcache_range(fd->addr1, fd->addr2);
 582 }
 583
 584 static void system_invalidate_dcache_range(unsigned long start,
 585                 unsigned long size)
 586 {
 587         struct flush_data fd = {
 588                 .addr1 = start,
 589                 .addr2 = size,
 590         };
 591         on_each_cpu(ipi_invalidate_dcache_range, &fd, 1);
 592 }
 593
 594 static void ipi_flush_invalidate_dcache_range(void *arg)
 595 {
 596         struct flush_data *fd = arg;
 597         __flush_invalidate_dcache_range(fd->addr1, fd->addr2);
 598 }
 599
 600 static void system_flush_invalidate_dcache_range(unsigned long start,
 601                 unsigned long size)
 602 {
 603         struct flush_data fd = {
 604                 .addr1 = start,
 605                 .addr2 = size,
 606         };
 607         on_each_cpu(ipi_flush_invalidate_dcache_range, &fd, 1);
 608 }