kernel/arch/arm/kernel/smp_tlb.c

   1 /*
   2  *  linux/arch/arm/kernel/smp_tlb.c
   3  *
   4  *  Copyright (C) 2002 ARM Limited, All Rights Reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10 #include <linux/preempt.h>
  11 #include <linux/smp.h>
  12 #include <linux/uaccess.h>
  13
  14 #include <asm/smp_plat.h>
  15 #include <asm/tlbflush.h>
  16 #include <asm/mmu_context.h>
  17
  18 /**********************************************************************/
  19
  20 /*
  21  * TLB operations
  22  */
  23 struct tlb_args {
  24         struct vm_area_struct *ta_vma;
  25         unsigned long ta_start;
  26         unsigned long ta_end;
  27 };
  28
  29 static inline void ipi_flush_tlb_all(void *ignored)
  30 {
  31         local_flush_tlb_all();
  32 }
  33
  34 static inline void ipi_flush_tlb_mm(void *arg)
  35 {
  36         struct mm_struct *mm = (struct mm_struct *)arg;
  37
  38         local_flush_tlb_mm(mm);
  39 }
  40
  41 static inline void ipi_flush_tlb_page(void *arg)
  42 {
  43         struct tlb_args *ta = (struct tlb_args *)arg;
  44         unsigned int __ua_flags = uaccess_save_and_enable();
  45
  46         local_flush_tlb_page(ta->ta_vma, ta->ta_start);
  47
  48         uaccess_restore(__ua_flags);
  49 }
  50
  51 static inline void ipi_flush_tlb_kernel_page(void *arg)
  52 {
  53         struct tlb_args *ta = (struct tlb_args *)arg;
  54
  55         local_flush_tlb_kernel_page(ta->ta_start);
  56 }
  57
  58 static inline void ipi_flush_tlb_range(void *arg)
  59 {
  60         struct tlb_args *ta = (struct tlb_args *)arg;
  61         unsigned int __ua_flags = uaccess_save_and_enable();
  62
  63         local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
  64
  65         uaccess_restore(__ua_flags);
  66 }
  67
  68 static inline void ipi_flush_tlb_kernel_range(void *arg)
  69 {
  70         struct tlb_args *ta = (struct tlb_args *)arg;
  71
  72         local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
  73 }
  74
  75 static inline void ipi_flush_bp_all(void *ignored)
  76 {
  77         local_flush_bp_all();
  78 }
  79
  80 #ifdef CONFIG_ARM_ERRATA_798181
  81 bool (*erratum_a15_798181_handler)(void);
  82
  83 static bool erratum_a15_798181_partial(void)
  84 {
  85         asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0));
  86         dsb(ish);
  87         return false;
  88 }
  89
  90 static bool erratum_a15_798181_broadcast(void)
  91 {
  92         asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0));
  93         dsb(ish);
  94         return true;
  95 }
  96
  97 void erratum_a15_798181_init(void)
  98 {
  99         unsigned int midr = read_cpuid_id();
 100         unsigned int revidr = read_cpuid(CPUID_REVIDR);
 101
 102         /* Brahma-B15 r0p0..r0p2 affected
 103          * Cortex-A15 r0p0..r3p2 w/o ECO fix affected */
 104         if ((midr & 0xff0ffff0) == 0x420f00f0 && midr <= 0x420f00f2)
 105                 erratum_a15_798181_handler = erratum_a15_798181_broadcast;
 106         else if ((midr & 0xff0ffff0) == 0x410fc0f0 && midr <= 0x413fc0f2 &&
 107                  (revidr & 0x210) != 0x210) {
 108                 if (revidr & 0x10)
 109                         erratum_a15_798181_handler =
 110                                 erratum_a15_798181_partial;
 111                 else
 112                         erratum_a15_798181_handler =
 113                                 erratum_a15_798181_broadcast;
 114         }
 115 }
 116 #endif
 117
 118 static void ipi_flush_tlb_a15_erratum(void *arg)
 119 {
 120         dmb();
 121 }
 122
 123 static void broadcast_tlb_a15_erratum(void)
 124 {
 125         if (!erratum_a15_798181())
 126                 return;
 127
 128         smp_call_function(ipi_flush_tlb_a15_erratum, NULL, 1);
 129 }
 130
 131 static void broadcast_tlb_mm_a15_erratum(struct mm_struct *mm)
 132 {
 133         int this_cpu;
 134         cpumask_t mask = { CPU_BITS_NONE };
 135
 136         if (!erratum_a15_798181())
 137                 return;
 138
 139         this_cpu = get_cpu();
 140         a15_erratum_get_cpumask(this_cpu, mm, &mask);
 141         smp_call_function_many(&mask, ipi_flush_tlb_a15_erratum, NULL, 1);
 142         put_cpu();
 143 }
 144
 145 void flush_tlb_all(void)
 146 {
 147         if (tlb_ops_need_broadcast())
 148                 on_each_cpu(ipi_flush_tlb_all, NULL, 1);
 149         else
 150                 __flush_tlb_all();
 151         broadcast_tlb_a15_erratum();
 152 }
 153
 154 void flush_tlb_mm(struct mm_struct *mm)
 155 {
 156         if (tlb_ops_need_broadcast())
 157                 on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1);
 158         else
 159                 __flush_tlb_mm(mm);
 160         broadcast_tlb_mm_a15_erratum(mm);
 161 }
 162
 163 void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
 164 {
 165         if (tlb_ops_need_broadcast()) {
 166                 struct tlb_args ta;
 167                 ta.ta_vma = vma;
 168                 ta.ta_start = uaddr;
 169                 on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_page,
 170                                         &ta, 1);
 171         } else
 172                 __flush_tlb_page(vma, uaddr);
 173         broadcast_tlb_mm_a15_erratum(vma->vm_mm);
 174 }
 175
 176 void flush_tlb_kernel_page(unsigned long kaddr)
 177 {
 178         if (tlb_ops_need_broadcast()) {
 179                 struct tlb_args ta;
 180                 ta.ta_start = kaddr;
 181                 on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
 182         } else
 183                 __flush_tlb_kernel_page(kaddr);
 184         broadcast_tlb_a15_erratum();
 185 }
 186
 187 void flush_tlb_range(struct vm_area_struct *vma,
 188                      unsigned long start, unsigned long end)
 189 {
 190         if (tlb_ops_need_broadcast()) {
 191                 struct tlb_args ta;
 192                 ta.ta_vma = vma;
 193                 ta.ta_start = start;
 194                 ta.ta_end = end;
 195                 on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_range,
 196                                         &ta, 1);
 197         } else
 198                 local_flush_tlb_range(vma, start, end);
 199         broadcast_tlb_mm_a15_erratum(vma->vm_mm);
 200 }
 201
 202 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 203 {
 204         if (tlb_ops_need_broadcast()) {
 205                 struct tlb_args ta;
 206                 ta.ta_start = start;
 207                 ta.ta_end = end;
 208                 on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
 209         } else
 210                 local_flush_tlb_kernel_range(start, end);
 211         broadcast_tlb_a15_erratum();
 212 }
 213
 214 void flush_bp_all(void)
 215 {
 216         if (tlb_ops_need_broadcast())
 217                 on_each_cpu(ipi_flush_bp_all, NULL, 1);
 218         else
 219                 __flush_bp_all();
 220 }