kernel/arch/arm/kernel/perf_event_v7.c

   1 /*
   2  * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
   3  *
   4  * ARMv7 support: Jean Pihet <jpihet@mvista.com>
   5  * 2010 (c) MontaVista Software, LLC.
   6  *
   7  * Copied from ARMv6 code, with the low level code inspired
   8  *  by the ARMv7 Oprofile code.
   9  *
  10  * Cortex-A8 has up to 4 configurable performance counters and
  11  *  a single cycle counter.
  12  * Cortex-A9 has up to 31 configurable performance counters and
  13  *  a single cycle counter.
  14  *
  15  * All counters can be enabled/disabled and IRQ masked separately. The cycle
  16  *  counter and all 4 performance counters together can be reset separately.
  17  */
  18
  19 #ifdef CONFIG_CPU_V7
  20
  21 #include <asm/cp15.h>
  22 #include <asm/vfp.h>
  23 #include "../vfp/vfpinstr.h"
  24
  25 /*
  26  * Common ARMv7 event types
  27  *
  28  * Note: An implementation may not be able to count all of these events
  29  * but the encodings are considered to be `reserved' in the case that
  30  * they are not available.
  31  */
  32 enum armv7_perf_types {
  33         ARMV7_PERFCTR_PMNC_SW_INCR                      = 0x00,
  34         ARMV7_PERFCTR_L1_ICACHE_REFILL                  = 0x01,
  35         ARMV7_PERFCTR_ITLB_REFILL                       = 0x02,
  36         ARMV7_PERFCTR_L1_DCACHE_REFILL                  = 0x03,
  37         ARMV7_PERFCTR_L1_DCACHE_ACCESS                  = 0x04,
  38         ARMV7_PERFCTR_DTLB_REFILL                       = 0x05,
  39         ARMV7_PERFCTR_MEM_READ                          = 0x06,
  40         ARMV7_PERFCTR_MEM_WRITE                         = 0x07,
  41         ARMV7_PERFCTR_INSTR_EXECUTED                    = 0x08,
  42         ARMV7_PERFCTR_EXC_TAKEN                         = 0x09,
  43         ARMV7_PERFCTR_EXC_EXECUTED                      = 0x0A,
  44         ARMV7_PERFCTR_CID_WRITE                         = 0x0B,
  45
  46         /*
  47          * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
  48          * It counts:
  49          *  - all (taken) branch instructions,
  50          *  - instructions that explicitly write the PC,
  51          *  - exception generating instructions.
  52          */
  53         ARMV7_PERFCTR_PC_WRITE                          = 0x0C,
  54         ARMV7_PERFCTR_PC_IMM_BRANCH                     = 0x0D,
  55         ARMV7_PERFCTR_PC_PROC_RETURN                    = 0x0E,
  56         ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS              = 0x0F,
  57         ARMV7_PERFCTR_PC_BRANCH_MIS_PRED                = 0x10,
  58         ARMV7_PERFCTR_CLOCK_CYCLES                      = 0x11,
  59         ARMV7_PERFCTR_PC_BRANCH_PRED                    = 0x12,
  60
  61         /* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */
  62         ARMV7_PERFCTR_MEM_ACCESS                        = 0x13,
  63         ARMV7_PERFCTR_L1_ICACHE_ACCESS                  = 0x14,
  64         ARMV7_PERFCTR_L1_DCACHE_WB                      = 0x15,
  65         ARMV7_PERFCTR_L2_CACHE_ACCESS                   = 0x16,
  66         ARMV7_PERFCTR_L2_CACHE_REFILL                   = 0x17,
  67         ARMV7_PERFCTR_L2_CACHE_WB                       = 0x18,
  68         ARMV7_PERFCTR_BUS_ACCESS                        = 0x19,
  69         ARMV7_PERFCTR_MEM_ERROR                         = 0x1A,
  70         ARMV7_PERFCTR_INSTR_SPEC                        = 0x1B,
  71         ARMV7_PERFCTR_TTBR_WRITE                        = 0x1C,
  72         ARMV7_PERFCTR_BUS_CYCLES                        = 0x1D,
  73
  74         ARMV7_PERFCTR_CPU_CYCLES                        = 0xFF
  75 };
  76
  77 /* ARMv7 Cortex-A8 specific event types */
  78 enum armv7_a8_perf_types {
  79         ARMV7_A8_PERFCTR_L2_CACHE_ACCESS                = 0x43,
  80         ARMV7_A8_PERFCTR_L2_CACHE_REFILL                = 0x44,
  81         ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS               = 0x50,
  82         ARMV7_A8_PERFCTR_STALL_ISIDE                    = 0x56,
  83 };
  84
  85 /* ARMv7 Cortex-A9 specific event types */
  86 enum armv7_a9_perf_types {
  87         ARMV7_A9_PERFCTR_INSTR_CORE_RENAME              = 0x68,
  88         ARMV7_A9_PERFCTR_STALL_ICACHE                   = 0x60,
  89         ARMV7_A9_PERFCTR_STALL_DISPATCH                 = 0x66,
  90 };
  91
  92 /* ARMv7 Cortex-A5 specific event types */
  93 enum armv7_a5_perf_types {
  94         ARMV7_A5_PERFCTR_PREFETCH_LINEFILL              = 0xc2,
  95         ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP         = 0xc3,
  96 };
  97
  98 /* ARMv7 Cortex-A15 specific event types */
  99 enum armv7_a15_perf_types {
 100         ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ         = 0x40,
 101         ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE        = 0x41,
 102         ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ         = 0x42,
 103         ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE        = 0x43,
 104
 105         ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ           = 0x4C,
 106         ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE          = 0x4D,
 107
 108         ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ          = 0x50,
 109         ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE         = 0x51,
 110         ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ          = 0x52,
 111         ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE         = 0x53,
 112
 113         ARMV7_A15_PERFCTR_PC_WRITE_SPEC                 = 0x76,
 114 };
 115
 116 /* ARMv7 Cortex-A12 specific event types */
 117 enum armv7_a12_perf_types {
 118         ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ         = 0x40,
 119         ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE        = 0x41,
 120
 121         ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ          = 0x50,
 122         ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE         = 0x51,
 123
 124         ARMV7_A12_PERFCTR_PC_WRITE_SPEC                 = 0x76,
 125
 126         ARMV7_A12_PERFCTR_PF_TLB_REFILL                 = 0xe7,
 127 };
 128
 129 /* ARMv7 Krait specific event types */
 130 enum krait_perf_types {
 131         KRAIT_PMRESR0_GROUP0                            = 0xcc,
 132         KRAIT_PMRESR1_GROUP0                            = 0xd0,
 133         KRAIT_PMRESR2_GROUP0                            = 0xd4,
 134         KRAIT_VPMRESR0_GROUP0                           = 0xd8,
 135
 136         KRAIT_PERFCTR_L1_ICACHE_ACCESS                  = 0x10011,
 137         KRAIT_PERFCTR_L1_ICACHE_MISS                    = 0x10010,
 138
 139         KRAIT_PERFCTR_L1_ITLB_ACCESS                    = 0x12222,
 140         KRAIT_PERFCTR_L1_DTLB_ACCESS                    = 0x12210,
 141 };
 142
 143 /* ARMv7 Scorpion specific event types */
 144 enum scorpion_perf_types {
 145         SCORPION_LPM0_GROUP0                            = 0x4c,
 146         SCORPION_LPM1_GROUP0                            = 0x50,
 147         SCORPION_LPM2_GROUP0                            = 0x54,
 148         SCORPION_L2LPM_GROUP0                           = 0x58,
 149         SCORPION_VLPM_GROUP0                            = 0x5c,
 150
 151         SCORPION_ICACHE_ACCESS                          = 0x10053,
 152         SCORPION_ICACHE_MISS                            = 0x10052,
 153
 154         SCORPION_DTLB_ACCESS                            = 0x12013,
 155         SCORPION_DTLB_MISS                              = 0x12012,
 156
 157         SCORPION_ITLB_MISS                              = 0x12021,
 158 };
 159
 160 /*
 161  * Cortex-A8 HW events mapping
 162  *
 163  * The hardware events that we support. We do support cache operations but
 164  * we have harvard caches and no way to combine instruction and data
 165  * accesses/misses in hardware.
 166  */
 167 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
 168         PERF_MAP_ALL_UNSUPPORTED,
 169         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 170         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 171         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 172         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 173         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 174         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 175         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE,
 176 };
 177
 178 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 179                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 180                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 181         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 182
 183         /*
 184          * The performance counters don't differentiate between read and write
 185          * accesses/misses so this isn't strictly correct, but it's the best we
 186          * can do. Writes and reads get combined.
 187          */
 188         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 189         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 190         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 191         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 192
 193         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
 194         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 195
 196         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
 197         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
 198         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
 199         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
 200
 201         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 202         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 203
 204         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 205         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 206
 207         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 208         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 209         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 210         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 211 };
 212
 213 /*
 214  * Cortex-A9 HW events mapping
 215  */
 216 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
 217         PERF_MAP_ALL_UNSUPPORTED,
 218         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 219         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME,
 220         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 221         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 222         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 223         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 224         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE,
 225         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = ARMV7_A9_PERFCTR_STALL_DISPATCH,
 226 };
 227
 228 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 229                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 230                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 231         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 232
 233         /*
 234          * The performance counters don't differentiate between read and write
 235          * accesses/misses so this isn't strictly correct, but it's the best we
 236          * can do. Writes and reads get combined.
 237          */
 238         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 239         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 240         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 241         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 242
 243         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 244
 245         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 246         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 247
 248         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 249         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 250
 251         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 252         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 253         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 254         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 255 };
 256
 257 /*
 258  * Cortex-A5 HW events mapping
 259  */
 260 static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = {
 261         PERF_MAP_ALL_UNSUPPORTED,
 262         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 263         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 264         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 265         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 266         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 267         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 268 };
 269
 270 static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 271                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 272                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 273         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 274
 275         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 276         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 277         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 278         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 279         [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)]      = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
 280         [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)]        = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
 281
 282         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 283         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 284         /*
 285          * The prefetch counters don't differentiate between the I side and the
 286          * D side.
 287          */
 288         [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)]      = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
 289         [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)]        = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
 290
 291         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 292         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 293
 294         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 295         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 296
 297         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 298         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 299         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 300         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 301 };
 302
 303 /*
 304  * Cortex-A15 HW events mapping
 305  */
 306 static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = {
 307         PERF_MAP_ALL_UNSUPPORTED,
 308         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 309         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 310         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 311         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 312         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_A15_PERFCTR_PC_WRITE_SPEC,
 313         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 314         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
 315 };
 316
 317 static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 318                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 319                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 320         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 321
 322         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ,
 323         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ,
 324         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE,
 325         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE,
 326
 327         /*
 328          * Not all performance counters differentiate between read and write
 329          * accesses/misses so we're not always strictly correct, but it's the
 330          * best we can do. Writes and reads get combined in these cases.
 331          */
 332         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 333         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 334
 335         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ,
 336         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ,
 337         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE,
 338         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE,
 339
 340         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ,
 341         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE,
 342
 343         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 344         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 345
 346         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 347         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 348         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 349         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 350 };
 351
 352 /*
 353  * Cortex-A7 HW events mapping
 354  */
 355 static const unsigned armv7_a7_perf_map[PERF_COUNT_HW_MAX] = {
 356         PERF_MAP_ALL_UNSUPPORTED,
 357         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 358         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 359         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 360         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 361         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 362         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 363         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
 364 };
 365
 366 static const unsigned armv7_a7_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 367                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 368                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 369         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 370
 371         /*
 372          * The performance counters don't differentiate between read and write
 373          * accesses/misses so this isn't strictly correct, but it's the best we
 374          * can do. Writes and reads get combined.
 375          */
 376         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 377         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 378         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 379         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 380
 381         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 382         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 383
 384         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_PERFCTR_L2_CACHE_ACCESS,
 385         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_PERFCTR_L2_CACHE_REFILL,
 386         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L2_CACHE_ACCESS,
 387         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L2_CACHE_REFILL,
 388
 389         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 390         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 391
 392         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 393         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 394
 395         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 396         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 397         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 398         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 399 };
 400
 401 /*
 402  * Cortex-A12 HW events mapping
 403  */
 404 static const unsigned armv7_a12_perf_map[PERF_COUNT_HW_MAX] = {
 405         PERF_MAP_ALL_UNSUPPORTED,
 406         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 407         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 408         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 409         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 410         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_A12_PERFCTR_PC_WRITE_SPEC,
 411         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 412         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
 413 };
 414
 415 static const unsigned armv7_a12_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 416                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 417                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 418         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 419
 420         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_READ,
 421         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 422         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_A12_PERFCTR_L1_DCACHE_ACCESS_WRITE,
 423         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 424
 425         /*
 426          * Not all performance counters differentiate between read and write
 427          * accesses/misses so we're not always strictly correct, but it's the
 428          * best we can do. Writes and reads get combined in these cases.
 429          */
 430         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 431         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 432
 433         [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_READ,
 434         [C(LL)][C(OP_READ)][C(RESULT_MISS)]     = ARMV7_PERFCTR_L2_CACHE_REFILL,
 435         [C(LL)][C(OP_WRITE)][C(RESULT_ACCESS)]  = ARMV7_A12_PERFCTR_L2_CACHE_ACCESS_WRITE,
 436         [C(LL)][C(OP_WRITE)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L2_CACHE_REFILL,
 437
 438         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_DTLB_REFILL,
 439         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_DTLB_REFILL,
 440         [C(DTLB)][C(OP_PREFETCH)][C(RESULT_MISS)]       = ARMV7_A12_PERFCTR_PF_TLB_REFILL,
 441
 442         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV7_PERFCTR_ITLB_REFILL,
 443         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV7_PERFCTR_ITLB_REFILL,
 444
 445         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 446         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 447         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 448         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 449 };
 450
 451 /*
 452  * Krait HW events mapping
 453  */
 454 static const unsigned krait_perf_map[PERF_COUNT_HW_MAX] = {
 455         PERF_MAP_ALL_UNSUPPORTED,
 456         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
 457         [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
 458         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
 459         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 460         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
 461 };
 462
 463 static const unsigned krait_perf_map_no_branch[PERF_COUNT_HW_MAX] = {
 464         PERF_MAP_ALL_UNSUPPORTED,
 465         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
 466         [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
 467         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 468         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
 469 };
 470
 471 static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 472                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 473                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 474         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 475
 476         /*
 477          * The performance counters don't differentiate between read and write
 478          * accesses/misses so this isn't strictly correct, but it's the best we
 479          * can do. Writes and reads get combined.
 480          */
 481         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 482         [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 483         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 484         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 485
 486         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = KRAIT_PERFCTR_L1_ICACHE_ACCESS,
 487         [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = KRAIT_PERFCTR_L1_ICACHE_MISS,
 488
 489         [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_DTLB_ACCESS,
 490         [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)]        = KRAIT_PERFCTR_L1_DTLB_ACCESS,
 491
 492         [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = KRAIT_PERFCTR_L1_ITLB_ACCESS,
 493         [C(ITLB)][C(OP_WRITE)][C(RESULT_ACCESS)]        = KRAIT_PERFCTR_L1_ITLB_ACCESS,
 494
 495         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV7_PERFCTR_PC_BRANCH_PRED,
 496         [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 497         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 498         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 499 };
 500
 501 /*
 502  * Scorpion HW events mapping
 503  */
 504 static const unsigned scorpion_perf_map[PERF_COUNT_HW_MAX] = {
 505         PERF_MAP_ALL_UNSUPPORTED,
 506         [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
 507         [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
 508         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
 509         [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 510         [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
 511 };
 512
 513 static const unsigned scorpion_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 514                                             [PERF_COUNT_HW_CACHE_OP_MAX]
 515                                             [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 516         PERF_CACHE_MAP_ALL_UNSUPPORTED,
 517         /*
 518          * The performance counters don't differentiate between read and write
 519          * accesses/misses so this isn't strictly correct, but it's the best we
 520          * can do. Writes and reads get combined.
 521          */
 522         [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 523         [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 524         [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 525         [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 526         [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_ICACHE_ACCESS,
 527         [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ICACHE_MISS,
 528         /*
 529          * Only ITLB misses and DTLB refills are supported.  If users want the
 530          * DTLB refills misses a raw counter must be used.
 531          */
 532         [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
 533         [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
 534         [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS,
 535         [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_DTLB_MISS,
 536         [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
 537         [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_ITLB_MISS,
 538         [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 539         [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 540         [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED,
 541         [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 542 };
 543
 544 /*
 545  * Perf Events' indices
 546  */
 547 #define ARMV7_IDX_CYCLE_COUNTER 0
 548 #define ARMV7_IDX_COUNTER0      1
 549 #define ARMV7_IDX_COUNTER_LAST(cpu_pmu) \
 550         (ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
 551
 552 #define ARMV7_MAX_COUNTERS      32
 553 #define ARMV7_COUNTER_MASK      (ARMV7_MAX_COUNTERS - 1)
 554
 555 /*
 556  * ARMv7 low level PMNC access
 557  */
 558
 559 /*
 560  * Perf Event to low level counters mapping
 561  */
 562 #define ARMV7_IDX_TO_COUNTER(x) \
 563         (((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK)
 564
 565 /*
 566  * Per-CPU PMNC: config reg
 567  */
 568 #define ARMV7_PMNC_E            (1 << 0) /* Enable all counters */
 569 #define ARMV7_PMNC_P            (1 << 1) /* Reset all counters */
 570 #define ARMV7_PMNC_C            (1 << 2) /* Cycle counter reset */
 571 #define ARMV7_PMNC_D            (1 << 3) /* CCNT counts every 64th cpu cycle */
 572 #define ARMV7_PMNC_X            (1 << 4) /* Export to ETM */
 573 #define ARMV7_PMNC_DP           (1 << 5) /* Disable CCNT if non-invasive debug*/
 574 #define ARMV7_PMNC_N_SHIFT      11       /* Number of counters supported */
 575 #define ARMV7_PMNC_N_MASK       0x1f
 576 #define ARMV7_PMNC_MASK         0x3f     /* Mask for writable bits */
 577
 578 /*
 579  * FLAG: counters overflow flag status reg
 580  */
 581 #define ARMV7_FLAG_MASK         0xffffffff      /* Mask for writable bits */
 582 #define ARMV7_OVERFLOWED_MASK   ARMV7_FLAG_MASK
 583
 584 /*
 585  * PMXEVTYPER: Event selection reg
 586  */
 587 #define ARMV7_EVTYPE_MASK       0xc80000ff      /* Mask for writable bits */
 588 #define ARMV7_EVTYPE_EVENT      0xff            /* Mask for EVENT bits */
 589
 590 /*
 591  * Event filters for PMUv2
 592  */
 593 #define ARMV7_EXCLUDE_PL1       (1 << 31)
 594 #define ARMV7_EXCLUDE_USER      (1 << 30)
 595 #define ARMV7_INCLUDE_HYP       (1 << 27)
 596
 597 static inline u32 armv7_pmnc_read(void)
 598 {
 599         u32 val;
 600         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
 601         return val;
 602 }
 603
 604 static inline void armv7_pmnc_write(u32 val)
 605 {
 606         val &= ARMV7_PMNC_MASK;
 607         isb();
 608         asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
 609 }
 610
 611 static inline int armv7_pmnc_has_overflowed(u32 pmnc)
 612 {
 613         return pmnc & ARMV7_OVERFLOWED_MASK;
 614 }
 615
 616 static inline int armv7_pmnc_counter_valid(struct arm_pmu *cpu_pmu, int idx)
 617 {
 618         return idx >= ARMV7_IDX_CYCLE_COUNTER &&
 619                 idx <= ARMV7_IDX_COUNTER_LAST(cpu_pmu);
 620 }
 621
 622 static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx)
 623 {
 624         return pmnc & BIT(ARMV7_IDX_TO_COUNTER(idx));
 625 }
 626
 627 static inline void armv7_pmnc_select_counter(int idx)
 628 {
 629         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 630         asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter));
 631         isb();
 632 }
 633
 634 static inline u32 armv7pmu_read_counter(struct perf_event *event)
 635 {
 636         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 637         struct hw_perf_event *hwc = &event->hw;
 638         int idx = hwc->idx;
 639         u32 value = 0;
 640
 641         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 642                 pr_err("CPU%u reading wrong counter %d\n",
 643                         smp_processor_id(), idx);
 644         } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
 645                 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
 646         } else {
 647                 armv7_pmnc_select_counter(idx);
 648                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value));
 649         }
 650
 651         return value;
 652 }
 653
 654 static inline void armv7pmu_write_counter(struct perf_event *event, u32 value)
 655 {
 656         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 657         struct hw_perf_event *hwc = &event->hw;
 658         int idx = hwc->idx;
 659
 660         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 661                 pr_err("CPU%u writing wrong counter %d\n",
 662                         smp_processor_id(), idx);
 663         } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
 664                 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
 665         } else {
 666                 armv7_pmnc_select_counter(idx);
 667                 asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value));
 668         }
 669 }
 670
 671 static inline void armv7_pmnc_write_evtsel(int idx, u32 val)
 672 {
 673         armv7_pmnc_select_counter(idx);
 674         val &= ARMV7_EVTYPE_MASK;
 675         asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
 676 }
 677
 678 static inline void armv7_pmnc_enable_counter(int idx)
 679 {
 680         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 681         asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter)));
 682 }
 683
 684 static inline void armv7_pmnc_disable_counter(int idx)
 685 {
 686         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 687         asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter)));
 688 }
 689
 690 static inline void armv7_pmnc_enable_intens(int idx)
 691 {
 692         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 693         asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter)));
 694 }
 695
 696 static inline void armv7_pmnc_disable_intens(int idx)
 697 {
 698         u32 counter = ARMV7_IDX_TO_COUNTER(idx);
 699         asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter)));
 700         isb();
 701         /* Clear the overflow flag in case an interrupt is pending. */
 702         asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (BIT(counter)));
 703         isb();
 704 }
 705
 706 static inline u32 armv7_pmnc_getreset_flags(void)
 707 {
 708         u32 val;
 709
 710         /* Read */
 711         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
 712
 713         /* Write to clear flags */
 714         val &= ARMV7_FLAG_MASK;
 715         asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
 716
 717         return val;
 718 }
 719
 720 #ifdef DEBUG
 721 static void armv7_pmnc_dump_regs(struct arm_pmu *cpu_pmu)
 722 {
 723         u32 val;
 724         unsigned int cnt;
 725
 726         pr_info("PMNC registers dump:\n");
 727
 728         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
 729         pr_info("PMNC  =0x%08x\n", val);
 730
 731         asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
 732         pr_info("CNTENS=0x%08x\n", val);
 733
 734         asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
 735         pr_info("INTENS=0x%08x\n", val);
 736
 737         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
 738         pr_info("FLAGS =0x%08x\n", val);
 739
 740         asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
 741         pr_info("SELECT=0x%08x\n", val);
 742
 743         asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
 744         pr_info("CCNT  =0x%08x\n", val);
 745
 746         for (cnt = ARMV7_IDX_COUNTER0;
 747                         cnt <= ARMV7_IDX_COUNTER_LAST(cpu_pmu); cnt++) {
 748                 armv7_pmnc_select_counter(cnt);
 749                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
 750                 pr_info("CNT[%d] count =0x%08x\n",
 751                         ARMV7_IDX_TO_COUNTER(cnt), val);
 752                 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
 753                 pr_info("CNT[%d] evtsel=0x%08x\n",
 754                         ARMV7_IDX_TO_COUNTER(cnt), val);
 755         }
 756 }
 757 #endif
 758
 759 static void armv7pmu_enable_event(struct perf_event *event)
 760 {
 761         unsigned long flags;
 762         struct hw_perf_event *hwc = &event->hw;
 763         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 764         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 765         int idx = hwc->idx;
 766
 767         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 768                 pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
 769                         smp_processor_id(), idx);
 770                 return;
 771         }
 772
 773         /*
 774          * Enable counter and interrupt, and set the counter to count
 775          * the event that we're interested in.
 776          */
 777         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 778
 779         /*
 780          * Disable counter
 781          */
 782         armv7_pmnc_disable_counter(idx);
 783
 784         /*
 785          * Set event (if destined for PMNx counters)
 786          * We only need to set the event for the cycle counter if we
 787          * have the ability to perform event filtering.
 788          */
 789         if (cpu_pmu->set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)
 790                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
 791
 792         /*
 793          * Enable interrupt for this counter
 794          */
 795         armv7_pmnc_enable_intens(idx);
 796
 797         /*
 798          * Enable counter
 799          */
 800         armv7_pmnc_enable_counter(idx);
 801
 802         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 803 }
 804
 805 static void armv7pmu_disable_event(struct perf_event *event)
 806 {
 807         unsigned long flags;
 808         struct hw_perf_event *hwc = &event->hw;
 809         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 810         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 811         int idx = hwc->idx;
 812
 813         if (!armv7_pmnc_counter_valid(cpu_pmu, idx)) {
 814                 pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
 815                         smp_processor_id(), idx);
 816                 return;
 817         }
 818
 819         /*
 820          * Disable counter and interrupt
 821          */
 822         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 823
 824         /*
 825          * Disable counter
 826          */
 827         armv7_pmnc_disable_counter(idx);
 828
 829         /*
 830          * Disable interrupt for this counter
 831          */
 832         armv7_pmnc_disable_intens(idx);
 833
 834         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 835 }
 836
 837 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
 838 {
 839         u32 pmnc;
 840         struct perf_sample_data data;
 841         struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev;
 842         struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
 843         struct pt_regs *regs;
 844         int idx;
 845
 846         /*
 847          * Get and reset the IRQ flags
 848          */
 849         pmnc = armv7_pmnc_getreset_flags();
 850
 851         /*
 852          * Did an overflow occur?
 853          */
 854         if (!armv7_pmnc_has_overflowed(pmnc))
 855                 return IRQ_NONE;
 856
 857         /*
 858          * Handle the counter(s) overflow(s)
 859          */
 860         regs = get_irq_regs();
 861
 862         for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
 863                 struct perf_event *event = cpuc->events[idx];
 864                 struct hw_perf_event *hwc;
 865
 866                 /* Ignore if we don't have an event. */
 867                 if (!event)
 868                         continue;
 869
 870                 /*
 871                  * We have a single interrupt for all counters. Check that
 872                  * each counter has overflowed before we process it.
 873                  */
 874                 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
 875                         continue;
 876
 877                 hwc = &event->hw;
 878                 armpmu_event_update(event);
 879                 perf_sample_data_init(&data, 0, hwc->last_period);
 880                 if (!armpmu_event_set_period(event))
 881                         continue;
 882
 883                 if (perf_event_overflow(event, &data, regs))
 884                         cpu_pmu->disable(event);
 885         }
 886
 887         /*
 888          * Handle the pending perf events.
 889          *
 890          * Note: this call *must* be run with interrupts disabled. For
 891          * platforms that can have the PMU interrupts raised as an NMI, this
 892          * will not work.
 893          */
 894         irq_work_run();
 895
 896         return IRQ_HANDLED;
 897 }
 898
 899 static void armv7pmu_start(struct arm_pmu *cpu_pmu)
 900 {
 901         unsigned long flags;
 902         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 903
 904         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 905         /* Enable all counters */
 906         armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
 907         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 908 }
 909
 910 static void armv7pmu_stop(struct arm_pmu *cpu_pmu)
 911 {
 912         unsigned long flags;
 913         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
 914
 915         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 916         /* Disable all counters */
 917         armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
 918         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 919 }
 920
 921 static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc,
 922                                   struct perf_event *event)
 923 {
 924         int idx;
 925         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
 926         struct hw_perf_event *hwc = &event->hw;
 927         unsigned long evtype = hwc->config_base & ARMV7_EVTYPE_EVENT;
 928
 929         /* Always place a cycle counter into the cycle counter. */
 930         if (evtype == ARMV7_PERFCTR_CPU_CYCLES) {
 931                 if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))
 932                         return -EAGAIN;
 933
 934                 return ARMV7_IDX_CYCLE_COUNTER;
 935         }
 936
 937         /*
 938          * For anything other than a cycle counter, try and use
 939          * the events counters
 940          */
 941         for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
 942                 if (!test_and_set_bit(idx, cpuc->used_mask))
 943                         return idx;
 944         }
 945
 946         /* The counters are all in use. */
 947         return -EAGAIN;
 948 }
 949
 950 /*
 951  * Add an event filter to a given event. This will only work for PMUv2 PMUs.
 952  */
 953 static int armv7pmu_set_event_filter(struct hw_perf_event *event,
 954                                      struct perf_event_attr *attr)
 955 {
 956         unsigned long config_base = 0;
 957
 958         if (attr->exclude_idle)
 959                 return -EPERM;
 960         if (attr->exclude_user)
 961                 config_base |= ARMV7_EXCLUDE_USER;
 962         if (attr->exclude_kernel)
 963                 config_base |= ARMV7_EXCLUDE_PL1;
 964         if (!attr->exclude_hv)
 965                 config_base |= ARMV7_INCLUDE_HYP;
 966
 967         /*
 968          * Install the filter into config_base as this is used to
 969          * construct the event type.
 970          */
 971         event->config_base = config_base;
 972
 973         return 0;
 974 }
 975
 976 static void armv7pmu_reset(void *info)
 977 {
 978         struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
 979         u32 idx, nb_cnt = cpu_pmu->num_events;
 980
 981         /* The counter and interrupt enable registers are unknown at reset. */
 982         for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
 983                 armv7_pmnc_disable_counter(idx);
 984                 armv7_pmnc_disable_intens(idx);
 985         }
 986
 987         /* Initialize & Reset PMNC: C and P bits */
 988         armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
 989 }
 990
 991 static int armv7_a8_map_event(struct perf_event *event)
 992 {
 993         return armpmu_map_event(event, &armv7_a8_perf_map,
 994                                 &armv7_a8_perf_cache_map, 0xFF);
 995 }
 996
 997 static int armv7_a9_map_event(struct perf_event *event)
 998 {
 999         return armpmu_map_event(event, &armv7_a9_perf_map,
1000                                 &armv7_a9_perf_cache_map, 0xFF);
1001 }
1002
1003 static int armv7_a5_map_event(struct perf_event *event)
1004 {
1005         return armpmu_map_event(event, &armv7_a5_perf_map,
1006                                 &armv7_a5_perf_cache_map, 0xFF);
1007 }
1008
1009 static int armv7_a15_map_event(struct perf_event *event)
1010 {
1011         return armpmu_map_event(event, &armv7_a15_perf_map,
1012                                 &armv7_a15_perf_cache_map, 0xFF);
1013 }
1014
1015 static int armv7_a7_map_event(struct perf_event *event)
1016 {
1017         return armpmu_map_event(event, &armv7_a7_perf_map,
1018                                 &armv7_a7_perf_cache_map, 0xFF);
1019 }
1020
1021 static int armv7_a12_map_event(struct perf_event *event)
1022 {
1023         return armpmu_map_event(event, &armv7_a12_perf_map,
1024                                 &armv7_a12_perf_cache_map, 0xFF);
1025 }
1026
1027 static int krait_map_event(struct perf_event *event)
1028 {
1029         return armpmu_map_event(event, &krait_perf_map,
1030                                 &krait_perf_cache_map, 0xFFFFF);
1031 }
1032
1033 static int krait_map_event_no_branch(struct perf_event *event)
1034 {
1035         return armpmu_map_event(event, &krait_perf_map_no_branch,
1036                                 &krait_perf_cache_map, 0xFFFFF);
1037 }
1038
1039 static int scorpion_map_event(struct perf_event *event)
1040 {
1041         return armpmu_map_event(event, &scorpion_perf_map,
1042                                 &scorpion_perf_cache_map, 0xFFFFF);
1043 }
1044
1045 static void armv7pmu_init(struct arm_pmu *cpu_pmu)
1046 {
1047         cpu_pmu->handle_irq     = armv7pmu_handle_irq;
1048         cpu_pmu->enable         = armv7pmu_enable_event;
1049         cpu_pmu->disable        = armv7pmu_disable_event;
1050         cpu_pmu->read_counter   = armv7pmu_read_counter;
1051         cpu_pmu->write_counter  = armv7pmu_write_counter;
1052         cpu_pmu->get_event_idx  = armv7pmu_get_event_idx;
1053         cpu_pmu->start          = armv7pmu_start;
1054         cpu_pmu->stop           = armv7pmu_stop;
1055         cpu_pmu->reset          = armv7pmu_reset;
1056         cpu_pmu->max_period     = (1LLU << 32) - 1;
1057 };
1058
1059 static u32 armv7_read_num_pmnc_events(void)
1060 {
1061         u32 nb_cnt;
1062
1063         /* Read the nb of CNTx counters supported from PMNC */
1064         nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
1065
1066         /* Add the CPU cycles counter and return */
1067         return nb_cnt + 1;
1068 }
1069
1070 static int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu)
1071 {
1072         armv7pmu_init(cpu_pmu);
1073         cpu_pmu->name           = "armv7_cortex_a8";
1074         cpu_pmu->map_event      = armv7_a8_map_event;
1075         cpu_pmu->num_events     = armv7_read_num_pmnc_events();
1076         return 0;
1077 }
1078
1079 static int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu)
1080 {
1081         armv7pmu_init(cpu_pmu);
1082         cpu_pmu->name           = "armv7_cortex_a9";
1083         cpu_pmu->map_event      = armv7_a9_map_event;
1084         cpu_pmu->num_events     = armv7_read_num_pmnc_events();
1085         return 0;
1086 }
1087
1088 static int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu)
1089 {
1090         armv7pmu_init(cpu_pmu);
1091         cpu_pmu->name           = "armv7_cortex_a5";
1092         cpu_pmu->map_event      = armv7_a5_map_event;
1093         cpu_pmu->num_events     = armv7_read_num_pmnc_events();
1094         return 0;
1095 }
1096
1097 static int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu)
1098 {
1099         armv7pmu_init(cpu_pmu);
1100         cpu_pmu->name           = "armv7_cortex_a15";
1101         cpu_pmu->map_event      = armv7_a15_map_event;
1102         cpu_pmu->num_events     = armv7_read_num_pmnc_events();
1103         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1104         return 0;
1105 }
1106
1107 static int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu)
1108 {
1109         armv7pmu_init(cpu_pmu);
1110         cpu_pmu->name           = "armv7_cortex_a7";
1111         cpu_pmu->map_event      = armv7_a7_map_event;
1112         cpu_pmu->num_events     = armv7_read_num_pmnc_events();
1113         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1114         return 0;
1115 }
1116
1117 static int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu)
1118 {
1119         armv7pmu_init(cpu_pmu);
1120         cpu_pmu->name           = "armv7_cortex_a12";
1121         cpu_pmu->map_event      = armv7_a12_map_event;
1122         cpu_pmu->num_events     = armv7_read_num_pmnc_events();
1123         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1124         return 0;
1125 }
1126
1127 static int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu)
1128 {
1129         armv7_a12_pmu_init(cpu_pmu);
1130         cpu_pmu->name = "armv7_cortex_a17";
1131         return 0;
1132 }
1133
1134 /*
1135  * Krait Performance Monitor Region Event Selection Register (PMRESRn)
1136  *
1137  *            31   30     24     16     8      0
1138  *            +--------------------------------+
1139  *  PMRESR0   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 0
1140  *            +--------------------------------+
1141  *  PMRESR1   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 1
1142  *            +--------------------------------+
1143  *  PMRESR2   | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 2
1144  *            +--------------------------------+
1145  *  VPMRESR0  | EN |  CC  |  CC  |  CC  |  CC  |   N = 2, R = ?
1146  *            +--------------------------------+
1147  *              EN | G=3  | G=2  | G=1  | G=0
1148  *
1149  *  Event Encoding:
1150  *
1151  *      hwc->config_base = 0xNRCCG
1152  *
1153  *      N  = prefix, 1 for Krait CPU (PMRESRn), 2 for Venum VFP (VPMRESR)
1154  *      R  = region register
1155  *      CC = class of events the group G is choosing from
1156  *      G  = group or particular event
1157  *
1158  *  Example: 0x12021 is a Krait CPU event in PMRESR2's group 1 with code 2
1159  *
1160  *  A region (R) corresponds to a piece of the CPU (execution unit, instruction
1161  *  unit, etc.) while the event code (CC) corresponds to a particular class of
1162  *  events (interrupts for example). An event code is broken down into
1163  *  groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1164  *  example).
1165  */
1166
1167 #define KRAIT_EVENT             (1 << 16)
1168 #define VENUM_EVENT             (2 << 16)
1169 #define KRAIT_EVENT_MASK        (KRAIT_EVENT | VENUM_EVENT)
1170 #define PMRESRn_EN              BIT(31)
1171
1172 #define EVENT_REGION(event)     (((event) >> 12) & 0xf)         /* R */
1173 #define EVENT_GROUP(event)      ((event) & 0xf)                 /* G */
1174 #define EVENT_CODE(event)       (((event) >> 4) & 0xff)         /* CC */
1175 #define EVENT_VENUM(event)      (!!(event & VENUM_EVENT))       /* N=2 */
1176 #define EVENT_CPU(event)        (!!(event & KRAIT_EVENT))       /* N=1 */
1177
1178 static u32 krait_read_pmresrn(int n)
1179 {
1180         u32 val;
1181
1182         switch (n) {
1183         case 0:
1184                 asm volatile("mrc p15, 1, %0, c9, c15, 0" : "=r" (val));
1185                 break;
1186         case 1:
1187                 asm volatile("mrc p15, 1, %0, c9, c15, 1" : "=r" (val));
1188                 break;
1189         case 2:
1190                 asm volatile("mrc p15, 1, %0, c9, c15, 2" : "=r" (val));
1191                 break;
1192         default:
1193                 BUG(); /* Should be validated in krait_pmu_get_event_idx() */
1194         }
1195
1196         return val;
1197 }
1198
1199 static void krait_write_pmresrn(int n, u32 val)
1200 {
1201         switch (n) {
1202         case 0:
1203                 asm volatile("mcr p15, 1, %0, c9, c15, 0" : : "r" (val));
1204                 break;
1205         case 1:
1206                 asm volatile("mcr p15, 1, %0, c9, c15, 1" : : "r" (val));
1207                 break;
1208         case 2:
1209                 asm volatile("mcr p15, 1, %0, c9, c15, 2" : : "r" (val));
1210                 break;
1211         default:
1212                 BUG(); /* Should be validated in krait_pmu_get_event_idx() */
1213         }
1214 }
1215
1216 static u32 venum_read_pmresr(void)
1217 {
1218         u32 val;
1219         asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val));
1220         return val;
1221 }
1222
1223 static void venum_write_pmresr(u32 val)
1224 {
1225         asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val));
1226 }
1227
1228 static void venum_pre_pmresr(u32 *venum_orig_val, u32 *fp_orig_val)
1229 {
1230         u32 venum_new_val;
1231         u32 fp_new_val;
1232
1233         BUG_ON(preemptible());
1234         /* CPACR Enable CP10 and CP11 access */
1235         *venum_orig_val = get_copro_access();
1236         venum_new_val = *venum_orig_val | CPACC_SVC(10) | CPACC_SVC(11);
1237         set_copro_access(venum_new_val);
1238
1239         /* Enable FPEXC */
1240         *fp_orig_val = fmrx(FPEXC);
1241         fp_new_val = *fp_orig_val | FPEXC_EN;
1242         fmxr(FPEXC, fp_new_val);
1243 }
1244
1245 static void venum_post_pmresr(u32 venum_orig_val, u32 fp_orig_val)
1246 {
1247         BUG_ON(preemptible());
1248         /* Restore FPEXC */
1249         fmxr(FPEXC, fp_orig_val);
1250         isb();
1251         /* Restore CPACR */
1252         set_copro_access(venum_orig_val);
1253 }
1254
1255 static u32 krait_get_pmresrn_event(unsigned int region)
1256 {
1257         static const u32 pmresrn_table[] = { KRAIT_PMRESR0_GROUP0,
1258                                              KRAIT_PMRESR1_GROUP0,
1259                                              KRAIT_PMRESR2_GROUP0 };
1260         return pmresrn_table[region];
1261 }
1262
1263 static void krait_evt_setup(int idx, u32 config_base)
1264 {
1265         u32 val;
1266         u32 mask;
1267         u32 vval, fval;
1268         unsigned int region = EVENT_REGION(config_base);
1269         unsigned int group = EVENT_GROUP(config_base);
1270         unsigned int code = EVENT_CODE(config_base);
1271         unsigned int group_shift;
1272         bool venum_event = EVENT_VENUM(config_base);
1273
1274         group_shift = group * 8;
1275         mask = 0xff << group_shift;
1276
1277         /* Configure evtsel for the region and group */
1278         if (venum_event)
1279                 val = KRAIT_VPMRESR0_GROUP0;
1280         else
1281                 val = krait_get_pmresrn_event(region);
1282         val += group;
1283         /* Mix in mode-exclusion bits */
1284         val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1285         armv7_pmnc_write_evtsel(idx, val);
1286
1287         if (venum_event) {
1288                 venum_pre_pmresr(&vval, &fval);
1289                 val = venum_read_pmresr();
1290                 val &= ~mask;
1291                 val |= code << group_shift;
1292                 val |= PMRESRn_EN;
1293                 venum_write_pmresr(val);
1294                 venum_post_pmresr(vval, fval);
1295         } else {
1296                 val = krait_read_pmresrn(region);
1297                 val &= ~mask;
1298                 val |= code << group_shift;
1299                 val |= PMRESRn_EN;
1300                 krait_write_pmresrn(region, val);
1301         }
1302 }
1303
1304 static u32 clear_pmresrn_group(u32 val, int group)
1305 {
1306         u32 mask;
1307         int group_shift;
1308
1309         group_shift = group * 8;
1310         mask = 0xff << group_shift;
1311         val &= ~mask;
1312
1313         /* Don't clear enable bit if entire region isn't disabled */
1314         if (val & ~PMRESRn_EN)
1315                 return val |= PMRESRn_EN;
1316
1317         return 0;
1318 }
1319
1320 static void krait_clearpmu(u32 config_base)
1321 {
1322         u32 val;
1323         u32 vval, fval;
1324         unsigned int region = EVENT_REGION(config_base);
1325         unsigned int group = EVENT_GROUP(config_base);
1326         bool venum_event = EVENT_VENUM(config_base);
1327
1328         if (venum_event) {
1329                 venum_pre_pmresr(&vval, &fval);
1330                 val = venum_read_pmresr();
1331                 val = clear_pmresrn_group(val, group);
1332                 venum_write_pmresr(val);
1333                 venum_post_pmresr(vval, fval);
1334         } else {
1335                 val = krait_read_pmresrn(region);
1336                 val = clear_pmresrn_group(val, group);
1337                 krait_write_pmresrn(region, val);
1338         }
1339 }
1340
1341 static void krait_pmu_disable_event(struct perf_event *event)
1342 {
1343         unsigned long flags;
1344         struct hw_perf_event *hwc = &event->hw;
1345         int idx = hwc->idx;
1346         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1347         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1348
1349         /* Disable counter and interrupt */
1350         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1351
1352         /* Disable counter */
1353         armv7_pmnc_disable_counter(idx);
1354
1355         /*
1356          * Clear pmresr code (if destined for PMNx counters)
1357          */
1358         if (hwc->config_base & KRAIT_EVENT_MASK)
1359                 krait_clearpmu(hwc->config_base);
1360
1361         /* Disable interrupt for this counter */
1362         armv7_pmnc_disable_intens(idx);
1363
1364         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1365 }
1366
1367 static void krait_pmu_enable_event(struct perf_event *event)
1368 {
1369         unsigned long flags;
1370         struct hw_perf_event *hwc = &event->hw;
1371         int idx = hwc->idx;
1372         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1373         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1374
1375         /*
1376          * Enable counter and interrupt, and set the counter to count
1377          * the event that we're interested in.
1378          */
1379         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1380
1381         /* Disable counter */
1382         armv7_pmnc_disable_counter(idx);
1383
1384         /*
1385          * Set event (if destined for PMNx counters)
1386          * We set the event for the cycle counter because we
1387          * have the ability to perform event filtering.
1388          */
1389         if (hwc->config_base & KRAIT_EVENT_MASK)
1390                 krait_evt_setup(idx, hwc->config_base);
1391         else
1392                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
1393
1394         /* Enable interrupt for this counter */
1395         armv7_pmnc_enable_intens(idx);
1396
1397         /* Enable counter */
1398         armv7_pmnc_enable_counter(idx);
1399
1400         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1401 }
1402
1403 static void krait_pmu_reset(void *info)
1404 {
1405         u32 vval, fval;
1406         struct arm_pmu *cpu_pmu = info;
1407         u32 idx, nb_cnt = cpu_pmu->num_events;
1408
1409         armv7pmu_reset(info);
1410
1411         /* Clear all pmresrs */
1412         krait_write_pmresrn(0, 0);
1413         krait_write_pmresrn(1, 0);
1414         krait_write_pmresrn(2, 0);
1415
1416         venum_pre_pmresr(&vval, &fval);
1417         venum_write_pmresr(0);
1418         venum_post_pmresr(vval, fval);
1419
1420         /* Reset PMxEVNCTCR to sane default */
1421         for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1422                 armv7_pmnc_select_counter(idx);
1423                 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1424         }
1425
1426 }
1427
1428 static int krait_event_to_bit(struct perf_event *event, unsigned int region,
1429                               unsigned int group)
1430 {
1431         int bit;
1432         struct hw_perf_event *hwc = &event->hw;
1433         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1434
1435         if (hwc->config_base & VENUM_EVENT)
1436                 bit = KRAIT_VPMRESR0_GROUP0;
1437         else
1438                 bit = krait_get_pmresrn_event(region);
1439         bit -= krait_get_pmresrn_event(0);
1440         bit += group;
1441         /*
1442          * Lower bits are reserved for use by the counters (see
1443          * armv7pmu_get_event_idx() for more info)
1444          */
1445         bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1446
1447         return bit;
1448 }
1449
1450 /*
1451  * We check for column exclusion constraints here.
1452  * Two events cant use the same group within a pmresr register.
1453  */
1454 static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1455                                    struct perf_event *event)
1456 {
1457         int idx;
1458         int bit = -1;
1459         struct hw_perf_event *hwc = &event->hw;
1460         unsigned int region = EVENT_REGION(hwc->config_base);
1461         unsigned int code = EVENT_CODE(hwc->config_base);
1462         unsigned int group = EVENT_GROUP(hwc->config_base);
1463         bool venum_event = EVENT_VENUM(hwc->config_base);
1464         bool krait_event = EVENT_CPU(hwc->config_base);
1465
1466         if (venum_event || krait_event) {
1467                 /* Ignore invalid events */
1468                 if (group > 3 || region > 2)
1469                         return -EINVAL;
1470                 if (venum_event && (code & 0xe0))
1471                         return -EINVAL;
1472
1473                 bit = krait_event_to_bit(event, region, group);
1474                 if (test_and_set_bit(bit, cpuc->used_mask))
1475                         return -EAGAIN;
1476         }
1477
1478         idx = armv7pmu_get_event_idx(cpuc, event);
1479         if (idx < 0 && bit >= 0)
1480                 clear_bit(bit, cpuc->used_mask);
1481
1482         return idx;
1483 }
1484
1485 static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1486                                       struct perf_event *event)
1487 {
1488         int bit;
1489         struct hw_perf_event *hwc = &event->hw;
1490         unsigned int region = EVENT_REGION(hwc->config_base);
1491         unsigned int group = EVENT_GROUP(hwc->config_base);
1492         bool venum_event = EVENT_VENUM(hwc->config_base);
1493         bool krait_event = EVENT_CPU(hwc->config_base);
1494
1495         if (venum_event || krait_event) {
1496                 bit = krait_event_to_bit(event, region, group);
1497                 clear_bit(bit, cpuc->used_mask);
1498         }
1499 }
1500
1501 static int krait_pmu_init(struct arm_pmu *cpu_pmu)
1502 {
1503         armv7pmu_init(cpu_pmu);
1504         cpu_pmu->name           = "armv7_krait";
1505         /* Some early versions of Krait don't support PC write events */
1506         if (of_property_read_bool(cpu_pmu->plat_device->dev.of_node,
1507                                   "qcom,no-pc-write"))
1508                 cpu_pmu->map_event = krait_map_event_no_branch;
1509         else
1510                 cpu_pmu->map_event = krait_map_event;
1511         cpu_pmu->num_events     = armv7_read_num_pmnc_events();
1512         cpu_pmu->set_event_filter = armv7pmu_set_event_filter;
1513         cpu_pmu->reset          = krait_pmu_reset;
1514         cpu_pmu->enable         = krait_pmu_enable_event;
1515         cpu_pmu->disable        = krait_pmu_disable_event;
1516         cpu_pmu->get_event_idx  = krait_pmu_get_event_idx;
1517         cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx;
1518         return 0;
1519 }
1520
1521 /*
1522  * Scorpion Local Performance Monitor Register (LPMn)
1523  *
1524  *            31   30     24     16     8      0
1525  *            +--------------------------------+
1526  *  LPM0      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 0
1527  *            +--------------------------------+
1528  *  LPM1      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 1
1529  *            +--------------------------------+
1530  *  LPM2      | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 2
1531  *            +--------------------------------+
1532  *  L2LPM     | EN |  CC  |  CC  |  CC  |  CC  |   N = 1, R = 3
1533  *            +--------------------------------+
1534  *  VLPM      | EN |  CC  |  CC  |  CC  |  CC  |   N = 2, R = ?
1535  *            +--------------------------------+
1536  *              EN | G=3  | G=2  | G=1  | G=0
1537  *
1538  *
1539  *  Event Encoding:
1540  *
1541  *      hwc->config_base = 0xNRCCG
1542  *
1543  *      N  = prefix, 1 for Scorpion CPU (LPMn/L2LPM), 2 for Venum VFP (VLPM)
1544  *      R  = region register
1545  *      CC = class of events the group G is choosing from
1546  *      G  = group or particular event
1547  *
1548  *  Example: 0x12021 is a Scorpion CPU event in LPM2's group 1 with code 2
1549  *
1550  *  A region (R) corresponds to a piece of the CPU (execution unit, instruction
1551  *  unit, etc.) while the event code (CC) corresponds to a particular class of
1552  *  events (interrupts for example). An event code is broken down into
1553  *  groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for
1554  *  example).
1555  */
1556
1557 static u32 scorpion_read_pmresrn(int n)
1558 {
1559         u32 val;
1560
1561         switch (n) {
1562         case 0:
1563                 asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val));
1564                 break;
1565         case 1:
1566                 asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val));
1567                 break;
1568         case 2:
1569                 asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val));
1570                 break;
1571         case 3:
1572                 asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val));
1573                 break;
1574         default:
1575                 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1576         }
1577
1578         return val;
1579 }
1580
1581 static void scorpion_write_pmresrn(int n, u32 val)
1582 {
1583         switch (n) {
1584         case 0:
1585                 asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val));
1586                 break;
1587         case 1:
1588                 asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val));
1589                 break;
1590         case 2:
1591                 asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val));
1592                 break;
1593         case 3:
1594                 asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val));
1595                 break;
1596         default:
1597                 BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */
1598         }
1599 }
1600
1601 static u32 scorpion_get_pmresrn_event(unsigned int region)
1602 {
1603         static const u32 pmresrn_table[] = { SCORPION_LPM0_GROUP0,
1604                                              SCORPION_LPM1_GROUP0,
1605                                              SCORPION_LPM2_GROUP0,
1606                                              SCORPION_L2LPM_GROUP0 };
1607         return pmresrn_table[region];
1608 }
1609
1610 static void scorpion_evt_setup(int idx, u32 config_base)
1611 {
1612         u32 val;
1613         u32 mask;
1614         u32 vval, fval;
1615         unsigned int region = EVENT_REGION(config_base);
1616         unsigned int group = EVENT_GROUP(config_base);
1617         unsigned int code = EVENT_CODE(config_base);
1618         unsigned int group_shift;
1619         bool venum_event = EVENT_VENUM(config_base);
1620
1621         group_shift = group * 8;
1622         mask = 0xff << group_shift;
1623
1624         /* Configure evtsel for the region and group */
1625         if (venum_event)
1626                 val = SCORPION_VLPM_GROUP0;
1627         else
1628                 val = scorpion_get_pmresrn_event(region);
1629         val += group;
1630         /* Mix in mode-exclusion bits */
1631         val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1);
1632         armv7_pmnc_write_evtsel(idx, val);
1633
1634         asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1635
1636         if (venum_event) {
1637                 venum_pre_pmresr(&vval, &fval);
1638                 val = venum_read_pmresr();
1639                 val &= ~mask;
1640                 val |= code << group_shift;
1641                 val |= PMRESRn_EN;
1642                 venum_write_pmresr(val);
1643                 venum_post_pmresr(vval, fval);
1644         } else {
1645                 val = scorpion_read_pmresrn(region);
1646                 val &= ~mask;
1647                 val |= code << group_shift;
1648                 val |= PMRESRn_EN;
1649                 scorpion_write_pmresrn(region, val);
1650         }
1651 }
1652
1653 static void scorpion_clearpmu(u32 config_base)
1654 {
1655         u32 val;
1656         u32 vval, fval;
1657         unsigned int region = EVENT_REGION(config_base);
1658         unsigned int group = EVENT_GROUP(config_base);
1659         bool venum_event = EVENT_VENUM(config_base);
1660
1661         if (venum_event) {
1662                 venum_pre_pmresr(&vval, &fval);
1663                 val = venum_read_pmresr();
1664                 val = clear_pmresrn_group(val, group);
1665                 venum_write_pmresr(val);
1666                 venum_post_pmresr(vval, fval);
1667         } else {
1668                 val = scorpion_read_pmresrn(region);
1669                 val = clear_pmresrn_group(val, group);
1670                 scorpion_write_pmresrn(region, val);
1671         }
1672 }
1673
1674 static void scorpion_pmu_disable_event(struct perf_event *event)
1675 {
1676         unsigned long flags;
1677         struct hw_perf_event *hwc = &event->hw;
1678         int idx = hwc->idx;
1679         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1680         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1681
1682         /* Disable counter and interrupt */
1683         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1684
1685         /* Disable counter */
1686         armv7_pmnc_disable_counter(idx);
1687
1688         /*
1689          * Clear pmresr code (if destined for PMNx counters)
1690          */
1691         if (hwc->config_base & KRAIT_EVENT_MASK)
1692                 scorpion_clearpmu(hwc->config_base);
1693
1694         /* Disable interrupt for this counter */
1695         armv7_pmnc_disable_intens(idx);
1696
1697         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1698 }
1699
1700 static void scorpion_pmu_enable_event(struct perf_event *event)
1701 {
1702         unsigned long flags;
1703         struct hw_perf_event *hwc = &event->hw;
1704         int idx = hwc->idx;
1705         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1706         struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
1707
1708         /*
1709          * Enable counter and interrupt, and set the counter to count
1710          * the event that we're interested in.
1711          */
1712         raw_spin_lock_irqsave(&events->pmu_lock, flags);
1713
1714         /* Disable counter */
1715         armv7_pmnc_disable_counter(idx);
1716
1717         /*
1718          * Set event (if destined for PMNx counters)
1719          * We don't set the event for the cycle counter because we
1720          * don't have the ability to perform event filtering.
1721          */
1722         if (hwc->config_base & KRAIT_EVENT_MASK)
1723                 scorpion_evt_setup(idx, hwc->config_base);
1724         else if (idx != ARMV7_IDX_CYCLE_COUNTER)
1725                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
1726
1727         /* Enable interrupt for this counter */
1728         armv7_pmnc_enable_intens(idx);
1729
1730         /* Enable counter */
1731         armv7_pmnc_enable_counter(idx);
1732
1733         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1734 }
1735
1736 static void scorpion_pmu_reset(void *info)
1737 {
1738         u32 vval, fval;
1739         struct arm_pmu *cpu_pmu = info;
1740         u32 idx, nb_cnt = cpu_pmu->num_events;
1741
1742         armv7pmu_reset(info);
1743
1744         /* Clear all pmresrs */
1745         scorpion_write_pmresrn(0, 0);
1746         scorpion_write_pmresrn(1, 0);
1747         scorpion_write_pmresrn(2, 0);
1748         scorpion_write_pmresrn(3, 0);
1749
1750         venum_pre_pmresr(&vval, &fval);
1751         venum_write_pmresr(0);
1752         venum_post_pmresr(vval, fval);
1753
1754         /* Reset PMxEVNCTCR to sane default */
1755         for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
1756                 armv7_pmnc_select_counter(idx);
1757                 asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0));
1758         }
1759 }
1760
1761 static int scorpion_event_to_bit(struct perf_event *event, unsigned int region,
1762                               unsigned int group)
1763 {
1764         int bit;
1765         struct hw_perf_event *hwc = &event->hw;
1766         struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
1767
1768         if (hwc->config_base & VENUM_EVENT)
1769                 bit = SCORPION_VLPM_GROUP0;
1770         else
1771                 bit = scorpion_get_pmresrn_event(region);
1772         bit -= scorpion_get_pmresrn_event(0);
1773         bit += group;
1774         /*
1775          * Lower bits are reserved for use by the counters (see
1776          * armv7pmu_get_event_idx() for more info)
1777          */
1778         bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1;
1779
1780         return bit;
1781 }
1782
1783 /*
1784  * We check for column exclusion constraints here.
1785  * Two events cant use the same group within a pmresr register.
1786  */
1787 static int scorpion_pmu_get_event_idx(struct pmu_hw_events *cpuc,
1788                                    struct perf_event *event)
1789 {
1790         int idx;
1791         int bit = -1;
1792         struct hw_perf_event *hwc = &event->hw;
1793         unsigned int region = EVENT_REGION(hwc->config_base);
1794         unsigned int group = EVENT_GROUP(hwc->config_base);
1795         bool venum_event = EVENT_VENUM(hwc->config_base);
1796         bool scorpion_event = EVENT_CPU(hwc->config_base);
1797
1798         if (venum_event || scorpion_event) {
1799                 /* Ignore invalid events */
1800                 if (group > 3 || region > 3)
1801                         return -EINVAL;
1802
1803                 bit = scorpion_event_to_bit(event, region, group);
1804                 if (test_and_set_bit(bit, cpuc->used_mask))
1805                         return -EAGAIN;
1806         }
1807
1808         idx = armv7pmu_get_event_idx(cpuc, event);
1809         if (idx < 0 && bit >= 0)
1810                 clear_bit(bit, cpuc->used_mask);
1811
1812         return idx;
1813 }
1814
1815 static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
1816                                       struct perf_event *event)
1817 {
1818         int bit;
1819         struct hw_perf_event *hwc = &event->hw;
1820         unsigned int region = EVENT_REGION(hwc->config_base);
1821         unsigned int group = EVENT_GROUP(hwc->config_base);
1822         bool venum_event = EVENT_VENUM(hwc->config_base);
1823         bool scorpion_event = EVENT_CPU(hwc->config_base);
1824
1825         if (venum_event || scorpion_event) {
1826                 bit = scorpion_event_to_bit(event, region, group);
1827                 clear_bit(bit, cpuc->used_mask);
1828         }
1829 }
1830
1831 static int scorpion_pmu_init(struct arm_pmu *cpu_pmu)
1832 {
1833         armv7pmu_init(cpu_pmu);
1834         cpu_pmu->name           = "armv7_scorpion";
1835         cpu_pmu->map_event      = scorpion_map_event;
1836         cpu_pmu->num_events     = armv7_read_num_pmnc_events();
1837         cpu_pmu->reset          = scorpion_pmu_reset;
1838         cpu_pmu->enable         = scorpion_pmu_enable_event;
1839         cpu_pmu->disable        = scorpion_pmu_disable_event;
1840         cpu_pmu->get_event_idx  = scorpion_pmu_get_event_idx;
1841         cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1842         return 0;
1843 }
1844
1845 static int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu)
1846 {
1847         armv7pmu_init(cpu_pmu);
1848         cpu_pmu->name           = "armv7_scorpion_mp";
1849         cpu_pmu->map_event      = scorpion_map_event;
1850         cpu_pmu->num_events     = armv7_read_num_pmnc_events();
1851         cpu_pmu->reset          = scorpion_pmu_reset;
1852         cpu_pmu->enable         = scorpion_pmu_enable_event;
1853         cpu_pmu->disable        = scorpion_pmu_disable_event;
1854         cpu_pmu->get_event_idx  = scorpion_pmu_get_event_idx;
1855         cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx;
1856         return 0;
1857 }
1858 #else
1859 static inline int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu)
1860 {
1861         return -ENODEV;
1862 }
1863
1864 static inline int armv7_a9_pmu_init(struct arm_pmu *cpu_pmu)
1865 {
1866         return -ENODEV;
1867 }
1868
1869 static inline int armv7_a5_pmu_init(struct arm_pmu *cpu_pmu)
1870 {
1871         return -ENODEV;
1872 }
1873
1874 static inline int armv7_a15_pmu_init(struct arm_pmu *cpu_pmu)
1875 {
1876         return -ENODEV;
1877 }
1878
1879 static inline int armv7_a7_pmu_init(struct arm_pmu *cpu_pmu)
1880 {
1881         return -ENODEV;
1882 }
1883
1884 static inline int armv7_a12_pmu_init(struct arm_pmu *cpu_pmu)
1885 {
1886         return -ENODEV;
1887 }
1888
1889 static inline int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu)
1890 {
1891         return -ENODEV;
1892 }
1893
1894 static inline int krait_pmu_init(struct arm_pmu *cpu_pmu)
1895 {
1896         return -ENODEV;
1897 }
1898
1899 static inline int scorpion_pmu_init(struct arm_pmu *cpu_pmu)
1900 {
1901         return -ENODEV;
1902 }
1903
1904 static inline int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu)
1905 {
1906         return -ENODEV;
1907 }
1908 #endif  /* CONFIG_CPU_V7 */