2 * BTS PMU driver for perf
3 * Copyright (c) 2013-2014, Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19 #include <linux/bitops.h>
20 #include <linux/types.h>
21 #include <linux/slab.h>
22 #include <linux/debugfs.h>
23 #include <linux/device.h>
24 #include <linux/coredump.h>
26 #include <asm-generic/sizes.h>
27 #include <asm/perf_event.h>
29 #include "perf_event.h"
32 struct perf_output_handle handle;
33 struct debug_store ds_back;
37 static DEFINE_PER_CPU(struct bts_ctx, bts_ctx);
39 #define BTS_RECORD_SIZE 24
40 #define BTS_SAFETY_MARGIN 4080
46 unsigned long displacement;
50 size_t real_size; /* multiple of BTS_RECORD_SIZE */
51 unsigned int nr_pages;
60 struct bts_phys buf[0];
65 void intel_pmu_enable_bts(u64 config);
66 void intel_pmu_disable_bts(void);
68 static size_t buf_size(struct page *page)
70 return 1 << (PAGE_SHIFT + page_private(page));
74 bts_buffer_setup_aux(int cpu, void **pages, int nr_pages, bool overwrite)
76 struct bts_buffer *buf;
78 int node = (cpu == -1) ? cpu : cpu_to_node(cpu);
80 size_t size = nr_pages << PAGE_SHIFT;
83 /* count all the high order buffers */
84 for (pg = 0, nbuf = 0; pg < nr_pages;) {
85 page = virt_to_page(pages[pg]);
86 if (WARN_ON_ONCE(!PagePrivate(page) && nr_pages > 1))
88 pg += 1 << page_private(page);
93 * to avoid interrupts in overwrite mode, only allow one physical
95 if (overwrite && nbuf > 1)
98 buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);
102 buf->nr_pages = nr_pages;
104 buf->snapshot = overwrite;
105 buf->data_pages = pages;
106 buf->real_size = size - size % BTS_RECORD_SIZE;
108 for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {
109 unsigned int __nr_pages;
111 page = virt_to_page(pages[pg]);
112 __nr_pages = PagePrivate(page) ? 1 << page_private(page) : 1;
113 buf->buf[nbuf].page = page;
114 buf->buf[nbuf].offset = offset;
115 buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
116 buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;
117 pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;
118 buf->buf[nbuf].size -= pad;
121 offset += __nr_pages << PAGE_SHIFT;
127 static void bts_buffer_free_aux(void *data)
132 static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)
134 return buf->buf[idx].offset + buf->buf[idx].displacement;
138 bts_config_buffer(struct bts_buffer *buf)
140 int cpu = raw_smp_processor_id();
141 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
142 struct bts_phys *phys = &buf->buf[buf->cur_buf];
143 unsigned long index, thresh = 0, end = phys->size;
144 struct page *page = phys->page;
146 index = local_read(&buf->head);
148 if (!buf->snapshot) {
149 if (buf->end < phys->offset + buf_size(page))
150 end = buf->end - phys->offset - phys->displacement;
152 index -= phys->offset + phys->displacement;
154 if (end - index > BTS_SAFETY_MARGIN)
155 thresh = end - BTS_SAFETY_MARGIN;
156 else if (end - index > BTS_RECORD_SIZE)
157 thresh = end - BTS_RECORD_SIZE;
162 ds->bts_buffer_base = (u64)(long)page_address(page) + phys->displacement;
163 ds->bts_index = ds->bts_buffer_base + index;
164 ds->bts_absolute_maximum = ds->bts_buffer_base + end;
165 ds->bts_interrupt_threshold = !buf->snapshot
166 ? ds->bts_buffer_base + thresh
167 : ds->bts_absolute_maximum + BTS_RECORD_SIZE;
170 static void bts_buffer_pad_out(struct bts_phys *phys, unsigned long head)
172 unsigned long index = head - phys->offset;
174 memset(page_address(phys->page) + index, 0, phys->size - index);
177 static bool bts_buffer_is_full(struct bts_buffer *buf, struct bts_ctx *bts)
182 if (local_read(&buf->data_size) >= bts->handle.size ||
183 bts->handle.size - local_read(&buf->data_size) < BTS_RECORD_SIZE)
189 static void bts_update(struct bts_ctx *bts)
191 int cpu = raw_smp_processor_id();
192 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
193 struct bts_buffer *buf = perf_get_aux(&bts->handle);
194 unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head;
199 head = index + bts_buffer_offset(buf, buf->cur_buf);
200 old = local_xchg(&buf->head, head);
202 if (!buf->snapshot) {
206 if (ds->bts_index >= ds->bts_absolute_maximum)
207 local_inc(&buf->lost);
210 * old and head are always in the same physical buffer, so we
211 * can subtract them to get the data size.
213 local_add(head - old, &buf->data_size);
215 local_set(&buf->data_size, head);
219 static void __bts_event_start(struct perf_event *event)
221 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
222 struct bts_buffer *buf = perf_get_aux(&bts->handle);
225 if (!buf || bts_buffer_is_full(buf, bts))
231 config |= ARCH_PERFMON_EVENTSEL_INT;
232 if (!event->attr.exclude_kernel)
233 config |= ARCH_PERFMON_EVENTSEL_OS;
234 if (!event->attr.exclude_user)
235 config |= ARCH_PERFMON_EVENTSEL_USR;
237 bts_config_buffer(buf);
240 * local barrier to make sure that ds configuration made it
241 * before we enable BTS
245 intel_pmu_enable_bts(config);
248 static void bts_event_start(struct perf_event *event, int flags)
250 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
252 __bts_event_start(event);
254 /* PMI handler: this counter is running and likely generating PMIs */
255 ACCESS_ONCE(bts->started) = 1;
258 static void __bts_event_stop(struct perf_event *event)
261 * No extra synchronization is mandated by the documentation to have
262 * BTS data stores globally visible.
264 intel_pmu_disable_bts();
266 if (event->hw.state & PERF_HES_STOPPED)
269 ACCESS_ONCE(event->hw.state) |= PERF_HES_STOPPED;
272 static void bts_event_stop(struct perf_event *event, int flags)
274 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
276 /* PMI handler: don't restart this counter */
277 ACCESS_ONCE(bts->started) = 0;
279 __bts_event_stop(event);
281 if (flags & PERF_EF_UPDATE)
285 void intel_bts_enable_local(void)
287 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
289 if (bts->handle.event && bts->started)
290 __bts_event_start(bts->handle.event);
293 void intel_bts_disable_local(void)
295 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
297 if (bts->handle.event)
298 __bts_event_stop(bts->handle.event);
302 bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
304 unsigned long head, space, next_space, pad, gap, skip, wakeup;
305 unsigned int next_buf;
306 struct bts_phys *phys, *next_phys;
312 head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
313 if (WARN_ON_ONCE(head != local_read(&buf->head)))
316 phys = &buf->buf[buf->cur_buf];
317 space = phys->offset + phys->displacement + phys->size - head;
319 if (space > handle->size) {
320 space = handle->size;
321 space -= space % BTS_RECORD_SIZE;
323 if (space <= BTS_SAFETY_MARGIN) {
324 /* See if next phys buffer has more space */
325 next_buf = buf->cur_buf + 1;
326 if (next_buf >= buf->nr_bufs)
328 next_phys = &buf->buf[next_buf];
329 gap = buf_size(phys->page) - phys->displacement - phys->size +
330 next_phys->displacement;
332 if (handle->size >= skip) {
333 next_space = next_phys->size;
334 if (next_space + skip > handle->size) {
335 next_space = handle->size - skip;
336 next_space -= next_space % BTS_RECORD_SIZE;
338 if (next_space > space || !space) {
340 bts_buffer_pad_out(phys, head);
341 ret = perf_aux_output_skip(handle, skip);
344 /* Advance to next phys buffer */
347 head = phys->offset + phys->displacement;
349 * After this, cur_buf and head won't match ds
350 * anymore, so we must not be racing with
353 buf->cur_buf = next_buf;
354 local_set(&buf->head, head);
359 /* Don't go far beyond wakeup watermark */
360 wakeup = BTS_SAFETY_MARGIN + BTS_RECORD_SIZE + handle->wakeup -
362 if (space > wakeup) {
364 space -= space % BTS_RECORD_SIZE;
367 buf->end = head + space;
370 * If we have no space, the lost notification would have been sent when
371 * we hit absolute_maximum - see bts_update()
379 int intel_bts_interrupt(void)
381 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
382 struct perf_event *event = bts->handle.event;
383 struct bts_buffer *buf;
387 if (!event || !bts->started)
390 buf = perf_get_aux(&bts->handle);
392 * Skip snapshot counters: they don't use the interrupt, but
393 * there's no other way of telling, because the pointer will
396 if (!buf || buf->snapshot)
399 old_head = local_read(&buf->head);
403 if (old_head == local_read(&buf->head))
406 perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),
407 !!local_xchg(&buf->lost, 0));
409 buf = perf_aux_output_begin(&bts->handle, event);
413 err = bts_buffer_reset(buf, &bts->handle);
415 perf_aux_output_end(&bts->handle, 0, false);
420 static void bts_event_del(struct perf_event *event, int mode)
422 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
423 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
424 struct bts_buffer *buf = perf_get_aux(&bts->handle);
426 bts_event_stop(event, PERF_EF_UPDATE);
431 local_xchg(&buf->data_size,
432 buf->nr_pages << PAGE_SHIFT);
433 perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0),
434 !!local_xchg(&buf->lost, 0));
437 cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;
438 cpuc->ds->bts_buffer_base = bts->ds_back.bts_buffer_base;
439 cpuc->ds->bts_absolute_maximum = bts->ds_back.bts_absolute_maximum;
440 cpuc->ds->bts_interrupt_threshold = bts->ds_back.bts_interrupt_threshold;
443 static int bts_event_add(struct perf_event *event, int mode)
445 struct bts_buffer *buf;
446 struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
447 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
448 struct hw_perf_event *hwc = &event->hw;
451 event->hw.state = PERF_HES_STOPPED;
453 if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
456 if (bts->handle.event)
459 buf = perf_aux_output_begin(&bts->handle, event);
463 ret = bts_buffer_reset(buf, &bts->handle);
465 perf_aux_output_end(&bts->handle, 0, false);
469 bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base;
470 bts->ds_back.bts_absolute_maximum = cpuc->ds->bts_absolute_maximum;
471 bts->ds_back.bts_interrupt_threshold = cpuc->ds->bts_interrupt_threshold;
473 if (mode & PERF_EF_START) {
474 bts_event_start(event, 0);
475 if (hwc->state & PERF_HES_STOPPED) {
476 bts_event_del(event, 0);
484 static void bts_event_destroy(struct perf_event *event)
486 x86_release_hardware();
487 x86_del_exclusive(x86_lbr_exclusive_bts);
490 static int bts_event_init(struct perf_event *event)
494 if (event->attr.type != bts_pmu.type)
497 if (x86_add_exclusive(x86_lbr_exclusive_bts))
500 ret = x86_reserve_hardware();
502 x86_del_exclusive(x86_lbr_exclusive_bts);
506 event->destroy = bts_event_destroy;
511 static void bts_event_read(struct perf_event *event)
515 static __init int bts_init(void)
517 if (!boot_cpu_has(X86_FEATURE_DTES64) || !x86_pmu.bts)
520 bts_pmu.capabilities = PERF_PMU_CAP_AUX_NO_SG | PERF_PMU_CAP_ITRACE;
521 bts_pmu.task_ctx_nr = perf_sw_context;
522 bts_pmu.event_init = bts_event_init;
523 bts_pmu.add = bts_event_add;
524 bts_pmu.del = bts_event_del;
525 bts_pmu.start = bts_event_start;
526 bts_pmu.stop = bts_event_stop;
527 bts_pmu.read = bts_event_read;
528 bts_pmu.setup_aux = bts_buffer_setup_aux;
529 bts_pmu.free_aux = bts_buffer_free_aux;
531 return perf_pmu_register(&bts_pmu, "intel_bts", -1);
534 module_init(bts_init);