2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/tracefs.h>
24 #include <linux/pagemap.h>
25 #include <linux/hardirq.h>
26 #include <linux/linkage.h>
27 #include <linux/uaccess.h>
28 #include <linux/kprobes.h>
29 #include <linux/ftrace.h>
30 #include <linux/module.h>
31 #include <linux/percpu.h>
32 #include <linux/splice.h>
33 #include <linux/kdebug.h>
34 #include <linux/string.h>
35 #include <linux/mount.h>
36 #include <linux/rwsem.h>
37 #include <linux/slab.h>
38 #include <linux/ctype.h>
39 #include <linux/init.h>
40 #include <linux/poll.h>
41 #include <linux/nmi.h>
43 #include <linux/sched/rt.h>
46 #include "trace_output.h"
49 * On boot up, the ring buffer is set to the minimum size, so that
50 * we do not waste memory on systems that are not using tracing.
52 bool ring_buffer_expanded;
55 * We need to change this state when a selftest is running.
56 * A selftest will lurk into the ring-buffer to count the
57 * entries inserted during the selftest although some concurrent
58 * insertions into the ring-buffer such as trace_printk could occurred
59 * at the same time, giving false positive or negative results.
61 static bool __read_mostly tracing_selftest_running;
64 * If a tracer is running, we do not want to run SELFTEST.
66 bool __read_mostly tracing_selftest_disabled;
68 /* Pipe tracepoints to printk */
69 struct trace_iterator *tracepoint_print_iter;
70 int tracepoint_printk;
72 /* For tracers that don't implement custom flags */
73 static struct tracer_opt dummy_tracer_opt[] = {
77 static struct tracer_flags dummy_tracer_flags = {
79 .opts = dummy_tracer_opt
83 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
89 * To prevent the comm cache from being overwritten when no
90 * tracing is active, only save the comm when a trace event
93 static DEFINE_PER_CPU(bool, trace_cmdline_save);
96 * Kill all tracing for good (never come back).
97 * It is initialized to 1 but will turn to zero if the initialization
98 * of the tracer is successful. But that is the only place that sets
101 static int tracing_disabled = 1;
103 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
105 cpumask_var_t __read_mostly tracing_buffer_mask;
108 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
110 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
111 * is set, then ftrace_dump is called. This will output the contents
112 * of the ftrace buffers to the console. This is very useful for
113 * capturing traces that lead to crashes and outputing it to a
116 * It is default off, but you can enable it with either specifying
117 * "ftrace_dump_on_oops" in the kernel command line, or setting
118 * /proc/sys/kernel/ftrace_dump_on_oops
119 * Set 1 if you want to dump buffers of all CPUs
120 * Set 2 if you want to dump the buffer of the CPU that triggered oops
123 enum ftrace_dump_mode ftrace_dump_on_oops;
125 /* When set, tracing will stop when a WARN*() is hit */
126 int __disable_trace_on_warning;
128 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
129 /* Map of enums to their values, for "enum_map" file */
130 struct trace_enum_map_head {
132 unsigned long length;
135 union trace_enum_map_item;
137 struct trace_enum_map_tail {
139 * "end" is first and points to NULL as it must be different
140 * than "mod" or "enum_string"
142 union trace_enum_map_item *next;
143 const char *end; /* points to NULL */
146 static DEFINE_MUTEX(trace_enum_mutex);
149 * The trace_enum_maps are saved in an array with two extra elements,
150 * one at the beginning, and one at the end. The beginning item contains
151 * the count of the saved maps (head.length), and the module they
152 * belong to if not built in (head.mod). The ending item contains a
153 * pointer to the next array of saved enum_map items.
155 union trace_enum_map_item {
156 struct trace_enum_map map;
157 struct trace_enum_map_head head;
158 struct trace_enum_map_tail tail;
161 static union trace_enum_map_item *trace_enum_maps;
162 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
164 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
166 #define MAX_TRACER_SIZE 100
167 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
168 static char *default_bootup_tracer;
170 static bool allocate_snapshot;
172 static int __init set_cmdline_ftrace(char *str)
174 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
175 default_bootup_tracer = bootup_tracer_buf;
176 /* We are using ftrace early, expand it */
177 ring_buffer_expanded = true;
180 __setup("ftrace=", set_cmdline_ftrace);
182 static int __init set_ftrace_dump_on_oops(char *str)
184 if (*str++ != '=' || !*str) {
185 ftrace_dump_on_oops = DUMP_ALL;
189 if (!strcmp("orig_cpu", str)) {
190 ftrace_dump_on_oops = DUMP_ORIG;
196 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
198 static int __init stop_trace_on_warning(char *str)
200 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
201 __disable_trace_on_warning = 1;
204 __setup("traceoff_on_warning", stop_trace_on_warning);
206 static int __init boot_alloc_snapshot(char *str)
208 allocate_snapshot = true;
209 /* We also need the main ring buffer expanded */
210 ring_buffer_expanded = true;
213 __setup("alloc_snapshot", boot_alloc_snapshot);
216 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
217 static char *trace_boot_options __initdata;
219 static int __init set_trace_boot_options(char *str)
221 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
222 trace_boot_options = trace_boot_options_buf;
225 __setup("trace_options=", set_trace_boot_options);
227 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
228 static char *trace_boot_clock __initdata;
230 static int __init set_trace_boot_clock(char *str)
232 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
233 trace_boot_clock = trace_boot_clock_buf;
236 __setup("trace_clock=", set_trace_boot_clock);
238 static int __init set_tracepoint_printk(char *str)
240 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
241 tracepoint_printk = 1;
244 __setup("tp_printk", set_tracepoint_printk);
246 unsigned long long ns2usecs(cycle_t nsec)
254 * The global_trace is the descriptor that holds the tracing
255 * buffers for the live tracing. For each CPU, it contains
256 * a link list of pages that will store trace entries. The
257 * page descriptor of the pages in the memory is used to hold
258 * the link list by linking the lru item in the page descriptor
259 * to each of the pages in the buffer per CPU.
261 * For each active CPU there is a data field that holds the
262 * pages for the buffer for that CPU. Each CPU has the same number
263 * of pages allocated for its buffer.
265 static struct trace_array global_trace;
267 LIST_HEAD(ftrace_trace_arrays);
269 int trace_array_get(struct trace_array *this_tr)
271 struct trace_array *tr;
274 mutex_lock(&trace_types_lock);
275 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
282 mutex_unlock(&trace_types_lock);
287 static void __trace_array_put(struct trace_array *this_tr)
289 WARN_ON(!this_tr->ref);
293 void trace_array_put(struct trace_array *this_tr)
295 mutex_lock(&trace_types_lock);
296 __trace_array_put(this_tr);
297 mutex_unlock(&trace_types_lock);
300 int filter_check_discard(struct ftrace_event_file *file, void *rec,
301 struct ring_buffer *buffer,
302 struct ring_buffer_event *event)
304 if (unlikely(file->flags & FTRACE_EVENT_FL_FILTERED) &&
305 !filter_match_preds(file->filter, rec)) {
306 ring_buffer_discard_commit(buffer, event);
312 EXPORT_SYMBOL_GPL(filter_check_discard);
314 int call_filter_check_discard(struct ftrace_event_call *call, void *rec,
315 struct ring_buffer *buffer,
316 struct ring_buffer_event *event)
318 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
319 !filter_match_preds(call->filter, rec)) {
320 ring_buffer_discard_commit(buffer, event);
326 EXPORT_SYMBOL_GPL(call_filter_check_discard);
328 static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
332 /* Early boot up does not have a buffer yet */
334 return trace_clock_local();
336 ts = ring_buffer_time_stamp(buf->buffer, cpu);
337 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
342 cycle_t ftrace_now(int cpu)
344 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
348 * tracing_is_enabled - Show if global_trace has been disabled
350 * Shows if the global trace has been enabled or not. It uses the
351 * mirror flag "buffer_disabled" to be used in fast paths such as for
352 * the irqsoff tracer. But it may be inaccurate due to races. If you
353 * need to know the accurate state, use tracing_is_on() which is a little
354 * slower, but accurate.
356 int tracing_is_enabled(void)
359 * For quick access (irqsoff uses this in fast path), just
360 * return the mirror variable of the state of the ring buffer.
361 * It's a little racy, but we don't really care.
364 return !global_trace.buffer_disabled;
368 * trace_buf_size is the size in bytes that is allocated
369 * for a buffer. Note, the number of bytes is always rounded
372 * This number is purposely set to a low number of 16384.
373 * If the dump on oops happens, it will be much appreciated
374 * to not have to wait for all that output. Anyway this can be
375 * boot time and run time configurable.
377 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
379 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
381 /* trace_types holds a link list of available tracers. */
382 static struct tracer *trace_types __read_mostly;
385 * trace_types_lock is used to protect the trace_types list.
387 DEFINE_MUTEX(trace_types_lock);
390 * serialize the access of the ring buffer
392 * ring buffer serializes readers, but it is low level protection.
393 * The validity of the events (which returns by ring_buffer_peek() ..etc)
394 * are not protected by ring buffer.
396 * The content of events may become garbage if we allow other process consumes
397 * these events concurrently:
398 * A) the page of the consumed events may become a normal page
399 * (not reader page) in ring buffer, and this page will be rewrited
400 * by events producer.
401 * B) The page of the consumed events may become a page for splice_read,
402 * and this page will be returned to system.
404 * These primitives allow multi process access to different cpu ring buffer
407 * These primitives don't distinguish read-only and read-consume access.
408 * Multi read-only access are also serialized.
412 static DECLARE_RWSEM(all_cpu_access_lock);
413 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
415 static inline void trace_access_lock(int cpu)
417 if (cpu == RING_BUFFER_ALL_CPUS) {
418 /* gain it for accessing the whole ring buffer. */
419 down_write(&all_cpu_access_lock);
421 /* gain it for accessing a cpu ring buffer. */
423 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
424 down_read(&all_cpu_access_lock);
426 /* Secondly block other access to this @cpu ring buffer. */
427 mutex_lock(&per_cpu(cpu_access_lock, cpu));
431 static inline void trace_access_unlock(int cpu)
433 if (cpu == RING_BUFFER_ALL_CPUS) {
434 up_write(&all_cpu_access_lock);
436 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
437 up_read(&all_cpu_access_lock);
441 static inline void trace_access_lock_init(void)
445 for_each_possible_cpu(cpu)
446 mutex_init(&per_cpu(cpu_access_lock, cpu));
451 static DEFINE_MUTEX(access_lock);
453 static inline void trace_access_lock(int cpu)
456 mutex_lock(&access_lock);
459 static inline void trace_access_unlock(int cpu)
462 mutex_unlock(&access_lock);
465 static inline void trace_access_lock_init(void)
471 /* trace_flags holds trace_options default values */
472 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
473 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
474 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
475 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
477 static void tracer_tracing_on(struct trace_array *tr)
479 if (tr->trace_buffer.buffer)
480 ring_buffer_record_on(tr->trace_buffer.buffer);
482 * This flag is looked at when buffers haven't been allocated
483 * yet, or by some tracers (like irqsoff), that just want to
484 * know if the ring buffer has been disabled, but it can handle
485 * races of where it gets disabled but we still do a record.
486 * As the check is in the fast path of the tracers, it is more
487 * important to be fast than accurate.
489 tr->buffer_disabled = 0;
490 /* Make the flag seen by readers */
495 * tracing_on - enable tracing buffers
497 * This function enables tracing buffers that may have been
498 * disabled with tracing_off.
500 void tracing_on(void)
502 tracer_tracing_on(&global_trace);
504 EXPORT_SYMBOL_GPL(tracing_on);
507 * __trace_puts - write a constant string into the trace buffer.
508 * @ip: The address of the caller
509 * @str: The constant string to write
510 * @size: The size of the string.
512 int __trace_puts(unsigned long ip, const char *str, int size)
514 struct ring_buffer_event *event;
515 struct ring_buffer *buffer;
516 struct print_entry *entry;
517 unsigned long irq_flags;
521 if (!(trace_flags & TRACE_ITER_PRINTK))
524 pc = preempt_count();
526 if (unlikely(tracing_selftest_running || tracing_disabled))
529 alloc = sizeof(*entry) + size + 2; /* possible \n added */
531 local_save_flags(irq_flags);
532 buffer = global_trace.trace_buffer.buffer;
533 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
538 entry = ring_buffer_event_data(event);
541 memcpy(&entry->buf, str, size);
543 /* Add a newline if necessary */
544 if (entry->buf[size - 1] != '\n') {
545 entry->buf[size] = '\n';
546 entry->buf[size + 1] = '\0';
548 entry->buf[size] = '\0';
550 __buffer_unlock_commit(buffer, event);
551 ftrace_trace_stack(buffer, irq_flags, 4, pc);
555 EXPORT_SYMBOL_GPL(__trace_puts);
558 * __trace_bputs - write the pointer to a constant string into trace buffer
559 * @ip: The address of the caller
560 * @str: The constant string to write to the buffer to
562 int __trace_bputs(unsigned long ip, const char *str)
564 struct ring_buffer_event *event;
565 struct ring_buffer *buffer;
566 struct bputs_entry *entry;
567 unsigned long irq_flags;
568 int size = sizeof(struct bputs_entry);
571 if (!(trace_flags & TRACE_ITER_PRINTK))
574 pc = preempt_count();
576 if (unlikely(tracing_selftest_running || tracing_disabled))
579 local_save_flags(irq_flags);
580 buffer = global_trace.trace_buffer.buffer;
581 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
586 entry = ring_buffer_event_data(event);
590 __buffer_unlock_commit(buffer, event);
591 ftrace_trace_stack(buffer, irq_flags, 4, pc);
595 EXPORT_SYMBOL_GPL(__trace_bputs);
597 #ifdef CONFIG_TRACER_SNAPSHOT
599 * trace_snapshot - take a snapshot of the current buffer.
601 * This causes a swap between the snapshot buffer and the current live
602 * tracing buffer. You can use this to take snapshots of the live
603 * trace when some condition is triggered, but continue to trace.
605 * Note, make sure to allocate the snapshot with either
606 * a tracing_snapshot_alloc(), or by doing it manually
607 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
609 * If the snapshot buffer is not allocated, it will stop tracing.
610 * Basically making a permanent snapshot.
612 void tracing_snapshot(void)
614 struct trace_array *tr = &global_trace;
615 struct tracer *tracer = tr->current_trace;
619 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
620 internal_trace_puts("*** snapshot is being ignored ***\n");
624 if (!tr->allocated_snapshot) {
625 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
626 internal_trace_puts("*** stopping trace here! ***\n");
631 /* Note, snapshot can not be used when the tracer uses it */
632 if (tracer->use_max_tr) {
633 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
634 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
638 local_irq_save(flags);
639 update_max_tr(tr, current, smp_processor_id());
640 local_irq_restore(flags);
642 EXPORT_SYMBOL_GPL(tracing_snapshot);
644 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
645 struct trace_buffer *size_buf, int cpu_id);
646 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
648 static int alloc_snapshot(struct trace_array *tr)
652 if (!tr->allocated_snapshot) {
654 /* allocate spare buffer */
655 ret = resize_buffer_duplicate_size(&tr->max_buffer,
656 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
660 tr->allocated_snapshot = true;
666 static void free_snapshot(struct trace_array *tr)
669 * We don't free the ring buffer. instead, resize it because
670 * The max_tr ring buffer has some state (e.g. ring->clock) and
671 * we want preserve it.
673 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
674 set_buffer_entries(&tr->max_buffer, 1);
675 tracing_reset_online_cpus(&tr->max_buffer);
676 tr->allocated_snapshot = false;
680 * tracing_alloc_snapshot - allocate snapshot buffer.
682 * This only allocates the snapshot buffer if it isn't already
683 * allocated - it doesn't also take a snapshot.
685 * This is meant to be used in cases where the snapshot buffer needs
686 * to be set up for events that can't sleep but need to be able to
687 * trigger a snapshot.
689 int tracing_alloc_snapshot(void)
691 struct trace_array *tr = &global_trace;
694 ret = alloc_snapshot(tr);
699 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
702 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
704 * This is similar to trace_snapshot(), but it will allocate the
705 * snapshot buffer if it isn't already allocated. Use this only
706 * where it is safe to sleep, as the allocation may sleep.
708 * This causes a swap between the snapshot buffer and the current live
709 * tracing buffer. You can use this to take snapshots of the live
710 * trace when some condition is triggered, but continue to trace.
712 void tracing_snapshot_alloc(void)
716 ret = tracing_alloc_snapshot();
722 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
724 void tracing_snapshot(void)
726 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
728 EXPORT_SYMBOL_GPL(tracing_snapshot);
729 int tracing_alloc_snapshot(void)
731 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
734 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
735 void tracing_snapshot_alloc(void)
740 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
741 #endif /* CONFIG_TRACER_SNAPSHOT */
743 static void tracer_tracing_off(struct trace_array *tr)
745 if (tr->trace_buffer.buffer)
746 ring_buffer_record_off(tr->trace_buffer.buffer);
748 * This flag is looked at when buffers haven't been allocated
749 * yet, or by some tracers (like irqsoff), that just want to
750 * know if the ring buffer has been disabled, but it can handle
751 * races of where it gets disabled but we still do a record.
752 * As the check is in the fast path of the tracers, it is more
753 * important to be fast than accurate.
755 tr->buffer_disabled = 1;
756 /* Make the flag seen by readers */
761 * tracing_off - turn off tracing buffers
763 * This function stops the tracing buffers from recording data.
764 * It does not disable any overhead the tracers themselves may
765 * be causing. This function simply causes all recording to
766 * the ring buffers to fail.
768 void tracing_off(void)
770 tracer_tracing_off(&global_trace);
772 EXPORT_SYMBOL_GPL(tracing_off);
774 void disable_trace_on_warning(void)
776 if (__disable_trace_on_warning)
781 * tracer_tracing_is_on - show real state of ring buffer enabled
782 * @tr : the trace array to know if ring buffer is enabled
784 * Shows real state of the ring buffer if it is enabled or not.
786 static int tracer_tracing_is_on(struct trace_array *tr)
788 if (tr->trace_buffer.buffer)
789 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
790 return !tr->buffer_disabled;
794 * tracing_is_on - show state of ring buffers enabled
796 int tracing_is_on(void)
798 return tracer_tracing_is_on(&global_trace);
800 EXPORT_SYMBOL_GPL(tracing_is_on);
802 static int __init set_buf_size(char *str)
804 unsigned long buf_size;
808 buf_size = memparse(str, &str);
809 /* nr_entries can not be zero */
812 trace_buf_size = buf_size;
815 __setup("trace_buf_size=", set_buf_size);
817 static int __init set_tracing_thresh(char *str)
819 unsigned long threshold;
824 ret = kstrtoul(str, 0, &threshold);
827 tracing_thresh = threshold * 1000;
830 __setup("tracing_thresh=", set_tracing_thresh);
832 unsigned long nsecs_to_usecs(unsigned long nsecs)
837 /* These must match the bit postions in trace_iterator_flags */
838 static const char *trace_options[] = {
871 int in_ns; /* is this clock in nanoseconds? */
873 { trace_clock_local, "local", 1 },
874 { trace_clock_global, "global", 1 },
875 { trace_clock_counter, "counter", 0 },
876 { trace_clock_jiffies, "uptime", 0 },
877 { trace_clock, "perf", 1 },
878 { ktime_get_mono_fast_ns, "mono", 1 },
883 * trace_parser_get_init - gets the buffer for trace parser
885 int trace_parser_get_init(struct trace_parser *parser, int size)
887 memset(parser, 0, sizeof(*parser));
889 parser->buffer = kmalloc(size, GFP_KERNEL);
898 * trace_parser_put - frees the buffer for trace parser
900 void trace_parser_put(struct trace_parser *parser)
902 kfree(parser->buffer);
906 * trace_get_user - reads the user input string separated by space
907 * (matched by isspace(ch))
909 * For each string found the 'struct trace_parser' is updated,
910 * and the function returns.
912 * Returns number of bytes read.
914 * See kernel/trace/trace.h for 'struct trace_parser' details.
916 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
917 size_t cnt, loff_t *ppos)
924 trace_parser_clear(parser);
926 ret = get_user(ch, ubuf++);
934 * The parser is not finished with the last write,
935 * continue reading the user input without skipping spaces.
938 /* skip white space */
939 while (cnt && isspace(ch)) {
940 ret = get_user(ch, ubuf++);
947 /* only spaces were written */
957 /* read the non-space input */
958 while (cnt && !isspace(ch)) {
959 if (parser->idx < parser->size - 1)
960 parser->buffer[parser->idx++] = ch;
965 ret = get_user(ch, ubuf++);
972 /* We either got finished input or we have to wait for another call. */
974 parser->buffer[parser->idx] = 0;
975 parser->cont = false;
976 } else if (parser->idx < parser->size - 1) {
978 parser->buffer[parser->idx++] = ch;
991 /* TODO add a seq_buf_to_buffer() */
992 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
996 if (trace_seq_used(s) <= s->seq.readpos)
999 len = trace_seq_used(s) - s->seq.readpos;
1002 memcpy(buf, s->buffer + s->seq.readpos, cnt);
1004 s->seq.readpos += cnt;
1008 unsigned long __read_mostly tracing_thresh;
1010 #ifdef CONFIG_TRACER_MAX_TRACE
1012 * Copy the new maximum trace into the separate maximum-trace
1013 * structure. (this way the maximum trace is permanently saved,
1014 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
1017 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1019 struct trace_buffer *trace_buf = &tr->trace_buffer;
1020 struct trace_buffer *max_buf = &tr->max_buffer;
1021 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1022 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1025 max_buf->time_start = data->preempt_timestamp;
1027 max_data->saved_latency = tr->max_latency;
1028 max_data->critical_start = data->critical_start;
1029 max_data->critical_end = data->critical_end;
1031 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1032 max_data->pid = tsk->pid;
1034 * If tsk == current, then use current_uid(), as that does not use
1035 * RCU. The irq tracer can be called out of RCU scope.
1038 max_data->uid = current_uid();
1040 max_data->uid = task_uid(tsk);
1042 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1043 max_data->policy = tsk->policy;
1044 max_data->rt_priority = tsk->rt_priority;
1046 /* record this tasks comm */
1047 tracing_record_cmdline(tsk);
1051 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1053 * @tsk: the task with the latency
1054 * @cpu: The cpu that initiated the trace.
1056 * Flip the buffers between the @tr and the max_tr and record information
1057 * about which task was the cause of this latency.
1060 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1062 struct ring_buffer *buf;
1067 WARN_ON_ONCE(!irqs_disabled());
1069 if (!tr->allocated_snapshot) {
1070 /* Only the nop tracer should hit this when disabling */
1071 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1075 arch_spin_lock(&tr->max_lock);
1077 buf = tr->trace_buffer.buffer;
1078 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1079 tr->max_buffer.buffer = buf;
1081 __update_max_tr(tr, tsk, cpu);
1082 arch_spin_unlock(&tr->max_lock);
1086 * update_max_tr_single - only copy one trace over, and reset the rest
1088 * @tsk - task with the latency
1089 * @cpu - the cpu of the buffer to copy.
1091 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1094 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1101 WARN_ON_ONCE(!irqs_disabled());
1102 if (!tr->allocated_snapshot) {
1103 /* Only the nop tracer should hit this when disabling */
1104 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1108 arch_spin_lock(&tr->max_lock);
1110 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1112 if (ret == -EBUSY) {
1114 * We failed to swap the buffer due to a commit taking
1115 * place on this CPU. We fail to record, but we reset
1116 * the max trace buffer (no one writes directly to it)
1117 * and flag that it failed.
1119 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1120 "Failed to swap buffers due to commit in progress\n");
1123 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1125 __update_max_tr(tr, tsk, cpu);
1126 arch_spin_unlock(&tr->max_lock);
1128 #endif /* CONFIG_TRACER_MAX_TRACE */
1130 static int wait_on_pipe(struct trace_iterator *iter, bool full)
1132 /* Iterators are static, they should be filled or empty */
1133 if (trace_buffer_iter(iter, iter->cpu_file))
1136 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1140 #ifdef CONFIG_FTRACE_STARTUP_TEST
1141 static int run_tracer_selftest(struct tracer *type)
1143 struct trace_array *tr = &global_trace;
1144 struct tracer *saved_tracer = tr->current_trace;
1147 if (!type->selftest || tracing_selftest_disabled)
1151 * Run a selftest on this tracer.
1152 * Here we reset the trace buffer, and set the current
1153 * tracer to be this tracer. The tracer can then run some
1154 * internal tracing to verify that everything is in order.
1155 * If we fail, we do not register this tracer.
1157 tracing_reset_online_cpus(&tr->trace_buffer);
1159 tr->current_trace = type;
1161 #ifdef CONFIG_TRACER_MAX_TRACE
1162 if (type->use_max_tr) {
1163 /* If we expanded the buffers, make sure the max is expanded too */
1164 if (ring_buffer_expanded)
1165 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1166 RING_BUFFER_ALL_CPUS);
1167 tr->allocated_snapshot = true;
1171 /* the test is responsible for initializing and enabling */
1172 pr_info("Testing tracer %s: ", type->name);
1173 ret = type->selftest(type, tr);
1174 /* the test is responsible for resetting too */
1175 tr->current_trace = saved_tracer;
1177 printk(KERN_CONT "FAILED!\n");
1178 /* Add the warning after printing 'FAILED' */
1182 /* Only reset on passing, to avoid touching corrupted buffers */
1183 tracing_reset_online_cpus(&tr->trace_buffer);
1185 #ifdef CONFIG_TRACER_MAX_TRACE
1186 if (type->use_max_tr) {
1187 tr->allocated_snapshot = false;
1189 /* Shrink the max buffer again */
1190 if (ring_buffer_expanded)
1191 ring_buffer_resize(tr->max_buffer.buffer, 1,
1192 RING_BUFFER_ALL_CPUS);
1196 printk(KERN_CONT "PASSED\n");
1200 static inline int run_tracer_selftest(struct tracer *type)
1204 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1207 * register_tracer - register a tracer with the ftrace system.
1208 * @type - the plugin for the tracer
1210 * Register a new plugin tracer.
1212 int register_tracer(struct tracer *type)
1218 pr_info("Tracer must have a name\n");
1222 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1223 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1227 mutex_lock(&trace_types_lock);
1229 tracing_selftest_running = true;
1231 for (t = trace_types; t; t = t->next) {
1232 if (strcmp(type->name, t->name) == 0) {
1234 pr_info("Tracer %s already registered\n",
1241 if (!type->set_flag)
1242 type->set_flag = &dummy_set_flag;
1244 type->flags = &dummy_tracer_flags;
1246 if (!type->flags->opts)
1247 type->flags->opts = dummy_tracer_opt;
1249 ret = run_tracer_selftest(type);
1253 type->next = trace_types;
1257 tracing_selftest_running = false;
1258 mutex_unlock(&trace_types_lock);
1260 if (ret || !default_bootup_tracer)
1263 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1266 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1267 /* Do we want this tracer to start on bootup? */
1268 tracing_set_tracer(&global_trace, type->name);
1269 default_bootup_tracer = NULL;
1270 /* disable other selftests, since this will break it. */
1271 tracing_selftest_disabled = true;
1272 #ifdef CONFIG_FTRACE_STARTUP_TEST
1273 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1281 void tracing_reset(struct trace_buffer *buf, int cpu)
1283 struct ring_buffer *buffer = buf->buffer;
1288 ring_buffer_record_disable(buffer);
1290 /* Make sure all commits have finished */
1291 synchronize_sched();
1292 ring_buffer_reset_cpu(buffer, cpu);
1294 ring_buffer_record_enable(buffer);
1297 void tracing_reset_online_cpus(struct trace_buffer *buf)
1299 struct ring_buffer *buffer = buf->buffer;
1305 ring_buffer_record_disable(buffer);
1307 /* Make sure all commits have finished */
1308 synchronize_sched();
1310 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1312 for_each_online_cpu(cpu)
1313 ring_buffer_reset_cpu(buffer, cpu);
1315 ring_buffer_record_enable(buffer);
1318 /* Must have trace_types_lock held */
1319 void tracing_reset_all_online_cpus(void)
1321 struct trace_array *tr;
1323 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1324 tracing_reset_online_cpus(&tr->trace_buffer);
1325 #ifdef CONFIG_TRACER_MAX_TRACE
1326 tracing_reset_online_cpus(&tr->max_buffer);
1331 #define SAVED_CMDLINES_DEFAULT 128
1332 #define NO_CMDLINE_MAP UINT_MAX
1333 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1334 struct saved_cmdlines_buffer {
1335 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1336 unsigned *map_cmdline_to_pid;
1337 unsigned cmdline_num;
1339 char *saved_cmdlines;
1341 static struct saved_cmdlines_buffer *savedcmd;
1343 /* temporary disable recording */
1344 static atomic_t trace_record_cmdline_disabled __read_mostly;
1346 static inline char *get_saved_cmdlines(int idx)
1348 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1351 static inline void set_cmdline(int idx, const char *cmdline)
1353 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1356 static int allocate_cmdlines_buffer(unsigned int val,
1357 struct saved_cmdlines_buffer *s)
1359 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1361 if (!s->map_cmdline_to_pid)
1364 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1365 if (!s->saved_cmdlines) {
1366 kfree(s->map_cmdline_to_pid);
1371 s->cmdline_num = val;
1372 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1373 sizeof(s->map_pid_to_cmdline));
1374 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1375 val * sizeof(*s->map_cmdline_to_pid));
1380 static int trace_create_savedcmd(void)
1384 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1388 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1398 int is_tracing_stopped(void)
1400 return global_trace.stop_count;
1404 * tracing_start - quick start of the tracer
1406 * If tracing is enabled but was stopped by tracing_stop,
1407 * this will start the tracer back up.
1409 void tracing_start(void)
1411 struct ring_buffer *buffer;
1412 unsigned long flags;
1414 if (tracing_disabled)
1417 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1418 if (--global_trace.stop_count) {
1419 if (global_trace.stop_count < 0) {
1420 /* Someone screwed up their debugging */
1422 global_trace.stop_count = 0;
1427 /* Prevent the buffers from switching */
1428 arch_spin_lock(&global_trace.max_lock);
1430 buffer = global_trace.trace_buffer.buffer;
1432 ring_buffer_record_enable(buffer);
1434 #ifdef CONFIG_TRACER_MAX_TRACE
1435 buffer = global_trace.max_buffer.buffer;
1437 ring_buffer_record_enable(buffer);
1440 arch_spin_unlock(&global_trace.max_lock);
1443 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1446 static void tracing_start_tr(struct trace_array *tr)
1448 struct ring_buffer *buffer;
1449 unsigned long flags;
1451 if (tracing_disabled)
1454 /* If global, we need to also start the max tracer */
1455 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1456 return tracing_start();
1458 raw_spin_lock_irqsave(&tr->start_lock, flags);
1460 if (--tr->stop_count) {
1461 if (tr->stop_count < 0) {
1462 /* Someone screwed up their debugging */
1469 buffer = tr->trace_buffer.buffer;
1471 ring_buffer_record_enable(buffer);
1474 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1478 * tracing_stop - quick stop of the tracer
1480 * Light weight way to stop tracing. Use in conjunction with
1483 void tracing_stop(void)
1485 struct ring_buffer *buffer;
1486 unsigned long flags;
1488 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1489 if (global_trace.stop_count++)
1492 /* Prevent the buffers from switching */
1493 arch_spin_lock(&global_trace.max_lock);
1495 buffer = global_trace.trace_buffer.buffer;
1497 ring_buffer_record_disable(buffer);
1499 #ifdef CONFIG_TRACER_MAX_TRACE
1500 buffer = global_trace.max_buffer.buffer;
1502 ring_buffer_record_disable(buffer);
1505 arch_spin_unlock(&global_trace.max_lock);
1508 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1511 static void tracing_stop_tr(struct trace_array *tr)
1513 struct ring_buffer *buffer;
1514 unsigned long flags;
1516 /* If global, we need to also stop the max tracer */
1517 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1518 return tracing_stop();
1520 raw_spin_lock_irqsave(&tr->start_lock, flags);
1521 if (tr->stop_count++)
1524 buffer = tr->trace_buffer.buffer;
1526 ring_buffer_record_disable(buffer);
1529 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1532 void trace_stop_cmdline_recording(void);
1534 static int trace_save_cmdline(struct task_struct *tsk)
1538 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1542 * It's not the end of the world if we don't get
1543 * the lock, but we also don't want to spin
1544 * nor do we want to disable interrupts,
1545 * so if we miss here, then better luck next time.
1547 if (!arch_spin_trylock(&trace_cmdline_lock))
1550 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
1551 if (idx == NO_CMDLINE_MAP) {
1552 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1555 * Check whether the cmdline buffer at idx has a pid
1556 * mapped. We are going to overwrite that entry so we
1557 * need to clear the map_pid_to_cmdline. Otherwise we
1558 * would read the new comm for the old pid.
1560 pid = savedcmd->map_cmdline_to_pid[idx];
1561 if (pid != NO_CMDLINE_MAP)
1562 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1564 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1565 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
1567 savedcmd->cmdline_idx = idx;
1570 set_cmdline(idx, tsk->comm);
1572 arch_spin_unlock(&trace_cmdline_lock);
1577 static void __trace_find_cmdline(int pid, char comm[])
1582 strcpy(comm, "<idle>");
1586 if (WARN_ON_ONCE(pid < 0)) {
1587 strcpy(comm, "<XXX>");
1591 if (pid > PID_MAX_DEFAULT) {
1592 strcpy(comm, "<...>");
1596 map = savedcmd->map_pid_to_cmdline[pid];
1597 if (map != NO_CMDLINE_MAP)
1598 strcpy(comm, get_saved_cmdlines(map));
1600 strcpy(comm, "<...>");
1603 void trace_find_cmdline(int pid, char comm[])
1606 arch_spin_lock(&trace_cmdline_lock);
1608 __trace_find_cmdline(pid, comm);
1610 arch_spin_unlock(&trace_cmdline_lock);
1614 void tracing_record_cmdline(struct task_struct *tsk)
1616 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1619 if (!__this_cpu_read(trace_cmdline_save))
1622 if (trace_save_cmdline(tsk))
1623 __this_cpu_write(trace_cmdline_save, false);
1627 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1630 struct task_struct *tsk = current;
1632 entry->preempt_count = pc & 0xff;
1633 entry->preempt_lazy_count = preempt_lazy_count();
1634 entry->pid = (tsk) ? tsk->pid : 0;
1636 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1637 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1639 TRACE_FLAG_IRQS_NOSUPPORT |
1641 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1642 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1643 (tif_need_resched_now() ? TRACE_FLAG_NEED_RESCHED : 0) |
1644 (need_resched_lazy() ? TRACE_FLAG_NEED_RESCHED_LAZY : 0) |
1645 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1647 entry->migrate_disable = (tsk) ? __migrate_disabled(tsk) & 0xFF : 0;
1649 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1651 struct ring_buffer_event *
1652 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1655 unsigned long flags, int pc)
1657 struct ring_buffer_event *event;
1659 event = ring_buffer_lock_reserve(buffer, len);
1660 if (event != NULL) {
1661 struct trace_entry *ent = ring_buffer_event_data(event);
1663 tracing_generic_entry_update(ent, flags, pc);
1671 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1673 __this_cpu_write(trace_cmdline_save, true);
1674 ring_buffer_unlock_commit(buffer, event);
1678 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1679 struct ring_buffer_event *event,
1680 unsigned long flags, int pc)
1682 __buffer_unlock_commit(buffer, event);
1684 ftrace_trace_stack(buffer, flags, 6, pc);
1685 ftrace_trace_userstack(buffer, flags, pc);
1688 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1689 struct ring_buffer_event *event,
1690 unsigned long flags, int pc)
1692 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1694 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1696 static struct ring_buffer *temp_buffer;
1698 struct ring_buffer_event *
1699 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1700 struct ftrace_event_file *ftrace_file,
1701 int type, unsigned long len,
1702 unsigned long flags, int pc)
1704 struct ring_buffer_event *entry;
1706 *current_rb = ftrace_file->tr->trace_buffer.buffer;
1707 entry = trace_buffer_lock_reserve(*current_rb,
1708 type, len, flags, pc);
1710 * If tracing is off, but we have triggers enabled
1711 * we still need to look at the event data. Use the temp_buffer
1712 * to store the trace event for the tigger to use. It's recusive
1713 * safe and will not be recorded anywhere.
1715 if (!entry && ftrace_file->flags & FTRACE_EVENT_FL_TRIGGER_COND) {
1716 *current_rb = temp_buffer;
1717 entry = trace_buffer_lock_reserve(*current_rb,
1718 type, len, flags, pc);
1722 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1724 struct ring_buffer_event *
1725 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1726 int type, unsigned long len,
1727 unsigned long flags, int pc)
1729 *current_rb = global_trace.trace_buffer.buffer;
1730 return trace_buffer_lock_reserve(*current_rb,
1731 type, len, flags, pc);
1733 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1735 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1736 struct ring_buffer_event *event,
1737 unsigned long flags, int pc)
1739 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1741 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1743 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1744 struct ring_buffer_event *event,
1745 unsigned long flags, int pc,
1746 struct pt_regs *regs)
1748 __buffer_unlock_commit(buffer, event);
1750 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1751 ftrace_trace_userstack(buffer, flags, pc);
1753 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1755 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1756 struct ring_buffer_event *event)
1758 ring_buffer_discard_commit(buffer, event);
1760 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1763 trace_function(struct trace_array *tr,
1764 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1767 struct ftrace_event_call *call = &event_function;
1768 struct ring_buffer *buffer = tr->trace_buffer.buffer;
1769 struct ring_buffer_event *event;
1770 struct ftrace_entry *entry;
1772 /* If we are reading the ring buffer, don't trace */
1773 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1776 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1780 entry = ring_buffer_event_data(event);
1782 entry->parent_ip = parent_ip;
1784 if (!call_filter_check_discard(call, entry, buffer, event))
1785 __buffer_unlock_commit(buffer, event);
1788 #ifdef CONFIG_STACKTRACE
1790 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1791 struct ftrace_stack {
1792 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1795 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1796 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1798 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1799 unsigned long flags,
1800 int skip, int pc, struct pt_regs *regs)
1802 struct ftrace_event_call *call = &event_kernel_stack;
1803 struct ring_buffer_event *event;
1804 struct stack_entry *entry;
1805 struct stack_trace trace;
1807 int size = FTRACE_STACK_ENTRIES;
1809 trace.nr_entries = 0;
1813 * Since events can happen in NMIs there's no safe way to
1814 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1815 * or NMI comes in, it will just have to use the default
1816 * FTRACE_STACK_SIZE.
1818 preempt_disable_notrace();
1820 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1822 * We don't need any atomic variables, just a barrier.
1823 * If an interrupt comes in, we don't care, because it would
1824 * have exited and put the counter back to what we want.
1825 * We just need a barrier to keep gcc from moving things
1829 if (use_stack == 1) {
1830 trace.entries = this_cpu_ptr(ftrace_stack.calls);
1831 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1834 save_stack_trace_regs(regs, &trace);
1836 save_stack_trace(&trace);
1838 if (trace.nr_entries > size)
1839 size = trace.nr_entries;
1841 /* From now on, use_stack is a boolean */
1844 size *= sizeof(unsigned long);
1846 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1847 sizeof(*entry) + size, flags, pc);
1850 entry = ring_buffer_event_data(event);
1852 memset(&entry->caller, 0, size);
1855 memcpy(&entry->caller, trace.entries,
1856 trace.nr_entries * sizeof(unsigned long));
1858 trace.max_entries = FTRACE_STACK_ENTRIES;
1859 trace.entries = entry->caller;
1861 save_stack_trace_regs(regs, &trace);
1863 save_stack_trace(&trace);
1866 entry->size = trace.nr_entries;
1868 if (!call_filter_check_discard(call, entry, buffer, event))
1869 __buffer_unlock_commit(buffer, event);
1872 /* Again, don't let gcc optimize things here */
1874 __this_cpu_dec(ftrace_stack_reserve);
1875 preempt_enable_notrace();
1879 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1880 int skip, int pc, struct pt_regs *regs)
1882 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1885 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1888 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1891 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1894 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1897 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1900 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1904 * trace_dump_stack - record a stack back trace in the trace buffer
1905 * @skip: Number of functions to skip (helper handlers)
1907 void trace_dump_stack(int skip)
1909 unsigned long flags;
1911 if (tracing_disabled || tracing_selftest_running)
1914 local_save_flags(flags);
1917 * Skip 3 more, seems to get us at the caller of
1921 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1922 flags, skip, preempt_count(), NULL);
1925 static DEFINE_PER_CPU(int, user_stack_count);
1928 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1930 struct ftrace_event_call *call = &event_user_stack;
1931 struct ring_buffer_event *event;
1932 struct userstack_entry *entry;
1933 struct stack_trace trace;
1935 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1939 * NMIs can not handle page faults, even with fix ups.
1940 * The save user stack can (and often does) fault.
1942 if (unlikely(in_nmi()))
1946 * prevent recursion, since the user stack tracing may
1947 * trigger other kernel events.
1950 if (__this_cpu_read(user_stack_count))
1953 __this_cpu_inc(user_stack_count);
1955 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1956 sizeof(*entry), flags, pc);
1958 goto out_drop_count;
1959 entry = ring_buffer_event_data(event);
1961 entry->tgid = current->tgid;
1962 memset(&entry->caller, 0, sizeof(entry->caller));
1964 trace.nr_entries = 0;
1965 trace.max_entries = FTRACE_STACK_ENTRIES;
1967 trace.entries = entry->caller;
1969 save_stack_trace_user(&trace);
1970 if (!call_filter_check_discard(call, entry, buffer, event))
1971 __buffer_unlock_commit(buffer, event);
1974 __this_cpu_dec(user_stack_count);
1980 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1982 ftrace_trace_userstack(tr, flags, preempt_count());
1986 #endif /* CONFIG_STACKTRACE */
1988 /* created for use with alloc_percpu */
1989 struct trace_buffer_struct {
1990 char buffer[TRACE_BUF_SIZE];
1993 static struct trace_buffer_struct *trace_percpu_buffer;
1994 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1995 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1996 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1999 * The buffer used is dependent on the context. There is a per cpu
2000 * buffer for normal context, softirq contex, hard irq context and
2001 * for NMI context. Thise allows for lockless recording.
2003 * Note, if the buffers failed to be allocated, then this returns NULL
2005 static char *get_trace_buf(void)
2007 struct trace_buffer_struct *percpu_buffer;
2010 * If we have allocated per cpu buffers, then we do not
2011 * need to do any locking.
2014 percpu_buffer = trace_percpu_nmi_buffer;
2016 percpu_buffer = trace_percpu_irq_buffer;
2017 else if (in_softirq())
2018 percpu_buffer = trace_percpu_sirq_buffer;
2020 percpu_buffer = trace_percpu_buffer;
2025 return this_cpu_ptr(&percpu_buffer->buffer[0]);
2028 static int alloc_percpu_trace_buffer(void)
2030 struct trace_buffer_struct *buffers;
2031 struct trace_buffer_struct *sirq_buffers;
2032 struct trace_buffer_struct *irq_buffers;
2033 struct trace_buffer_struct *nmi_buffers;
2035 buffers = alloc_percpu(struct trace_buffer_struct);
2039 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
2043 irq_buffers = alloc_percpu(struct trace_buffer_struct);
2047 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
2051 trace_percpu_buffer = buffers;
2052 trace_percpu_sirq_buffer = sirq_buffers;
2053 trace_percpu_irq_buffer = irq_buffers;
2054 trace_percpu_nmi_buffer = nmi_buffers;
2059 free_percpu(irq_buffers);
2061 free_percpu(sirq_buffers);
2063 free_percpu(buffers);
2065 WARN(1, "Could not allocate percpu trace_printk buffer");
2069 static int buffers_allocated;
2071 void trace_printk_init_buffers(void)
2073 if (buffers_allocated)
2076 if (alloc_percpu_trace_buffer())
2079 /* trace_printk() is for debug use only. Don't use it in production. */
2082 pr_warning("**********************************************************\n");
2083 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2084 pr_warning("** **\n");
2085 pr_warning("** trace_printk() being used. Allocating extra memory. **\n");
2086 pr_warning("** **\n");
2087 pr_warning("** This means that this is a DEBUG kernel and it is **\n");
2088 pr_warning("** unsafe for production use. **\n");
2089 pr_warning("** **\n");
2090 pr_warning("** If you see this message and you are not debugging **\n");
2091 pr_warning("** the kernel, report this immediately to your vendor! **\n");
2092 pr_warning("** **\n");
2093 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2094 pr_warning("**********************************************************\n");
2096 /* Expand the buffers to set size */
2097 tracing_update_buffers();
2099 buffers_allocated = 1;
2102 * trace_printk_init_buffers() can be called by modules.
2103 * If that happens, then we need to start cmdline recording
2104 * directly here. If the global_trace.buffer is already
2105 * allocated here, then this was called by module code.
2107 if (global_trace.trace_buffer.buffer)
2108 tracing_start_cmdline_record();
2111 void trace_printk_start_comm(void)
2113 /* Start tracing comms if trace printk is set */
2114 if (!buffers_allocated)
2116 tracing_start_cmdline_record();
2119 static void trace_printk_start_stop_comm(int enabled)
2121 if (!buffers_allocated)
2125 tracing_start_cmdline_record();
2127 tracing_stop_cmdline_record();
2131 * trace_vbprintk - write binary msg to tracing buffer
2134 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2136 struct ftrace_event_call *call = &event_bprint;
2137 struct ring_buffer_event *event;
2138 struct ring_buffer *buffer;
2139 struct trace_array *tr = &global_trace;
2140 struct bprint_entry *entry;
2141 unsigned long flags;
2143 int len = 0, size, pc;
2145 if (unlikely(tracing_selftest_running || tracing_disabled))
2148 /* Don't pollute graph traces with trace_vprintk internals */
2149 pause_graph_tracing();
2151 pc = preempt_count();
2152 preempt_disable_notrace();
2154 tbuffer = get_trace_buf();
2160 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2162 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2165 local_save_flags(flags);
2166 size = sizeof(*entry) + sizeof(u32) * len;
2167 buffer = tr->trace_buffer.buffer;
2168 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2172 entry = ring_buffer_event_data(event);
2176 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2177 if (!call_filter_check_discard(call, entry, buffer, event)) {
2178 __buffer_unlock_commit(buffer, event);
2179 ftrace_trace_stack(buffer, flags, 6, pc);
2183 preempt_enable_notrace();
2184 unpause_graph_tracing();
2188 EXPORT_SYMBOL_GPL(trace_vbprintk);
2191 __trace_array_vprintk(struct ring_buffer *buffer,
2192 unsigned long ip, const char *fmt, va_list args)
2194 struct ftrace_event_call *call = &event_print;
2195 struct ring_buffer_event *event;
2196 int len = 0, size, pc;
2197 struct print_entry *entry;
2198 unsigned long flags;
2201 if (tracing_disabled || tracing_selftest_running)
2204 /* Don't pollute graph traces with trace_vprintk internals */
2205 pause_graph_tracing();
2207 pc = preempt_count();
2208 preempt_disable_notrace();
2211 tbuffer = get_trace_buf();
2217 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2219 local_save_flags(flags);
2220 size = sizeof(*entry) + len + 1;
2221 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2225 entry = ring_buffer_event_data(event);
2228 memcpy(&entry->buf, tbuffer, len + 1);
2229 if (!call_filter_check_discard(call, entry, buffer, event)) {
2230 __buffer_unlock_commit(buffer, event);
2231 ftrace_trace_stack(buffer, flags, 6, pc);
2234 preempt_enable_notrace();
2235 unpause_graph_tracing();
2240 int trace_array_vprintk(struct trace_array *tr,
2241 unsigned long ip, const char *fmt, va_list args)
2243 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2246 int trace_array_printk(struct trace_array *tr,
2247 unsigned long ip, const char *fmt, ...)
2252 if (!(trace_flags & TRACE_ITER_PRINTK))
2256 ret = trace_array_vprintk(tr, ip, fmt, ap);
2261 int trace_array_printk_buf(struct ring_buffer *buffer,
2262 unsigned long ip, const char *fmt, ...)
2267 if (!(trace_flags & TRACE_ITER_PRINTK))
2271 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2276 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2278 return trace_array_vprintk(&global_trace, ip, fmt, args);
2280 EXPORT_SYMBOL_GPL(trace_vprintk);
2282 static void trace_iterator_increment(struct trace_iterator *iter)
2284 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2288 ring_buffer_read(buf_iter, NULL);
2291 static struct trace_entry *
2292 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2293 unsigned long *lost_events)
2295 struct ring_buffer_event *event;
2296 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2299 event = ring_buffer_iter_peek(buf_iter, ts);
2301 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2305 iter->ent_size = ring_buffer_event_length(event);
2306 return ring_buffer_event_data(event);
2312 static struct trace_entry *
2313 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2314 unsigned long *missing_events, u64 *ent_ts)
2316 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2317 struct trace_entry *ent, *next = NULL;
2318 unsigned long lost_events = 0, next_lost = 0;
2319 int cpu_file = iter->cpu_file;
2320 u64 next_ts = 0, ts;
2326 * If we are in a per_cpu trace file, don't bother by iterating over
2327 * all cpu and peek directly.
2329 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2330 if (ring_buffer_empty_cpu(buffer, cpu_file))
2332 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2334 *ent_cpu = cpu_file;
2339 for_each_tracing_cpu(cpu) {
2341 if (ring_buffer_empty_cpu(buffer, cpu))
2344 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2347 * Pick the entry with the smallest timestamp:
2349 if (ent && (!next || ts < next_ts)) {
2353 next_lost = lost_events;
2354 next_size = iter->ent_size;
2358 iter->ent_size = next_size;
2361 *ent_cpu = next_cpu;
2367 *missing_events = next_lost;
2372 /* Find the next real entry, without updating the iterator itself */
2373 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2374 int *ent_cpu, u64 *ent_ts)
2376 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2379 /* Find the next real entry, and increment the iterator to the next entry */
2380 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2382 iter->ent = __find_next_entry(iter, &iter->cpu,
2383 &iter->lost_events, &iter->ts);
2386 trace_iterator_increment(iter);
2388 return iter->ent ? iter : NULL;
2391 static void trace_consume(struct trace_iterator *iter)
2393 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2394 &iter->lost_events);
2397 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2399 struct trace_iterator *iter = m->private;
2403 WARN_ON_ONCE(iter->leftover);
2407 /* can't go backwards */
2412 ent = trace_find_next_entry_inc(iter);
2416 while (ent && iter->idx < i)
2417 ent = trace_find_next_entry_inc(iter);
2424 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2426 struct ring_buffer_event *event;
2427 struct ring_buffer_iter *buf_iter;
2428 unsigned long entries = 0;
2431 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2433 buf_iter = trace_buffer_iter(iter, cpu);
2437 ring_buffer_iter_reset(buf_iter);
2440 * We could have the case with the max latency tracers
2441 * that a reset never took place on a cpu. This is evident
2442 * by the timestamp being before the start of the buffer.
2444 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2445 if (ts >= iter->trace_buffer->time_start)
2448 ring_buffer_read(buf_iter, NULL);
2451 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2455 * The current tracer is copied to avoid a global locking
2458 static void *s_start(struct seq_file *m, loff_t *pos)
2460 struct trace_iterator *iter = m->private;
2461 struct trace_array *tr = iter->tr;
2462 int cpu_file = iter->cpu_file;
2468 * copy the tracer to avoid using a global lock all around.
2469 * iter->trace is a copy of current_trace, the pointer to the
2470 * name may be used instead of a strcmp(), as iter->trace->name
2471 * will point to the same string as current_trace->name.
2473 mutex_lock(&trace_types_lock);
2474 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2475 *iter->trace = *tr->current_trace;
2476 mutex_unlock(&trace_types_lock);
2478 #ifdef CONFIG_TRACER_MAX_TRACE
2479 if (iter->snapshot && iter->trace->use_max_tr)
2480 return ERR_PTR(-EBUSY);
2483 if (!iter->snapshot)
2484 atomic_inc(&trace_record_cmdline_disabled);
2486 if (*pos != iter->pos) {
2491 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2492 for_each_tracing_cpu(cpu)
2493 tracing_iter_reset(iter, cpu);
2495 tracing_iter_reset(iter, cpu_file);
2498 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2503 * If we overflowed the seq_file before, then we want
2504 * to just reuse the trace_seq buffer again.
2510 p = s_next(m, p, &l);
2514 trace_event_read_lock();
2515 trace_access_lock(cpu_file);
2519 static void s_stop(struct seq_file *m, void *p)
2521 struct trace_iterator *iter = m->private;
2523 #ifdef CONFIG_TRACER_MAX_TRACE
2524 if (iter->snapshot && iter->trace->use_max_tr)
2528 if (!iter->snapshot)
2529 atomic_dec(&trace_record_cmdline_disabled);
2531 trace_access_unlock(iter->cpu_file);
2532 trace_event_read_unlock();
2536 get_total_entries(struct trace_buffer *buf,
2537 unsigned long *total, unsigned long *entries)
2539 unsigned long count;
2545 for_each_tracing_cpu(cpu) {
2546 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2548 * If this buffer has skipped entries, then we hold all
2549 * entries for the trace and we need to ignore the
2550 * ones before the time stamp.
2552 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2553 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2554 /* total is the same as the entries */
2558 ring_buffer_overrun_cpu(buf->buffer, cpu);
2563 static void print_lat_help_header(struct seq_file *m)
2565 seq_puts(m, "# _--------=> CPU# \n"
2566 "# / _-------=> irqs-off \n"
2567 "# | / _------=> need-resched \n"
2568 "# || / _-----=> need-resched_lazy \n"
2569 "# ||| / _----=> hardirq/softirq \n"
2570 "# |||| / _---=> preempt-depth \n"
2571 "# ||||| / _--=> preempt-lazy-depth\n"
2572 "# |||||| / _-=> migrate-disable \n"
2573 "# ||||||| / delay \n"
2574 "# cmd pid |||||||| time | caller \n"
2575 "# \\ / |||||||| \\ | / \n");
2578 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2580 unsigned long total;
2581 unsigned long entries;
2583 get_total_entries(buf, &total, &entries);
2584 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2585 entries, total, num_online_cpus());
2589 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2591 print_event_info(buf, m);
2592 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2596 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2598 print_event_info(buf, m);
2599 seq_puts(m, "# _-----=> irqs-off\n"
2600 "# / _----=> need-resched\n"
2601 "# |/ _-----=> need-resched_lazy\n"
2602 "# || / _---=> hardirq/softirq\n"
2603 "# ||| / _--=> preempt-depth\n"
2604 "# |||| /_--=> preempt-lazy-depth\n"
2605 "# ||||| _-=> migrate-disable \n"
2607 "# TASK-PID CPU# |||||| TIMESTAMP FUNCTION\n"
2608 "# | | | |||||| | |\n");
2612 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2614 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2615 struct trace_buffer *buf = iter->trace_buffer;
2616 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2617 struct tracer *type = iter->trace;
2618 unsigned long entries;
2619 unsigned long total;
2620 const char *name = "preemption";
2624 get_total_entries(buf, &total, &entries);
2626 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2628 seq_puts(m, "# -----------------------------------"
2629 "---------------------------------\n");
2630 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2631 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2632 nsecs_to_usecs(data->saved_latency),
2636 #if defined(CONFIG_PREEMPT_NONE)
2638 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2640 #elif defined(CONFIG_PREEMPT)
2645 /* These are reserved for later use */
2648 seq_printf(m, " #P:%d)\n", num_online_cpus());
2652 seq_puts(m, "# -----------------\n");
2653 seq_printf(m, "# | task: %.16s-%d "
2654 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2655 data->comm, data->pid,
2656 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2657 data->policy, data->rt_priority);
2658 seq_puts(m, "# -----------------\n");
2660 if (data->critical_start) {
2661 seq_puts(m, "# => started at: ");
2662 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2663 trace_print_seq(m, &iter->seq);
2664 seq_puts(m, "\n# => ended at: ");
2665 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2666 trace_print_seq(m, &iter->seq);
2667 seq_puts(m, "\n#\n");
2673 static void test_cpu_buff_start(struct trace_iterator *iter)
2675 struct trace_seq *s = &iter->seq;
2677 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2680 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2683 if (cpumask_test_cpu(iter->cpu, iter->started))
2686 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2689 cpumask_set_cpu(iter->cpu, iter->started);
2691 /* Don't print started cpu buffer for the first entry of the trace */
2693 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2697 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2699 struct trace_seq *s = &iter->seq;
2700 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2701 struct trace_entry *entry;
2702 struct trace_event *event;
2706 test_cpu_buff_start(iter);
2708 event = ftrace_find_event(entry->type);
2710 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2711 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2712 trace_print_lat_context(iter);
2714 trace_print_context(iter);
2717 if (trace_seq_has_overflowed(s))
2718 return TRACE_TYPE_PARTIAL_LINE;
2721 return event->funcs->trace(iter, sym_flags, event);
2723 trace_seq_printf(s, "Unknown type %d\n", entry->type);
2725 return trace_handle_return(s);
2728 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2730 struct trace_seq *s = &iter->seq;
2731 struct trace_entry *entry;
2732 struct trace_event *event;
2736 if (trace_flags & TRACE_ITER_CONTEXT_INFO)
2737 trace_seq_printf(s, "%d %d %llu ",
2738 entry->pid, iter->cpu, iter->ts);
2740 if (trace_seq_has_overflowed(s))
2741 return TRACE_TYPE_PARTIAL_LINE;
2743 event = ftrace_find_event(entry->type);
2745 return event->funcs->raw(iter, 0, event);
2747 trace_seq_printf(s, "%d ?\n", entry->type);
2749 return trace_handle_return(s);
2752 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2754 struct trace_seq *s = &iter->seq;
2755 unsigned char newline = '\n';
2756 struct trace_entry *entry;
2757 struct trace_event *event;
2761 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2762 SEQ_PUT_HEX_FIELD(s, entry->pid);
2763 SEQ_PUT_HEX_FIELD(s, iter->cpu);
2764 SEQ_PUT_HEX_FIELD(s, iter->ts);
2765 if (trace_seq_has_overflowed(s))
2766 return TRACE_TYPE_PARTIAL_LINE;
2769 event = ftrace_find_event(entry->type);
2771 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2772 if (ret != TRACE_TYPE_HANDLED)
2776 SEQ_PUT_FIELD(s, newline);
2778 return trace_handle_return(s);
2781 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2783 struct trace_seq *s = &iter->seq;
2784 struct trace_entry *entry;
2785 struct trace_event *event;
2789 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2790 SEQ_PUT_FIELD(s, entry->pid);
2791 SEQ_PUT_FIELD(s, iter->cpu);
2792 SEQ_PUT_FIELD(s, iter->ts);
2793 if (trace_seq_has_overflowed(s))
2794 return TRACE_TYPE_PARTIAL_LINE;
2797 event = ftrace_find_event(entry->type);
2798 return event ? event->funcs->binary(iter, 0, event) :
2802 int trace_empty(struct trace_iterator *iter)
2804 struct ring_buffer_iter *buf_iter;
2807 /* If we are looking at one CPU buffer, only check that one */
2808 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2809 cpu = iter->cpu_file;
2810 buf_iter = trace_buffer_iter(iter, cpu);
2812 if (!ring_buffer_iter_empty(buf_iter))
2815 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2821 for_each_tracing_cpu(cpu) {
2822 buf_iter = trace_buffer_iter(iter, cpu);
2824 if (!ring_buffer_iter_empty(buf_iter))
2827 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2835 /* Called with trace_event_read_lock() held. */
2836 enum print_line_t print_trace_line(struct trace_iterator *iter)
2838 enum print_line_t ret;
2840 if (iter->lost_events) {
2841 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2842 iter->cpu, iter->lost_events);
2843 if (trace_seq_has_overflowed(&iter->seq))
2844 return TRACE_TYPE_PARTIAL_LINE;
2847 if (iter->trace && iter->trace->print_line) {
2848 ret = iter->trace->print_line(iter);
2849 if (ret != TRACE_TYPE_UNHANDLED)
2853 if (iter->ent->type == TRACE_BPUTS &&
2854 trace_flags & TRACE_ITER_PRINTK &&
2855 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2856 return trace_print_bputs_msg_only(iter);
2858 if (iter->ent->type == TRACE_BPRINT &&
2859 trace_flags & TRACE_ITER_PRINTK &&
2860 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2861 return trace_print_bprintk_msg_only(iter);
2863 if (iter->ent->type == TRACE_PRINT &&
2864 trace_flags & TRACE_ITER_PRINTK &&
2865 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2866 return trace_print_printk_msg_only(iter);
2868 if (trace_flags & TRACE_ITER_BIN)
2869 return print_bin_fmt(iter);
2871 if (trace_flags & TRACE_ITER_HEX)
2872 return print_hex_fmt(iter);
2874 if (trace_flags & TRACE_ITER_RAW)
2875 return print_raw_fmt(iter);
2877 return print_trace_fmt(iter);
2880 void trace_latency_header(struct seq_file *m)
2882 struct trace_iterator *iter = m->private;
2884 /* print nothing if the buffers are empty */
2885 if (trace_empty(iter))
2888 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2889 print_trace_header(m, iter);
2891 if (!(trace_flags & TRACE_ITER_VERBOSE))
2892 print_lat_help_header(m);
2895 void trace_default_header(struct seq_file *m)
2897 struct trace_iterator *iter = m->private;
2899 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2902 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2903 /* print nothing if the buffers are empty */
2904 if (trace_empty(iter))
2906 print_trace_header(m, iter);
2907 if (!(trace_flags & TRACE_ITER_VERBOSE))
2908 print_lat_help_header(m);
2910 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2911 if (trace_flags & TRACE_ITER_IRQ_INFO)
2912 print_func_help_header_irq(iter->trace_buffer, m);
2914 print_func_help_header(iter->trace_buffer, m);
2919 static void test_ftrace_alive(struct seq_file *m)
2921 if (!ftrace_is_dead())
2923 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
2924 "# MAY BE MISSING FUNCTION EVENTS\n");
2927 #ifdef CONFIG_TRACER_MAX_TRACE
2928 static void show_snapshot_main_help(struct seq_file *m)
2930 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
2931 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2932 "# Takes a snapshot of the main buffer.\n"
2933 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
2934 "# (Doesn't have to be '2' works with any number that\n"
2935 "# is not a '0' or '1')\n");
2938 static void show_snapshot_percpu_help(struct seq_file *m)
2940 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2941 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2942 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2943 "# Takes a snapshot of the main buffer for this cpu.\n");
2945 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
2946 "# Must use main snapshot file to allocate.\n");
2948 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
2949 "# (Doesn't have to be '2' works with any number that\n"
2950 "# is not a '0' or '1')\n");
2953 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2955 if (iter->tr->allocated_snapshot)
2956 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
2958 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
2960 seq_puts(m, "# Snapshot commands:\n");
2961 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2962 show_snapshot_main_help(m);
2964 show_snapshot_percpu_help(m);
2967 /* Should never be called */
2968 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2971 static int s_show(struct seq_file *m, void *v)
2973 struct trace_iterator *iter = v;
2976 if (iter->ent == NULL) {
2978 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2980 test_ftrace_alive(m);
2982 if (iter->snapshot && trace_empty(iter))
2983 print_snapshot_help(m, iter);
2984 else if (iter->trace && iter->trace->print_header)
2985 iter->trace->print_header(m);
2987 trace_default_header(m);
2989 } else if (iter->leftover) {
2991 * If we filled the seq_file buffer earlier, we
2992 * want to just show it now.
2994 ret = trace_print_seq(m, &iter->seq);
2996 /* ret should this time be zero, but you never know */
2997 iter->leftover = ret;
3000 print_trace_line(iter);
3001 ret = trace_print_seq(m, &iter->seq);
3003 * If we overflow the seq_file buffer, then it will
3004 * ask us for this data again at start up.
3006 * ret is 0 if seq_file write succeeded.
3009 iter->leftover = ret;
3016 * Should be used after trace_array_get(), trace_types_lock
3017 * ensures that i_cdev was already initialized.
3019 static inline int tracing_get_cpu(struct inode *inode)
3021 if (inode->i_cdev) /* See trace_create_cpu_file() */
3022 return (long)inode->i_cdev - 1;
3023 return RING_BUFFER_ALL_CPUS;
3026 static const struct seq_operations tracer_seq_ops = {
3033 static struct trace_iterator *
3034 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
3036 struct trace_array *tr = inode->i_private;
3037 struct trace_iterator *iter;
3040 if (tracing_disabled)
3041 return ERR_PTR(-ENODEV);
3043 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
3045 return ERR_PTR(-ENOMEM);
3047 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
3049 if (!iter->buffer_iter)
3053 * We make a copy of the current tracer to avoid concurrent
3054 * changes on it while we are reading.
3056 mutex_lock(&trace_types_lock);
3057 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
3061 *iter->trace = *tr->current_trace;
3063 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3068 #ifdef CONFIG_TRACER_MAX_TRACE
3069 /* Currently only the top directory has a snapshot */
3070 if (tr->current_trace->print_max || snapshot)
3071 iter->trace_buffer = &tr->max_buffer;
3074 iter->trace_buffer = &tr->trace_buffer;
3075 iter->snapshot = snapshot;
3077 iter->cpu_file = tracing_get_cpu(inode);
3078 mutex_init(&iter->mutex);
3080 /* Notify the tracer early; before we stop tracing. */
3081 if (iter->trace && iter->trace->open)
3082 iter->trace->open(iter);
3084 /* Annotate start of buffers if we had overruns */
3085 if (ring_buffer_overruns(iter->trace_buffer->buffer))
3086 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3088 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3089 if (trace_clocks[tr->clock_id].in_ns)
3090 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3092 /* stop the trace while dumping if we are not opening "snapshot" */
3093 if (!iter->snapshot)
3094 tracing_stop_tr(tr);
3096 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3097 for_each_tracing_cpu(cpu) {
3098 iter->buffer_iter[cpu] =
3099 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3101 ring_buffer_read_prepare_sync();
3102 for_each_tracing_cpu(cpu) {
3103 ring_buffer_read_start(iter->buffer_iter[cpu]);
3104 tracing_iter_reset(iter, cpu);
3107 cpu = iter->cpu_file;
3108 iter->buffer_iter[cpu] =
3109 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3110 ring_buffer_read_prepare_sync();
3111 ring_buffer_read_start(iter->buffer_iter[cpu]);
3112 tracing_iter_reset(iter, cpu);
3115 mutex_unlock(&trace_types_lock);
3120 mutex_unlock(&trace_types_lock);
3122 kfree(iter->buffer_iter);
3124 seq_release_private(inode, file);
3125 return ERR_PTR(-ENOMEM);
3128 int tracing_open_generic(struct inode *inode, struct file *filp)
3130 if (tracing_disabled)
3133 filp->private_data = inode->i_private;
3137 bool tracing_is_disabled(void)
3139 return (tracing_disabled) ? true: false;
3143 * Open and update trace_array ref count.
3144 * Must have the current trace_array passed to it.
3146 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3148 struct trace_array *tr = inode->i_private;
3150 if (tracing_disabled)
3153 if (trace_array_get(tr) < 0)
3156 filp->private_data = inode->i_private;
3161 static int tracing_release(struct inode *inode, struct file *file)
3163 struct trace_array *tr = inode->i_private;
3164 struct seq_file *m = file->private_data;
3165 struct trace_iterator *iter;
3168 if (!(file->f_mode & FMODE_READ)) {
3169 trace_array_put(tr);
3173 /* Writes do not use seq_file */
3175 mutex_lock(&trace_types_lock);
3177 for_each_tracing_cpu(cpu) {
3178 if (iter->buffer_iter[cpu])
3179 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3182 if (iter->trace && iter->trace->close)
3183 iter->trace->close(iter);
3185 if (!iter->snapshot)
3186 /* reenable tracing if it was previously enabled */
3187 tracing_start_tr(tr);
3189 __trace_array_put(tr);
3191 mutex_unlock(&trace_types_lock);
3193 mutex_destroy(&iter->mutex);
3194 free_cpumask_var(iter->started);
3196 kfree(iter->buffer_iter);
3197 seq_release_private(inode, file);
3202 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3204 struct trace_array *tr = inode->i_private;
3206 trace_array_put(tr);
3210 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3212 struct trace_array *tr = inode->i_private;
3214 trace_array_put(tr);
3216 return single_release(inode, file);
3219 static int tracing_open(struct inode *inode, struct file *file)
3221 struct trace_array *tr = inode->i_private;
3222 struct trace_iterator *iter;
3225 if (trace_array_get(tr) < 0)
3228 /* If this file was open for write, then erase contents */
3229 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3230 int cpu = tracing_get_cpu(inode);
3232 if (cpu == RING_BUFFER_ALL_CPUS)
3233 tracing_reset_online_cpus(&tr->trace_buffer);
3235 tracing_reset(&tr->trace_buffer, cpu);
3238 if (file->f_mode & FMODE_READ) {
3239 iter = __tracing_open(inode, file, false);
3241 ret = PTR_ERR(iter);
3242 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
3243 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3247 trace_array_put(tr);
3253 * Some tracers are not suitable for instance buffers.
3254 * A tracer is always available for the global array (toplevel)
3255 * or if it explicitly states that it is.
3258 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3260 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3263 /* Find the next tracer that this trace array may use */
3264 static struct tracer *
3265 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3267 while (t && !trace_ok_for_array(t, tr))
3274 t_next(struct seq_file *m, void *v, loff_t *pos)
3276 struct trace_array *tr = m->private;
3277 struct tracer *t = v;
3282 t = get_tracer_for_array(tr, t->next);
3287 static void *t_start(struct seq_file *m, loff_t *pos)
3289 struct trace_array *tr = m->private;
3293 mutex_lock(&trace_types_lock);
3295 t = get_tracer_for_array(tr, trace_types);
3296 for (; t && l < *pos; t = t_next(m, t, &l))
3302 static void t_stop(struct seq_file *m, void *p)
3304 mutex_unlock(&trace_types_lock);
3307 static int t_show(struct seq_file *m, void *v)
3309 struct tracer *t = v;
3314 seq_puts(m, t->name);
3323 static const struct seq_operations show_traces_seq_ops = {
3330 static int show_traces_open(struct inode *inode, struct file *file)
3332 struct trace_array *tr = inode->i_private;
3336 if (tracing_disabled)
3339 ret = seq_open(file, &show_traces_seq_ops);
3343 m = file->private_data;
3350 tracing_write_stub(struct file *filp, const char __user *ubuf,
3351 size_t count, loff_t *ppos)
3356 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3360 if (file->f_mode & FMODE_READ)
3361 ret = seq_lseek(file, offset, whence);
3363 file->f_pos = ret = 0;
3368 static const struct file_operations tracing_fops = {
3369 .open = tracing_open,
3371 .write = tracing_write_stub,
3372 .llseek = tracing_lseek,
3373 .release = tracing_release,
3376 static const struct file_operations show_traces_fops = {
3377 .open = show_traces_open,
3379 .release = seq_release,
3380 .llseek = seq_lseek,
3384 * The tracer itself will not take this lock, but still we want
3385 * to provide a consistent cpumask to user-space:
3387 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3390 * Temporary storage for the character representation of the
3391 * CPU bitmask (and one more byte for the newline):
3393 static char mask_str[NR_CPUS + 1];
3396 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3397 size_t count, loff_t *ppos)
3399 struct trace_array *tr = file_inode(filp)->i_private;
3402 mutex_lock(&tracing_cpumask_update_lock);
3404 len = snprintf(mask_str, count, "%*pb\n",
3405 cpumask_pr_args(tr->tracing_cpumask));
3410 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3413 mutex_unlock(&tracing_cpumask_update_lock);
3419 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3420 size_t count, loff_t *ppos)
3422 struct trace_array *tr = file_inode(filp)->i_private;
3423 cpumask_var_t tracing_cpumask_new;
3426 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3429 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3433 mutex_lock(&tracing_cpumask_update_lock);
3435 local_irq_disable();
3436 arch_spin_lock(&tr->max_lock);
3437 for_each_tracing_cpu(cpu) {
3439 * Increase/decrease the disabled counter if we are
3440 * about to flip a bit in the cpumask:
3442 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3443 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3444 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3445 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3447 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3448 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3449 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3450 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3453 arch_spin_unlock(&tr->max_lock);
3456 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3458 mutex_unlock(&tracing_cpumask_update_lock);
3459 free_cpumask_var(tracing_cpumask_new);
3464 free_cpumask_var(tracing_cpumask_new);
3469 static const struct file_operations tracing_cpumask_fops = {
3470 .open = tracing_open_generic_tr,
3471 .read = tracing_cpumask_read,
3472 .write = tracing_cpumask_write,
3473 .release = tracing_release_generic_tr,
3474 .llseek = generic_file_llseek,
3477 static int tracing_trace_options_show(struct seq_file *m, void *v)
3479 struct tracer_opt *trace_opts;
3480 struct trace_array *tr = m->private;
3484 mutex_lock(&trace_types_lock);
3485 tracer_flags = tr->current_trace->flags->val;
3486 trace_opts = tr->current_trace->flags->opts;
3488 for (i = 0; trace_options[i]; i++) {
3489 if (trace_flags & (1 << i))
3490 seq_printf(m, "%s\n", trace_options[i]);
3492 seq_printf(m, "no%s\n", trace_options[i]);
3495 for (i = 0; trace_opts[i].name; i++) {
3496 if (tracer_flags & trace_opts[i].bit)
3497 seq_printf(m, "%s\n", trace_opts[i].name);
3499 seq_printf(m, "no%s\n", trace_opts[i].name);
3501 mutex_unlock(&trace_types_lock);
3506 static int __set_tracer_option(struct trace_array *tr,
3507 struct tracer_flags *tracer_flags,
3508 struct tracer_opt *opts, int neg)
3510 struct tracer *trace = tr->current_trace;
3513 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3518 tracer_flags->val &= ~opts->bit;
3520 tracer_flags->val |= opts->bit;
3524 /* Try to assign a tracer specific option */
3525 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3527 struct tracer *trace = tr->current_trace;
3528 struct tracer_flags *tracer_flags = trace->flags;
3529 struct tracer_opt *opts = NULL;
3532 for (i = 0; tracer_flags->opts[i].name; i++) {
3533 opts = &tracer_flags->opts[i];
3535 if (strcmp(cmp, opts->name) == 0)
3536 return __set_tracer_option(tr, trace->flags, opts, neg);
3542 /* Some tracers require overwrite to stay enabled */
3543 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3545 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3551 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3553 /* do nothing if flag is already set */
3554 if (!!(trace_flags & mask) == !!enabled)
3557 /* Give the tracer a chance to approve the change */
3558 if (tr->current_trace->flag_changed)
3559 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3563 trace_flags |= mask;
3565 trace_flags &= ~mask;
3567 if (mask == TRACE_ITER_RECORD_CMD)
3568 trace_event_enable_cmd_record(enabled);
3570 if (mask == TRACE_ITER_OVERWRITE) {
3571 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3572 #ifdef CONFIG_TRACER_MAX_TRACE
3573 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3577 if (mask == TRACE_ITER_PRINTK)
3578 trace_printk_start_stop_comm(enabled);
3583 static int trace_set_options(struct trace_array *tr, char *option)
3590 cmp = strstrip(option);
3592 if (strncmp(cmp, "no", 2) == 0) {
3597 mutex_lock(&trace_types_lock);
3599 for (i = 0; trace_options[i]; i++) {
3600 if (strcmp(cmp, trace_options[i]) == 0) {
3601 ret = set_tracer_flag(tr, 1 << i, !neg);
3606 /* If no option could be set, test the specific tracer options */
3607 if (!trace_options[i])
3608 ret = set_tracer_option(tr, cmp, neg);
3610 mutex_unlock(&trace_types_lock);
3616 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3617 size_t cnt, loff_t *ppos)
3619 struct seq_file *m = filp->private_data;
3620 struct trace_array *tr = m->private;
3624 if (cnt >= sizeof(buf))
3627 if (copy_from_user(&buf, ubuf, cnt))
3632 ret = trace_set_options(tr, buf);
3641 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3643 struct trace_array *tr = inode->i_private;
3646 if (tracing_disabled)
3649 if (trace_array_get(tr) < 0)
3652 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3654 trace_array_put(tr);
3659 static const struct file_operations tracing_iter_fops = {
3660 .open = tracing_trace_options_open,
3662 .llseek = seq_lseek,
3663 .release = tracing_single_release_tr,
3664 .write = tracing_trace_options_write,
3667 static const char readme_msg[] =
3668 "tracing mini-HOWTO:\n\n"
3669 "# echo 0 > tracing_on : quick way to disable tracing\n"
3670 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3671 " Important files:\n"
3672 " trace\t\t\t- The static contents of the buffer\n"
3673 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3674 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3675 " current_tracer\t- function and latency tracers\n"
3676 " available_tracers\t- list of configured tracers for current_tracer\n"
3677 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3678 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3679 " trace_clock\t\t-change the clock used to order events\n"
3680 " local: Per cpu clock but may not be synced across CPUs\n"
3681 " global: Synced across CPUs but slows tracing down.\n"
3682 " counter: Not a clock, but just an increment\n"
3683 " uptime: Jiffy counter from time of boot\n"
3684 " perf: Same clock that perf events use\n"
3685 #ifdef CONFIG_X86_64
3686 " x86-tsc: TSC cycle counter\n"
3688 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3689 " tracing_cpumask\t- Limit which CPUs to trace\n"
3690 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3691 "\t\t\t Remove sub-buffer with rmdir\n"
3692 " trace_options\t\t- Set format or modify how tracing happens\n"
3693 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3694 "\t\t\t option name\n"
3695 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
3696 #ifdef CONFIG_DYNAMIC_FTRACE
3697 "\n available_filter_functions - list of functions that can be filtered on\n"
3698 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3699 "\t\t\t functions\n"
3700 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3701 "\t modules: Can select a group via module\n"
3702 "\t Format: :mod:<module-name>\n"
3703 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3704 "\t triggers: a command to perform when function is hit\n"
3705 "\t Format: <function>:<trigger>[:count]\n"
3706 "\t trigger: traceon, traceoff\n"
3707 "\t\t enable_event:<system>:<event>\n"
3708 "\t\t disable_event:<system>:<event>\n"
3709 #ifdef CONFIG_STACKTRACE
3712 #ifdef CONFIG_TRACER_SNAPSHOT
3717 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3718 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3719 "\t The first one will disable tracing every time do_fault is hit\n"
3720 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3721 "\t The first time do trap is hit and it disables tracing, the\n"
3722 "\t counter will decrement to 2. If tracing is already disabled,\n"
3723 "\t the counter will not decrement. It only decrements when the\n"
3724 "\t trigger did work\n"
3725 "\t To remove trigger without count:\n"
3726 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3727 "\t To remove trigger with a count:\n"
3728 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3729 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3730 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3731 "\t modules: Can select a group via module command :mod:\n"
3732 "\t Does not accept triggers\n"
3733 #endif /* CONFIG_DYNAMIC_FTRACE */
3734 #ifdef CONFIG_FUNCTION_TRACER
3735 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3738 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3739 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3740 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
3741 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3743 #ifdef CONFIG_TRACER_SNAPSHOT
3744 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3745 "\t\t\t snapshot buffer. Read the contents for more\n"
3746 "\t\t\t information\n"
3748 #ifdef CONFIG_STACK_TRACER
3749 " stack_trace\t\t- Shows the max stack trace when active\n"
3750 " stack_max_size\t- Shows current max stack size that was traced\n"
3751 "\t\t\t Write into this file to reset the max size (trigger a\n"
3752 "\t\t\t new trace)\n"
3753 #ifdef CONFIG_DYNAMIC_FTRACE
3754 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3757 #endif /* CONFIG_STACK_TRACER */
3758 " events/\t\t- Directory containing all trace event subsystems:\n"
3759 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3760 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3761 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3763 " filter\t\t- If set, only events passing filter are traced\n"
3764 " events/<system>/<event>/\t- Directory containing control files for\n"
3766 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3767 " filter\t\t- If set, only events passing filter are traced\n"
3768 " trigger\t\t- If set, a command to perform when event is hit\n"
3769 "\t Format: <trigger>[:count][if <filter>]\n"
3770 "\t trigger: traceon, traceoff\n"
3771 "\t enable_event:<system>:<event>\n"
3772 "\t disable_event:<system>:<event>\n"
3773 #ifdef CONFIG_STACKTRACE
3776 #ifdef CONFIG_TRACER_SNAPSHOT
3779 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3780 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3781 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3782 "\t events/block/block_unplug/trigger\n"
3783 "\t The first disables tracing every time block_unplug is hit.\n"
3784 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3785 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3786 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3787 "\t Like function triggers, the counter is only decremented if it\n"
3788 "\t enabled or disabled tracing.\n"
3789 "\t To remove a trigger without a count:\n"
3790 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3791 "\t To remove a trigger with a count:\n"
3792 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3793 "\t Filters can be ignored when removing a trigger.\n"
3797 tracing_readme_read(struct file *filp, char __user *ubuf,
3798 size_t cnt, loff_t *ppos)
3800 return simple_read_from_buffer(ubuf, cnt, ppos,
3801 readme_msg, strlen(readme_msg));
3804 static const struct file_operations tracing_readme_fops = {
3805 .open = tracing_open_generic,
3806 .read = tracing_readme_read,
3807 .llseek = generic_file_llseek,
3810 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
3812 unsigned int *ptr = v;
3814 if (*pos || m->count)
3819 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
3821 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
3830 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
3836 arch_spin_lock(&trace_cmdline_lock);
3838 v = &savedcmd->map_cmdline_to_pid[0];
3840 v = saved_cmdlines_next(m, v, &l);
3848 static void saved_cmdlines_stop(struct seq_file *m, void *v)
3850 arch_spin_unlock(&trace_cmdline_lock);
3854 static int saved_cmdlines_show(struct seq_file *m, void *v)
3856 char buf[TASK_COMM_LEN];
3857 unsigned int *pid = v;
3859 __trace_find_cmdline(*pid, buf);
3860 seq_printf(m, "%d %s\n", *pid, buf);
3864 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
3865 .start = saved_cmdlines_start,
3866 .next = saved_cmdlines_next,
3867 .stop = saved_cmdlines_stop,
3868 .show = saved_cmdlines_show,
3871 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
3873 if (tracing_disabled)
3876 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
3879 static const struct file_operations tracing_saved_cmdlines_fops = {
3880 .open = tracing_saved_cmdlines_open,
3882 .llseek = seq_lseek,
3883 .release = seq_release,
3887 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
3888 size_t cnt, loff_t *ppos)
3893 arch_spin_lock(&trace_cmdline_lock);
3894 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
3895 arch_spin_unlock(&trace_cmdline_lock);
3897 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3900 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
3902 kfree(s->saved_cmdlines);
3903 kfree(s->map_cmdline_to_pid);
3907 static int tracing_resize_saved_cmdlines(unsigned int val)
3909 struct saved_cmdlines_buffer *s, *savedcmd_temp;
3911 s = kmalloc(sizeof(*s), GFP_KERNEL);
3915 if (allocate_cmdlines_buffer(val, s) < 0) {
3920 arch_spin_lock(&trace_cmdline_lock);
3921 savedcmd_temp = savedcmd;
3923 arch_spin_unlock(&trace_cmdline_lock);
3924 free_saved_cmdlines_buffer(savedcmd_temp);
3930 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
3931 size_t cnt, loff_t *ppos)
3936 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3940 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
3941 if (!val || val > PID_MAX_DEFAULT)
3944 ret = tracing_resize_saved_cmdlines((unsigned int)val);
3953 static const struct file_operations tracing_saved_cmdlines_size_fops = {
3954 .open = tracing_open_generic,
3955 .read = tracing_saved_cmdlines_size_read,
3956 .write = tracing_saved_cmdlines_size_write,
3959 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
3960 static union trace_enum_map_item *
3961 update_enum_map(union trace_enum_map_item *ptr)
3963 if (!ptr->map.enum_string) {
3964 if (ptr->tail.next) {
3965 ptr = ptr->tail.next;
3966 /* Set ptr to the next real item (skip head) */
3974 static void *enum_map_next(struct seq_file *m, void *v, loff_t *pos)
3976 union trace_enum_map_item *ptr = v;
3979 * Paranoid! If ptr points to end, we don't want to increment past it.
3980 * This really should never happen.
3982 ptr = update_enum_map(ptr);
3983 if (WARN_ON_ONCE(!ptr))
3990 ptr = update_enum_map(ptr);
3995 static void *enum_map_start(struct seq_file *m, loff_t *pos)
3997 union trace_enum_map_item *v;
4000 mutex_lock(&trace_enum_mutex);
4002 v = trace_enum_maps;
4006 while (v && l < *pos) {
4007 v = enum_map_next(m, v, &l);
4013 static void enum_map_stop(struct seq_file *m, void *v)
4015 mutex_unlock(&trace_enum_mutex);
4018 static int enum_map_show(struct seq_file *m, void *v)
4020 union trace_enum_map_item *ptr = v;
4022 seq_printf(m, "%s %ld (%s)\n",
4023 ptr->map.enum_string, ptr->map.enum_value,
4029 static const struct seq_operations tracing_enum_map_seq_ops = {
4030 .start = enum_map_start,
4031 .next = enum_map_next,
4032 .stop = enum_map_stop,
4033 .show = enum_map_show,
4036 static int tracing_enum_map_open(struct inode *inode, struct file *filp)
4038 if (tracing_disabled)
4041 return seq_open(filp, &tracing_enum_map_seq_ops);
4044 static const struct file_operations tracing_enum_map_fops = {
4045 .open = tracing_enum_map_open,
4047 .llseek = seq_lseek,
4048 .release = seq_release,
4051 static inline union trace_enum_map_item *
4052 trace_enum_jmp_to_tail(union trace_enum_map_item *ptr)
4054 /* Return tail of array given the head */
4055 return ptr + ptr->head.length + 1;
4059 trace_insert_enum_map_file(struct module *mod, struct trace_enum_map **start,
4062 struct trace_enum_map **stop;
4063 struct trace_enum_map **map;
4064 union trace_enum_map_item *map_array;
4065 union trace_enum_map_item *ptr;
4070 * The trace_enum_maps contains the map plus a head and tail item,
4071 * where the head holds the module and length of array, and the
4072 * tail holds a pointer to the next list.
4074 map_array = kmalloc(sizeof(*map_array) * (len + 2), GFP_KERNEL);
4076 pr_warning("Unable to allocate trace enum mapping\n");
4080 mutex_lock(&trace_enum_mutex);
4082 if (!trace_enum_maps)
4083 trace_enum_maps = map_array;
4085 ptr = trace_enum_maps;
4087 ptr = trace_enum_jmp_to_tail(ptr);
4088 if (!ptr->tail.next)
4090 ptr = ptr->tail.next;
4093 ptr->tail.next = map_array;
4095 map_array->head.mod = mod;
4096 map_array->head.length = len;
4099 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
4100 map_array->map = **map;
4103 memset(map_array, 0, sizeof(*map_array));
4105 mutex_unlock(&trace_enum_mutex);
4108 static void trace_create_enum_file(struct dentry *d_tracer)
4110 trace_create_file("enum_map", 0444, d_tracer,
4111 NULL, &tracing_enum_map_fops);
4114 #else /* CONFIG_TRACE_ENUM_MAP_FILE */
4115 static inline void trace_create_enum_file(struct dentry *d_tracer) { }
4116 static inline void trace_insert_enum_map_file(struct module *mod,
4117 struct trace_enum_map **start, int len) { }
4118 #endif /* !CONFIG_TRACE_ENUM_MAP_FILE */
4120 static void trace_insert_enum_map(struct module *mod,
4121 struct trace_enum_map **start, int len)
4123 struct trace_enum_map **map;
4130 trace_event_enum_update(map, len);
4132 trace_insert_enum_map_file(mod, start, len);
4136 tracing_set_trace_read(struct file *filp, char __user *ubuf,
4137 size_t cnt, loff_t *ppos)
4139 struct trace_array *tr = filp->private_data;
4140 char buf[MAX_TRACER_SIZE+2];
4143 mutex_lock(&trace_types_lock);
4144 r = sprintf(buf, "%s\n", tr->current_trace->name);
4145 mutex_unlock(&trace_types_lock);
4147 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4150 int tracer_init(struct tracer *t, struct trace_array *tr)
4152 tracing_reset_online_cpus(&tr->trace_buffer);
4156 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
4160 for_each_tracing_cpu(cpu)
4161 per_cpu_ptr(buf->data, cpu)->entries = val;
4164 #ifdef CONFIG_TRACER_MAX_TRACE
4165 /* resize @tr's buffer to the size of @size_tr's entries */
4166 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
4167 struct trace_buffer *size_buf, int cpu_id)
4171 if (cpu_id == RING_BUFFER_ALL_CPUS) {
4172 for_each_tracing_cpu(cpu) {
4173 ret = ring_buffer_resize(trace_buf->buffer,
4174 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
4177 per_cpu_ptr(trace_buf->data, cpu)->entries =
4178 per_cpu_ptr(size_buf->data, cpu)->entries;
4181 ret = ring_buffer_resize(trace_buf->buffer,
4182 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
4184 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
4185 per_cpu_ptr(size_buf->data, cpu_id)->entries;
4190 #endif /* CONFIG_TRACER_MAX_TRACE */
4192 static int __tracing_resize_ring_buffer(struct trace_array *tr,
4193 unsigned long size, int cpu)
4198 * If kernel or user changes the size of the ring buffer
4199 * we use the size that was given, and we can forget about
4200 * expanding it later.
4202 ring_buffer_expanded = true;
4204 /* May be called before buffers are initialized */
4205 if (!tr->trace_buffer.buffer)
4208 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
4212 #ifdef CONFIG_TRACER_MAX_TRACE
4213 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
4214 !tr->current_trace->use_max_tr)
4217 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
4219 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
4220 &tr->trace_buffer, cpu);
4223 * AARGH! We are left with different
4224 * size max buffer!!!!
4225 * The max buffer is our "snapshot" buffer.
4226 * When a tracer needs a snapshot (one of the
4227 * latency tracers), it swaps the max buffer
4228 * with the saved snap shot. We succeeded to
4229 * update the size of the main buffer, but failed to
4230 * update the size of the max buffer. But when we tried
4231 * to reset the main buffer to the original size, we
4232 * failed there too. This is very unlikely to
4233 * happen, but if it does, warn and kill all
4237 tracing_disabled = 1;
4242 if (cpu == RING_BUFFER_ALL_CPUS)
4243 set_buffer_entries(&tr->max_buffer, size);
4245 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
4248 #endif /* CONFIG_TRACER_MAX_TRACE */
4250 if (cpu == RING_BUFFER_ALL_CPUS)
4251 set_buffer_entries(&tr->trace_buffer, size);
4253 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
4258 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4259 unsigned long size, int cpu_id)
4263 mutex_lock(&trace_types_lock);
4265 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4266 /* make sure, this cpu is enabled in the mask */
4267 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4273 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4278 mutex_unlock(&trace_types_lock);
4285 * tracing_update_buffers - used by tracing facility to expand ring buffers
4287 * To save on memory when the tracing is never used on a system with it
4288 * configured in. The ring buffers are set to a minimum size. But once
4289 * a user starts to use the tracing facility, then they need to grow
4290 * to their default size.
4292 * This function is to be called when a tracer is about to be used.
4294 int tracing_update_buffers(void)
4298 mutex_lock(&trace_types_lock);
4299 if (!ring_buffer_expanded)
4300 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
4301 RING_BUFFER_ALL_CPUS);
4302 mutex_unlock(&trace_types_lock);
4307 struct trace_option_dentry;
4309 static struct trace_option_dentry *
4310 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
4313 destroy_trace_option_files(struct trace_option_dentry *topts);
4316 * Used to clear out the tracer before deletion of an instance.
4317 * Must have trace_types_lock held.
4319 static void tracing_set_nop(struct trace_array *tr)
4321 if (tr->current_trace == &nop_trace)
4324 tr->current_trace->enabled--;
4326 if (tr->current_trace->reset)
4327 tr->current_trace->reset(tr);
4329 tr->current_trace = &nop_trace;
4332 static void update_tracer_options(struct trace_array *tr, struct tracer *t)
4334 static struct trace_option_dentry *topts;
4336 /* Only enable if the directory has been created already. */
4340 /* Currently, only the top instance has options */
4341 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL))
4344 destroy_trace_option_files(topts);
4345 topts = create_trace_option_files(tr, t);
4348 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4351 #ifdef CONFIG_TRACER_MAX_TRACE
4356 mutex_lock(&trace_types_lock);
4358 if (!ring_buffer_expanded) {
4359 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4360 RING_BUFFER_ALL_CPUS);
4366 for (t = trace_types; t; t = t->next) {
4367 if (strcmp(t->name, buf) == 0)
4374 if (t == tr->current_trace)
4377 /* Some tracers are only allowed for the top level buffer */
4378 if (!trace_ok_for_array(t, tr)) {
4383 /* If trace pipe files are being read, we can't change the tracer */
4384 if (tr->current_trace->ref) {
4389 trace_branch_disable();
4391 tr->current_trace->enabled--;
4393 if (tr->current_trace->reset)
4394 tr->current_trace->reset(tr);
4396 /* Current trace needs to be nop_trace before synchronize_sched */
4397 tr->current_trace = &nop_trace;
4399 #ifdef CONFIG_TRACER_MAX_TRACE
4400 had_max_tr = tr->allocated_snapshot;
4402 if (had_max_tr && !t->use_max_tr) {
4404 * We need to make sure that the update_max_tr sees that
4405 * current_trace changed to nop_trace to keep it from
4406 * swapping the buffers after we resize it.
4407 * The update_max_tr is called from interrupts disabled
4408 * so a synchronized_sched() is sufficient.
4410 synchronize_sched();
4414 update_tracer_options(tr, t);
4416 #ifdef CONFIG_TRACER_MAX_TRACE
4417 if (t->use_max_tr && !had_max_tr) {
4418 ret = alloc_snapshot(tr);
4425 ret = tracer_init(t, tr);
4430 tr->current_trace = t;
4431 tr->current_trace->enabled++;
4432 trace_branch_enable(tr);
4434 mutex_unlock(&trace_types_lock);
4440 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4441 size_t cnt, loff_t *ppos)
4443 struct trace_array *tr = filp->private_data;
4444 char buf[MAX_TRACER_SIZE+1];
4451 if (cnt > MAX_TRACER_SIZE)
4452 cnt = MAX_TRACER_SIZE;
4454 if (copy_from_user(&buf, ubuf, cnt))
4459 /* strip ending whitespace. */
4460 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4463 err = tracing_set_tracer(tr, buf);
4473 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
4474 size_t cnt, loff_t *ppos)
4479 r = snprintf(buf, sizeof(buf), "%ld\n",
4480 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4481 if (r > sizeof(buf))
4483 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4487 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
4488 size_t cnt, loff_t *ppos)
4493 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4503 tracing_thresh_read(struct file *filp, char __user *ubuf,
4504 size_t cnt, loff_t *ppos)
4506 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
4510 tracing_thresh_write(struct file *filp, const char __user *ubuf,
4511 size_t cnt, loff_t *ppos)
4513 struct trace_array *tr = filp->private_data;
4516 mutex_lock(&trace_types_lock);
4517 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
4521 if (tr->current_trace->update_thresh) {
4522 ret = tr->current_trace->update_thresh(tr);
4529 mutex_unlock(&trace_types_lock);
4535 tracing_max_lat_read(struct file *filp, char __user *ubuf,
4536 size_t cnt, loff_t *ppos)
4538 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
4542 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4543 size_t cnt, loff_t *ppos)
4545 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
4548 static int tracing_open_pipe(struct inode *inode, struct file *filp)
4550 struct trace_array *tr = inode->i_private;
4551 struct trace_iterator *iter;
4554 if (tracing_disabled)
4557 if (trace_array_get(tr) < 0)
4560 mutex_lock(&trace_types_lock);
4562 /* create a buffer to store the information to pass to userspace */
4563 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4566 __trace_array_put(tr);
4570 trace_seq_init(&iter->seq);
4571 iter->trace = tr->current_trace;
4573 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4578 /* trace pipe does not show start of buffer */
4579 cpumask_setall(iter->started);
4581 if (trace_flags & TRACE_ITER_LATENCY_FMT)
4582 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4584 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4585 if (trace_clocks[tr->clock_id].in_ns)
4586 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4589 iter->trace_buffer = &tr->trace_buffer;
4590 iter->cpu_file = tracing_get_cpu(inode);
4591 mutex_init(&iter->mutex);
4592 filp->private_data = iter;
4594 if (iter->trace->pipe_open)
4595 iter->trace->pipe_open(iter);
4597 nonseekable_open(inode, filp);
4599 tr->current_trace->ref++;
4601 mutex_unlock(&trace_types_lock);
4607 __trace_array_put(tr);
4608 mutex_unlock(&trace_types_lock);
4612 static int tracing_release_pipe(struct inode *inode, struct file *file)
4614 struct trace_iterator *iter = file->private_data;
4615 struct trace_array *tr = inode->i_private;
4617 mutex_lock(&trace_types_lock);
4619 tr->current_trace->ref--;
4621 if (iter->trace->pipe_close)
4622 iter->trace->pipe_close(iter);
4624 mutex_unlock(&trace_types_lock);
4626 free_cpumask_var(iter->started);
4627 mutex_destroy(&iter->mutex);
4630 trace_array_put(tr);
4636 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4638 /* Iterators are static, they should be filled or empty */
4639 if (trace_buffer_iter(iter, iter->cpu_file))
4640 return POLLIN | POLLRDNORM;
4642 if (trace_flags & TRACE_ITER_BLOCK)
4644 * Always select as readable when in blocking mode
4646 return POLLIN | POLLRDNORM;
4648 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4653 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4655 struct trace_iterator *iter = filp->private_data;
4657 return trace_poll(iter, filp, poll_table);
4660 /* Must be called with iter->mutex held. */
4661 static int tracing_wait_pipe(struct file *filp)
4663 struct trace_iterator *iter = filp->private_data;
4666 while (trace_empty(iter)) {
4668 if ((filp->f_flags & O_NONBLOCK)) {
4673 * We block until we read something and tracing is disabled.
4674 * We still block if tracing is disabled, but we have never
4675 * read anything. This allows a user to cat this file, and
4676 * then enable tracing. But after we have read something,
4677 * we give an EOF when tracing is again disabled.
4679 * iter->pos will be 0 if we haven't read anything.
4681 if (!tracing_is_on() && iter->pos)
4684 mutex_unlock(&iter->mutex);
4686 ret = wait_on_pipe(iter, false);
4688 mutex_lock(&iter->mutex);
4701 tracing_read_pipe(struct file *filp, char __user *ubuf,
4702 size_t cnt, loff_t *ppos)
4704 struct trace_iterator *iter = filp->private_data;
4707 /* return any leftover data */
4708 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4712 trace_seq_init(&iter->seq);
4715 * Avoid more than one consumer on a single file descriptor
4716 * This is just a matter of traces coherency, the ring buffer itself
4719 mutex_lock(&iter->mutex);
4720 if (iter->trace->read) {
4721 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4727 sret = tracing_wait_pipe(filp);
4731 /* stop when tracing is finished */
4732 if (trace_empty(iter)) {
4737 if (cnt >= PAGE_SIZE)
4738 cnt = PAGE_SIZE - 1;
4740 /* reset all but tr, trace, and overruns */
4741 memset(&iter->seq, 0,
4742 sizeof(struct trace_iterator) -
4743 offsetof(struct trace_iterator, seq));
4744 cpumask_clear(iter->started);
4747 trace_event_read_lock();
4748 trace_access_lock(iter->cpu_file);
4749 while (trace_find_next_entry_inc(iter) != NULL) {
4750 enum print_line_t ret;
4751 int save_len = iter->seq.seq.len;
4753 ret = print_trace_line(iter);
4754 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4755 /* don't print partial lines */
4756 iter->seq.seq.len = save_len;
4759 if (ret != TRACE_TYPE_NO_CONSUME)
4760 trace_consume(iter);
4762 if (trace_seq_used(&iter->seq) >= cnt)
4766 * Setting the full flag means we reached the trace_seq buffer
4767 * size and we should leave by partial output condition above.
4768 * One of the trace_seq_* functions is not used properly.
4770 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4773 trace_access_unlock(iter->cpu_file);
4774 trace_event_read_unlock();
4776 /* Now copy what we have to the user */
4777 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4778 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
4779 trace_seq_init(&iter->seq);
4782 * If there was nothing to send to user, in spite of consuming trace
4783 * entries, go back to wait for more entries.
4789 mutex_unlock(&iter->mutex);
4794 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4797 __free_page(spd->pages[idx]);
4800 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4802 .confirm = generic_pipe_buf_confirm,
4803 .release = generic_pipe_buf_release,
4804 .steal = generic_pipe_buf_steal,
4805 .get = generic_pipe_buf_get,
4809 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4815 /* Seq buffer is page-sized, exactly what we need. */
4817 save_len = iter->seq.seq.len;
4818 ret = print_trace_line(iter);
4820 if (trace_seq_has_overflowed(&iter->seq)) {
4821 iter->seq.seq.len = save_len;
4826 * This should not be hit, because it should only
4827 * be set if the iter->seq overflowed. But check it
4828 * anyway to be safe.
4830 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4831 iter->seq.seq.len = save_len;
4835 count = trace_seq_used(&iter->seq) - save_len;
4838 iter->seq.seq.len = save_len;
4842 if (ret != TRACE_TYPE_NO_CONSUME)
4843 trace_consume(iter);
4845 if (!trace_find_next_entry_inc(iter)) {
4855 static ssize_t tracing_splice_read_pipe(struct file *filp,
4857 struct pipe_inode_info *pipe,
4861 struct page *pages_def[PIPE_DEF_BUFFERS];
4862 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4863 struct trace_iterator *iter = filp->private_data;
4864 struct splice_pipe_desc spd = {
4866 .partial = partial_def,
4867 .nr_pages = 0, /* This gets updated below. */
4868 .nr_pages_max = PIPE_DEF_BUFFERS,
4870 .ops = &tracing_pipe_buf_ops,
4871 .spd_release = tracing_spd_release_pipe,
4877 if (splice_grow_spd(pipe, &spd))
4880 mutex_lock(&iter->mutex);
4882 if (iter->trace->splice_read) {
4883 ret = iter->trace->splice_read(iter, filp,
4884 ppos, pipe, len, flags);
4889 ret = tracing_wait_pipe(filp);
4893 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4898 trace_event_read_lock();
4899 trace_access_lock(iter->cpu_file);
4901 /* Fill as many pages as possible. */
4902 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
4903 spd.pages[i] = alloc_page(GFP_KERNEL);
4907 rem = tracing_fill_pipe_page(rem, iter);
4909 /* Copy the data into the page, so we can start over. */
4910 ret = trace_seq_to_buffer(&iter->seq,
4911 page_address(spd.pages[i]),
4912 trace_seq_used(&iter->seq));
4914 __free_page(spd.pages[i]);
4917 spd.partial[i].offset = 0;
4918 spd.partial[i].len = trace_seq_used(&iter->seq);
4920 trace_seq_init(&iter->seq);
4923 trace_access_unlock(iter->cpu_file);
4924 trace_event_read_unlock();
4925 mutex_unlock(&iter->mutex);
4929 ret = splice_to_pipe(pipe, &spd);
4931 splice_shrink_spd(&spd);
4935 mutex_unlock(&iter->mutex);
4940 tracing_entries_read(struct file *filp, char __user *ubuf,
4941 size_t cnt, loff_t *ppos)
4943 struct inode *inode = file_inode(filp);
4944 struct trace_array *tr = inode->i_private;
4945 int cpu = tracing_get_cpu(inode);
4950 mutex_lock(&trace_types_lock);
4952 if (cpu == RING_BUFFER_ALL_CPUS) {
4953 int cpu, buf_size_same;
4958 /* check if all cpu sizes are same */
4959 for_each_tracing_cpu(cpu) {
4960 /* fill in the size from first enabled cpu */
4962 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4963 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4969 if (buf_size_same) {
4970 if (!ring_buffer_expanded)
4971 r = sprintf(buf, "%lu (expanded: %lu)\n",
4973 trace_buf_size >> 10);
4975 r = sprintf(buf, "%lu\n", size >> 10);
4977 r = sprintf(buf, "X\n");
4979 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
4981 mutex_unlock(&trace_types_lock);
4983 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4988 tracing_entries_write(struct file *filp, const char __user *ubuf,
4989 size_t cnt, loff_t *ppos)
4991 struct inode *inode = file_inode(filp);
4992 struct trace_array *tr = inode->i_private;
4996 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5000 /* must have at least 1 entry */
5004 /* value is in KB */
5006 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
5016 tracing_total_entries_read(struct file *filp, char __user *ubuf,
5017 size_t cnt, loff_t *ppos)
5019 struct trace_array *tr = filp->private_data;
5022 unsigned long size = 0, expanded_size = 0;
5024 mutex_lock(&trace_types_lock);
5025 for_each_tracing_cpu(cpu) {
5026 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
5027 if (!ring_buffer_expanded)
5028 expanded_size += trace_buf_size >> 10;
5030 if (ring_buffer_expanded)
5031 r = sprintf(buf, "%lu\n", size);
5033 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
5034 mutex_unlock(&trace_types_lock);
5036 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5040 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
5041 size_t cnt, loff_t *ppos)
5044 * There is no need to read what the user has written, this function
5045 * is just to make sure that there is no error when "echo" is used
5054 tracing_free_buffer_release(struct inode *inode, struct file *filp)
5056 struct trace_array *tr = inode->i_private;
5058 /* disable tracing ? */
5059 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
5060 tracer_tracing_off(tr);
5061 /* resize the ring buffer to 0 */
5062 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
5064 trace_array_put(tr);
5070 tracing_mark_write(struct file *filp, const char __user *ubuf,
5071 size_t cnt, loff_t *fpos)
5073 unsigned long addr = (unsigned long)ubuf;
5074 struct trace_array *tr = filp->private_data;
5075 struct ring_buffer_event *event;
5076 struct ring_buffer *buffer;
5077 struct print_entry *entry;
5078 unsigned long irq_flags;
5079 struct page *pages[2];
5089 if (tracing_disabled)
5092 if (!(trace_flags & TRACE_ITER_MARKERS))
5095 if (cnt > TRACE_BUF_SIZE)
5096 cnt = TRACE_BUF_SIZE;
5099 * Userspace is injecting traces into the kernel trace buffer.
5100 * We want to be as non intrusive as possible.
5101 * To do so, we do not want to allocate any special buffers
5102 * or take any locks, but instead write the userspace data
5103 * straight into the ring buffer.
5105 * First we need to pin the userspace buffer into memory,
5106 * which, most likely it is, because it just referenced it.
5107 * But there's no guarantee that it is. By using get_user_pages_fast()
5108 * and kmap_atomic/kunmap_atomic() we can get access to the
5109 * pages directly. We then write the data directly into the
5112 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
5114 /* check if we cross pages */
5115 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
5118 offset = addr & (PAGE_SIZE - 1);
5121 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
5122 if (ret < nr_pages) {
5124 put_page(pages[ret]);
5129 for (i = 0; i < nr_pages; i++)
5130 map_page[i] = kmap_atomic(pages[i]);
5132 local_save_flags(irq_flags);
5133 size = sizeof(*entry) + cnt + 2; /* possible \n added */
5134 buffer = tr->trace_buffer.buffer;
5135 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
5136 irq_flags, preempt_count());
5138 /* Ring buffer disabled, return as if not open for write */
5143 entry = ring_buffer_event_data(event);
5144 entry->ip = _THIS_IP_;
5146 if (nr_pages == 2) {
5147 len = PAGE_SIZE - offset;
5148 memcpy(&entry->buf, map_page[0] + offset, len);
5149 memcpy(&entry->buf[len], map_page[1], cnt - len);
5151 memcpy(&entry->buf, map_page[0] + offset, cnt);
5153 if (entry->buf[cnt - 1] != '\n') {
5154 entry->buf[cnt] = '\n';
5155 entry->buf[cnt + 1] = '\0';
5157 entry->buf[cnt] = '\0';
5159 __buffer_unlock_commit(buffer, event);
5166 for (i = nr_pages - 1; i >= 0; i--) {
5167 kunmap_atomic(map_page[i]);
5174 static int tracing_clock_show(struct seq_file *m, void *v)
5176 struct trace_array *tr = m->private;
5179 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
5181 "%s%s%s%s", i ? " " : "",
5182 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
5183 i == tr->clock_id ? "]" : "");
5189 static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
5193 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
5194 if (strcmp(trace_clocks[i].name, clockstr) == 0)
5197 if (i == ARRAY_SIZE(trace_clocks))
5200 mutex_lock(&trace_types_lock);
5204 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
5207 * New clock may not be consistent with the previous clock.
5208 * Reset the buffer so that it doesn't have incomparable timestamps.
5210 tracing_reset_online_cpus(&tr->trace_buffer);
5212 #ifdef CONFIG_TRACER_MAX_TRACE
5213 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
5214 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
5215 tracing_reset_online_cpus(&tr->max_buffer);
5218 mutex_unlock(&trace_types_lock);
5223 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
5224 size_t cnt, loff_t *fpos)
5226 struct seq_file *m = filp->private_data;
5227 struct trace_array *tr = m->private;
5229 const char *clockstr;
5232 if (cnt >= sizeof(buf))
5235 if (copy_from_user(&buf, ubuf, cnt))
5240 clockstr = strstrip(buf);
5242 ret = tracing_set_clock(tr, clockstr);
5251 static int tracing_clock_open(struct inode *inode, struct file *file)
5253 struct trace_array *tr = inode->i_private;
5256 if (tracing_disabled)
5259 if (trace_array_get(tr))
5262 ret = single_open(file, tracing_clock_show, inode->i_private);
5264 trace_array_put(tr);
5269 struct ftrace_buffer_info {
5270 struct trace_iterator iter;
5275 #ifdef CONFIG_TRACER_SNAPSHOT
5276 static int tracing_snapshot_open(struct inode *inode, struct file *file)
5278 struct trace_array *tr = inode->i_private;
5279 struct trace_iterator *iter;
5283 if (trace_array_get(tr) < 0)
5286 if (file->f_mode & FMODE_READ) {
5287 iter = __tracing_open(inode, file, true);
5289 ret = PTR_ERR(iter);
5291 /* Writes still need the seq_file to hold the private data */
5293 m = kzalloc(sizeof(*m), GFP_KERNEL);
5296 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5304 iter->trace_buffer = &tr->max_buffer;
5305 iter->cpu_file = tracing_get_cpu(inode);
5307 file->private_data = m;
5311 trace_array_put(tr);
5317 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5320 struct seq_file *m = filp->private_data;
5321 struct trace_iterator *iter = m->private;
5322 struct trace_array *tr = iter->tr;
5326 ret = tracing_update_buffers();
5330 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5334 mutex_lock(&trace_types_lock);
5336 if (tr->current_trace->use_max_tr) {
5343 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5347 if (tr->allocated_snapshot)
5351 /* Only allow per-cpu swap if the ring buffer supports it */
5352 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5353 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5358 if (!tr->allocated_snapshot) {
5359 ret = alloc_snapshot(tr);
5363 local_irq_disable();
5364 /* Now, we're going to swap */
5365 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5366 update_max_tr(tr, current, smp_processor_id());
5368 update_max_tr_single(tr, current, iter->cpu_file);
5372 if (tr->allocated_snapshot) {
5373 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5374 tracing_reset_online_cpus(&tr->max_buffer);
5376 tracing_reset(&tr->max_buffer, iter->cpu_file);
5386 mutex_unlock(&trace_types_lock);
5390 static int tracing_snapshot_release(struct inode *inode, struct file *file)
5392 struct seq_file *m = file->private_data;
5395 ret = tracing_release(inode, file);
5397 if (file->f_mode & FMODE_READ)
5400 /* If write only, the seq_file is just a stub */
5408 static int tracing_buffers_open(struct inode *inode, struct file *filp);
5409 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5410 size_t count, loff_t *ppos);
5411 static int tracing_buffers_release(struct inode *inode, struct file *file);
5412 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5413 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5415 static int snapshot_raw_open(struct inode *inode, struct file *filp)
5417 struct ftrace_buffer_info *info;
5420 ret = tracing_buffers_open(inode, filp);
5424 info = filp->private_data;
5426 if (info->iter.trace->use_max_tr) {
5427 tracing_buffers_release(inode, filp);
5431 info->iter.snapshot = true;
5432 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5437 #endif /* CONFIG_TRACER_SNAPSHOT */
5440 static const struct file_operations tracing_thresh_fops = {
5441 .open = tracing_open_generic,
5442 .read = tracing_thresh_read,
5443 .write = tracing_thresh_write,
5444 .llseek = generic_file_llseek,
5447 static const struct file_operations tracing_max_lat_fops = {
5448 .open = tracing_open_generic,
5449 .read = tracing_max_lat_read,
5450 .write = tracing_max_lat_write,
5451 .llseek = generic_file_llseek,
5454 static const struct file_operations set_tracer_fops = {
5455 .open = tracing_open_generic,
5456 .read = tracing_set_trace_read,
5457 .write = tracing_set_trace_write,
5458 .llseek = generic_file_llseek,
5461 static const struct file_operations tracing_pipe_fops = {
5462 .open = tracing_open_pipe,
5463 .poll = tracing_poll_pipe,
5464 .read = tracing_read_pipe,
5465 .splice_read = tracing_splice_read_pipe,
5466 .release = tracing_release_pipe,
5467 .llseek = no_llseek,
5470 static const struct file_operations tracing_entries_fops = {
5471 .open = tracing_open_generic_tr,
5472 .read = tracing_entries_read,
5473 .write = tracing_entries_write,
5474 .llseek = generic_file_llseek,
5475 .release = tracing_release_generic_tr,
5478 static const struct file_operations tracing_total_entries_fops = {
5479 .open = tracing_open_generic_tr,
5480 .read = tracing_total_entries_read,
5481 .llseek = generic_file_llseek,
5482 .release = tracing_release_generic_tr,
5485 static const struct file_operations tracing_free_buffer_fops = {
5486 .open = tracing_open_generic_tr,
5487 .write = tracing_free_buffer_write,
5488 .release = tracing_free_buffer_release,
5491 static const struct file_operations tracing_mark_fops = {
5492 .open = tracing_open_generic_tr,
5493 .write = tracing_mark_write,
5494 .llseek = generic_file_llseek,
5495 .release = tracing_release_generic_tr,
5498 static const struct file_operations trace_clock_fops = {
5499 .open = tracing_clock_open,
5501 .llseek = seq_lseek,
5502 .release = tracing_single_release_tr,
5503 .write = tracing_clock_write,
5506 #ifdef CONFIG_TRACER_SNAPSHOT
5507 static const struct file_operations snapshot_fops = {
5508 .open = tracing_snapshot_open,
5510 .write = tracing_snapshot_write,
5511 .llseek = tracing_lseek,
5512 .release = tracing_snapshot_release,
5515 static const struct file_operations snapshot_raw_fops = {
5516 .open = snapshot_raw_open,
5517 .read = tracing_buffers_read,
5518 .release = tracing_buffers_release,
5519 .splice_read = tracing_buffers_splice_read,
5520 .llseek = no_llseek,
5523 #endif /* CONFIG_TRACER_SNAPSHOT */
5525 static int tracing_buffers_open(struct inode *inode, struct file *filp)
5527 struct trace_array *tr = inode->i_private;
5528 struct ftrace_buffer_info *info;
5531 if (tracing_disabled)
5534 if (trace_array_get(tr) < 0)
5537 info = kzalloc(sizeof(*info), GFP_KERNEL);
5539 trace_array_put(tr);
5543 mutex_lock(&trace_types_lock);
5546 info->iter.cpu_file = tracing_get_cpu(inode);
5547 info->iter.trace = tr->current_trace;
5548 info->iter.trace_buffer = &tr->trace_buffer;
5550 /* Force reading ring buffer for first read */
5551 info->read = (unsigned int)-1;
5553 filp->private_data = info;
5555 tr->current_trace->ref++;
5557 mutex_unlock(&trace_types_lock);
5559 ret = nonseekable_open(inode, filp);
5561 trace_array_put(tr);
5567 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5569 struct ftrace_buffer_info *info = filp->private_data;
5570 struct trace_iterator *iter = &info->iter;
5572 return trace_poll(iter, filp, poll_table);
5576 tracing_buffers_read(struct file *filp, char __user *ubuf,
5577 size_t count, loff_t *ppos)
5579 struct ftrace_buffer_info *info = filp->private_data;
5580 struct trace_iterator *iter = &info->iter;
5587 #ifdef CONFIG_TRACER_MAX_TRACE
5588 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5593 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5598 /* Do we have previous read data to read? */
5599 if (info->read < PAGE_SIZE)
5603 trace_access_lock(iter->cpu_file);
5604 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
5608 trace_access_unlock(iter->cpu_file);
5611 if (trace_empty(iter)) {
5612 if ((filp->f_flags & O_NONBLOCK))
5615 ret = wait_on_pipe(iter, false);
5626 size = PAGE_SIZE - info->read;
5630 ret = copy_to_user(ubuf, info->spare + info->read, size);
5642 static int tracing_buffers_release(struct inode *inode, struct file *file)
5644 struct ftrace_buffer_info *info = file->private_data;
5645 struct trace_iterator *iter = &info->iter;
5647 mutex_lock(&trace_types_lock);
5649 iter->tr->current_trace->ref--;
5651 __trace_array_put(iter->tr);
5654 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5657 mutex_unlock(&trace_types_lock);
5663 struct ring_buffer *buffer;
5668 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5669 struct pipe_buffer *buf)
5671 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5676 ring_buffer_free_read_page(ref->buffer, ref->page);
5681 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5682 struct pipe_buffer *buf)
5684 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5689 /* Pipe buffer operations for a buffer. */
5690 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5692 .confirm = generic_pipe_buf_confirm,
5693 .release = buffer_pipe_buf_release,
5694 .steal = generic_pipe_buf_steal,
5695 .get = buffer_pipe_buf_get,
5699 * Callback from splice_to_pipe(), if we need to release some pages
5700 * at the end of the spd in case we error'ed out in filling the pipe.
5702 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5704 struct buffer_ref *ref =
5705 (struct buffer_ref *)spd->partial[i].private;
5710 ring_buffer_free_read_page(ref->buffer, ref->page);
5712 spd->partial[i].private = 0;
5716 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5717 struct pipe_inode_info *pipe, size_t len,
5720 struct ftrace_buffer_info *info = file->private_data;
5721 struct trace_iterator *iter = &info->iter;
5722 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5723 struct page *pages_def[PIPE_DEF_BUFFERS];
5724 struct splice_pipe_desc spd = {
5726 .partial = partial_def,
5727 .nr_pages_max = PIPE_DEF_BUFFERS,
5729 .ops = &buffer_pipe_buf_ops,
5730 .spd_release = buffer_spd_release,
5732 struct buffer_ref *ref;
5733 int entries, size, i;
5736 #ifdef CONFIG_TRACER_MAX_TRACE
5737 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5741 if (splice_grow_spd(pipe, &spd))
5744 if (*ppos & (PAGE_SIZE - 1))
5747 if (len & (PAGE_SIZE - 1)) {
5748 if (len < PAGE_SIZE)
5754 trace_access_lock(iter->cpu_file);
5755 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5757 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
5761 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5768 ref->buffer = iter->trace_buffer->buffer;
5769 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5776 r = ring_buffer_read_page(ref->buffer, &ref->page,
5777 len, iter->cpu_file, 1);
5779 ring_buffer_free_read_page(ref->buffer, ref->page);
5785 * zero out any left over data, this is going to
5788 size = ring_buffer_page_len(ref->page);
5789 if (size < PAGE_SIZE)
5790 memset(ref->page + size, 0, PAGE_SIZE - size);
5792 page = virt_to_page(ref->page);
5794 spd.pages[i] = page;
5795 spd.partial[i].len = PAGE_SIZE;
5796 spd.partial[i].offset = 0;
5797 spd.partial[i].private = (unsigned long)ref;
5801 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5804 trace_access_unlock(iter->cpu_file);
5807 /* did we read anything? */
5808 if (!spd.nr_pages) {
5812 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
5815 ret = wait_on_pipe(iter, true);
5822 ret = splice_to_pipe(pipe, &spd);
5823 splice_shrink_spd(&spd);
5828 static const struct file_operations tracing_buffers_fops = {
5829 .open = tracing_buffers_open,
5830 .read = tracing_buffers_read,
5831 .poll = tracing_buffers_poll,
5832 .release = tracing_buffers_release,
5833 .splice_read = tracing_buffers_splice_read,
5834 .llseek = no_llseek,
5838 tracing_stats_read(struct file *filp, char __user *ubuf,
5839 size_t count, loff_t *ppos)
5841 struct inode *inode = file_inode(filp);
5842 struct trace_array *tr = inode->i_private;
5843 struct trace_buffer *trace_buf = &tr->trace_buffer;
5844 int cpu = tracing_get_cpu(inode);
5845 struct trace_seq *s;
5847 unsigned long long t;
5848 unsigned long usec_rem;
5850 s = kmalloc(sizeof(*s), GFP_KERNEL);
5856 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5857 trace_seq_printf(s, "entries: %ld\n", cnt);
5859 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5860 trace_seq_printf(s, "overrun: %ld\n", cnt);
5862 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5863 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5865 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5866 trace_seq_printf(s, "bytes: %ld\n", cnt);
5868 if (trace_clocks[tr->clock_id].in_ns) {
5869 /* local or global for trace_clock */
5870 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5871 usec_rem = do_div(t, USEC_PER_SEC);
5872 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5875 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5876 usec_rem = do_div(t, USEC_PER_SEC);
5877 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5879 /* counter or tsc mode for trace_clock */
5880 trace_seq_printf(s, "oldest event ts: %llu\n",
5881 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5883 trace_seq_printf(s, "now ts: %llu\n",
5884 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5887 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5888 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5890 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5891 trace_seq_printf(s, "read events: %ld\n", cnt);
5893 count = simple_read_from_buffer(ubuf, count, ppos,
5894 s->buffer, trace_seq_used(s));
5901 static const struct file_operations tracing_stats_fops = {
5902 .open = tracing_open_generic_tr,
5903 .read = tracing_stats_read,
5904 .llseek = generic_file_llseek,
5905 .release = tracing_release_generic_tr,
5908 #ifdef CONFIG_DYNAMIC_FTRACE
5910 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5916 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5917 size_t cnt, loff_t *ppos)
5919 static char ftrace_dyn_info_buffer[1024];
5920 static DEFINE_MUTEX(dyn_info_mutex);
5921 unsigned long *p = filp->private_data;
5922 char *buf = ftrace_dyn_info_buffer;
5923 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5926 mutex_lock(&dyn_info_mutex);
5927 r = sprintf(buf, "%ld ", *p);
5929 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5932 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5934 mutex_unlock(&dyn_info_mutex);
5939 static const struct file_operations tracing_dyn_info_fops = {
5940 .open = tracing_open_generic,
5941 .read = tracing_read_dyn_info,
5942 .llseek = generic_file_llseek,
5944 #endif /* CONFIG_DYNAMIC_FTRACE */
5946 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5948 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5954 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5956 unsigned long *count = (long *)data;
5968 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5969 struct ftrace_probe_ops *ops, void *data)
5971 long count = (long)data;
5973 seq_printf(m, "%ps:", (void *)ip);
5975 seq_puts(m, "snapshot");
5978 seq_puts(m, ":unlimited\n");
5980 seq_printf(m, ":count=%ld\n", count);
5985 static struct ftrace_probe_ops snapshot_probe_ops = {
5986 .func = ftrace_snapshot,
5987 .print = ftrace_snapshot_print,
5990 static struct ftrace_probe_ops snapshot_count_probe_ops = {
5991 .func = ftrace_count_snapshot,
5992 .print = ftrace_snapshot_print,
5996 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5997 char *glob, char *cmd, char *param, int enable)
5999 struct ftrace_probe_ops *ops;
6000 void *count = (void *)-1;
6004 /* hash funcs only work with set_ftrace_filter */
6008 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
6010 if (glob[0] == '!') {
6011 unregister_ftrace_function_probe_func(glob+1, ops);
6018 number = strsep(¶m, ":");
6020 if (!strlen(number))
6024 * We use the callback data field (which is a pointer)
6027 ret = kstrtoul(number, 0, (unsigned long *)&count);
6032 ret = register_ftrace_function_probe(glob, ops, count);
6035 alloc_snapshot(&global_trace);
6037 return ret < 0 ? ret : 0;
6040 static struct ftrace_func_command ftrace_snapshot_cmd = {
6042 .func = ftrace_trace_snapshot_callback,
6045 static __init int register_snapshot_cmd(void)
6047 return register_ftrace_command(&ftrace_snapshot_cmd);
6050 static inline __init int register_snapshot_cmd(void) { return 0; }
6051 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
6053 static struct dentry *tracing_get_dentry(struct trace_array *tr)
6055 if (WARN_ON(!tr->dir))
6056 return ERR_PTR(-ENODEV);
6058 /* Top directory uses NULL as the parent */
6059 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
6062 /* All sub buffers have a descriptor */
6066 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
6068 struct dentry *d_tracer;
6071 return tr->percpu_dir;
6073 d_tracer = tracing_get_dentry(tr);
6074 if (IS_ERR(d_tracer))
6077 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
6079 WARN_ONCE(!tr->percpu_dir,
6080 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
6082 return tr->percpu_dir;
6085 static struct dentry *
6086 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
6087 void *data, long cpu, const struct file_operations *fops)
6089 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
6091 if (ret) /* See tracing_get_cpu() */
6092 d_inode(ret)->i_cdev = (void *)(cpu + 1);
6097 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
6099 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
6100 struct dentry *d_cpu;
6101 char cpu_dir[30]; /* 30 characters should be more than enough */
6106 snprintf(cpu_dir, 30, "cpu%ld", cpu);
6107 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
6109 pr_warning("Could not create tracefs '%s' entry\n", cpu_dir);
6113 /* per cpu trace_pipe */
6114 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
6115 tr, cpu, &tracing_pipe_fops);
6118 trace_create_cpu_file("trace", 0644, d_cpu,
6119 tr, cpu, &tracing_fops);
6121 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
6122 tr, cpu, &tracing_buffers_fops);
6124 trace_create_cpu_file("stats", 0444, d_cpu,
6125 tr, cpu, &tracing_stats_fops);
6127 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
6128 tr, cpu, &tracing_entries_fops);
6130 #ifdef CONFIG_TRACER_SNAPSHOT
6131 trace_create_cpu_file("snapshot", 0644, d_cpu,
6132 tr, cpu, &snapshot_fops);
6134 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
6135 tr, cpu, &snapshot_raw_fops);
6139 #ifdef CONFIG_FTRACE_SELFTEST
6140 /* Let selftest have access to static functions in this file */
6141 #include "trace_selftest.c"
6144 struct trace_option_dentry {
6145 struct tracer_opt *opt;
6146 struct tracer_flags *flags;
6147 struct trace_array *tr;
6148 struct dentry *entry;
6152 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
6155 struct trace_option_dentry *topt = filp->private_data;
6158 if (topt->flags->val & topt->opt->bit)
6163 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6167 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
6170 struct trace_option_dentry *topt = filp->private_data;
6174 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6178 if (val != 0 && val != 1)
6181 if (!!(topt->flags->val & topt->opt->bit) != val) {
6182 mutex_lock(&trace_types_lock);
6183 ret = __set_tracer_option(topt->tr, topt->flags,
6185 mutex_unlock(&trace_types_lock);
6196 static const struct file_operations trace_options_fops = {
6197 .open = tracing_open_generic,
6198 .read = trace_options_read,
6199 .write = trace_options_write,
6200 .llseek = generic_file_llseek,
6204 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
6207 long index = (long)filp->private_data;
6210 if (trace_flags & (1 << index))
6215 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6219 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
6222 struct trace_array *tr = &global_trace;
6223 long index = (long)filp->private_data;
6227 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6231 if (val != 0 && val != 1)
6234 mutex_lock(&trace_types_lock);
6235 ret = set_tracer_flag(tr, 1 << index, val);
6236 mutex_unlock(&trace_types_lock);
6246 static const struct file_operations trace_options_core_fops = {
6247 .open = tracing_open_generic,
6248 .read = trace_options_core_read,
6249 .write = trace_options_core_write,
6250 .llseek = generic_file_llseek,
6253 struct dentry *trace_create_file(const char *name,
6255 struct dentry *parent,
6257 const struct file_operations *fops)
6261 ret = tracefs_create_file(name, mode, parent, data, fops);
6263 pr_warning("Could not create tracefs '%s' entry\n", name);
6269 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
6271 struct dentry *d_tracer;
6276 d_tracer = tracing_get_dentry(tr);
6277 if (IS_ERR(d_tracer))
6280 tr->options = tracefs_create_dir("options", d_tracer);
6282 pr_warning("Could not create tracefs directory 'options'\n");
6290 create_trace_option_file(struct trace_array *tr,
6291 struct trace_option_dentry *topt,
6292 struct tracer_flags *flags,
6293 struct tracer_opt *opt)
6295 struct dentry *t_options;
6297 t_options = trace_options_init_dentry(tr);
6301 topt->flags = flags;
6305 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
6306 &trace_options_fops);
6310 static struct trace_option_dentry *
6311 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
6313 struct trace_option_dentry *topts;
6314 struct tracer_flags *flags;
6315 struct tracer_opt *opts;
6321 flags = tracer->flags;
6323 if (!flags || !flags->opts)
6328 for (cnt = 0; opts[cnt].name; cnt++)
6331 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
6335 for (cnt = 0; opts[cnt].name; cnt++)
6336 create_trace_option_file(tr, &topts[cnt], flags,
6343 destroy_trace_option_files(struct trace_option_dentry *topts)
6350 for (cnt = 0; topts[cnt].opt; cnt++)
6351 tracefs_remove(topts[cnt].entry);
6356 static struct dentry *
6357 create_trace_option_core_file(struct trace_array *tr,
6358 const char *option, long index)
6360 struct dentry *t_options;
6362 t_options = trace_options_init_dentry(tr);
6366 return trace_create_file(option, 0644, t_options, (void *)index,
6367 &trace_options_core_fops);
6370 static __init void create_trace_options_dir(struct trace_array *tr)
6372 struct dentry *t_options;
6375 t_options = trace_options_init_dentry(tr);
6379 for (i = 0; trace_options[i]; i++)
6380 create_trace_option_core_file(tr, trace_options[i], i);
6384 rb_simple_read(struct file *filp, char __user *ubuf,
6385 size_t cnt, loff_t *ppos)
6387 struct trace_array *tr = filp->private_data;
6391 r = tracer_tracing_is_on(tr);
6392 r = sprintf(buf, "%d\n", r);
6394 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6398 rb_simple_write(struct file *filp, const char __user *ubuf,
6399 size_t cnt, loff_t *ppos)
6401 struct trace_array *tr = filp->private_data;
6402 struct ring_buffer *buffer = tr->trace_buffer.buffer;
6406 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6411 mutex_lock(&trace_types_lock);
6413 tracer_tracing_on(tr);
6414 if (tr->current_trace->start)
6415 tr->current_trace->start(tr);
6417 tracer_tracing_off(tr);
6418 if (tr->current_trace->stop)
6419 tr->current_trace->stop(tr);
6421 mutex_unlock(&trace_types_lock);
6429 static const struct file_operations rb_simple_fops = {
6430 .open = tracing_open_generic_tr,
6431 .read = rb_simple_read,
6432 .write = rb_simple_write,
6433 .release = tracing_release_generic_tr,
6434 .llseek = default_llseek,
6437 struct dentry *trace_instance_dir;
6440 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
6443 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
6445 enum ring_buffer_flags rb_flags;
6447 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
6451 buf->buffer = ring_buffer_alloc(size, rb_flags);
6455 buf->data = alloc_percpu(struct trace_array_cpu);
6457 ring_buffer_free(buf->buffer);
6461 /* Allocate the first page for all buffers */
6462 set_buffer_entries(&tr->trace_buffer,
6463 ring_buffer_size(tr->trace_buffer.buffer, 0));
6468 static int allocate_trace_buffers(struct trace_array *tr, int size)
6472 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6476 #ifdef CONFIG_TRACER_MAX_TRACE
6477 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6478 allocate_snapshot ? size : 1);
6480 ring_buffer_free(tr->trace_buffer.buffer);
6481 free_percpu(tr->trace_buffer.data);
6484 tr->allocated_snapshot = allocate_snapshot;
6487 * Only the top level trace array gets its snapshot allocated
6488 * from the kernel command line.
6490 allocate_snapshot = false;
6495 static void free_trace_buffer(struct trace_buffer *buf)
6498 ring_buffer_free(buf->buffer);
6500 free_percpu(buf->data);
6505 static void free_trace_buffers(struct trace_array *tr)
6510 free_trace_buffer(&tr->trace_buffer);
6512 #ifdef CONFIG_TRACER_MAX_TRACE
6513 free_trace_buffer(&tr->max_buffer);
6517 static int instance_mkdir(const char *name)
6519 struct trace_array *tr;
6522 mutex_lock(&trace_types_lock);
6525 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6526 if (tr->name && strcmp(tr->name, name) == 0)
6531 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6535 tr->name = kstrdup(name, GFP_KERNEL);
6539 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6542 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6544 raw_spin_lock_init(&tr->start_lock);
6546 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6548 tr->current_trace = &nop_trace;
6550 INIT_LIST_HEAD(&tr->systems);
6551 INIT_LIST_HEAD(&tr->events);
6553 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
6556 tr->dir = tracefs_create_dir(name, trace_instance_dir);
6560 ret = event_trace_add_tracer(tr->dir, tr);
6562 tracefs_remove_recursive(tr->dir);
6566 init_tracer_tracefs(tr, tr->dir);
6568 list_add(&tr->list, &ftrace_trace_arrays);
6570 mutex_unlock(&trace_types_lock);
6575 free_trace_buffers(tr);
6576 free_cpumask_var(tr->tracing_cpumask);
6581 mutex_unlock(&trace_types_lock);
6587 static int instance_rmdir(const char *name)
6589 struct trace_array *tr;
6593 mutex_lock(&trace_types_lock);
6596 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6597 if (tr->name && strcmp(tr->name, name) == 0) {
6606 if (tr->ref || (tr->current_trace && tr->current_trace->ref))
6609 list_del(&tr->list);
6611 tracing_set_nop(tr);
6612 event_trace_del_tracer(tr);
6613 ftrace_destroy_function_files(tr);
6614 debugfs_remove_recursive(tr->dir);
6615 free_trace_buffers(tr);
6623 mutex_unlock(&trace_types_lock);
6628 static __init void create_trace_instances(struct dentry *d_tracer)
6630 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
6633 if (WARN_ON(!trace_instance_dir))
6638 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
6642 trace_create_file("available_tracers", 0444, d_tracer,
6643 tr, &show_traces_fops);
6645 trace_create_file("current_tracer", 0644, d_tracer,
6646 tr, &set_tracer_fops);
6648 trace_create_file("tracing_cpumask", 0644, d_tracer,
6649 tr, &tracing_cpumask_fops);
6651 trace_create_file("trace_options", 0644, d_tracer,
6652 tr, &tracing_iter_fops);
6654 trace_create_file("trace", 0644, d_tracer,
6657 trace_create_file("trace_pipe", 0444, d_tracer,
6658 tr, &tracing_pipe_fops);
6660 trace_create_file("buffer_size_kb", 0644, d_tracer,
6661 tr, &tracing_entries_fops);
6663 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6664 tr, &tracing_total_entries_fops);
6666 trace_create_file("free_buffer", 0200, d_tracer,
6667 tr, &tracing_free_buffer_fops);
6669 trace_create_file("trace_marker", 0220, d_tracer,
6670 tr, &tracing_mark_fops);
6672 trace_create_file("trace_clock", 0644, d_tracer, tr,
6675 trace_create_file("tracing_on", 0644, d_tracer,
6676 tr, &rb_simple_fops);
6678 #ifdef CONFIG_TRACER_MAX_TRACE
6679 trace_create_file("tracing_max_latency", 0644, d_tracer,
6680 &tr->max_latency, &tracing_max_lat_fops);
6683 if (ftrace_create_function_files(tr, d_tracer))
6684 WARN(1, "Could not allocate function filter files");
6686 #ifdef CONFIG_TRACER_SNAPSHOT
6687 trace_create_file("snapshot", 0644, d_tracer,
6688 tr, &snapshot_fops);
6691 for_each_tracing_cpu(cpu)
6692 tracing_init_tracefs_percpu(tr, cpu);
6696 static struct vfsmount *trace_automount(void *ingore)
6698 struct vfsmount *mnt;
6699 struct file_system_type *type;
6702 * To maintain backward compatibility for tools that mount
6703 * debugfs to get to the tracing facility, tracefs is automatically
6704 * mounted to the debugfs/tracing directory.
6706 type = get_fs_type("tracefs");
6709 mnt = vfs_kern_mount(type, 0, "tracefs", NULL);
6710 put_filesystem(type);
6719 * tracing_init_dentry - initialize top level trace array
6721 * This is called when creating files or directories in the tracing
6722 * directory. It is called via fs_initcall() by any of the boot up code
6723 * and expects to return the dentry of the top level tracing directory.
6725 struct dentry *tracing_init_dentry(void)
6727 struct trace_array *tr = &global_trace;
6729 /* The top level trace array uses NULL as parent */
6733 if (WARN_ON(!debugfs_initialized()))
6734 return ERR_PTR(-ENODEV);
6737 * As there may still be users that expect the tracing
6738 * files to exist in debugfs/tracing, we must automount
6739 * the tracefs file system there, so older tools still
6740 * work with the newer kerenl.
6742 tr->dir = debugfs_create_automount("tracing", NULL,
6743 trace_automount, NULL);
6745 pr_warn_once("Could not create debugfs directory 'tracing'\n");
6746 return ERR_PTR(-ENOMEM);
6752 extern struct trace_enum_map *__start_ftrace_enum_maps[];
6753 extern struct trace_enum_map *__stop_ftrace_enum_maps[];
6755 static void __init trace_enum_init(void)
6759 len = __stop_ftrace_enum_maps - __start_ftrace_enum_maps;
6760 trace_insert_enum_map(NULL, __start_ftrace_enum_maps, len);
6763 #ifdef CONFIG_MODULES
6764 static void trace_module_add_enums(struct module *mod)
6766 if (!mod->num_trace_enums)
6770 * Modules with bad taint do not have events created, do
6771 * not bother with enums either.
6773 if (trace_module_has_bad_taint(mod))
6776 trace_insert_enum_map(mod, mod->trace_enums, mod->num_trace_enums);
6779 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
6780 static void trace_module_remove_enums(struct module *mod)
6782 union trace_enum_map_item *map;
6783 union trace_enum_map_item **last = &trace_enum_maps;
6785 if (!mod->num_trace_enums)
6788 mutex_lock(&trace_enum_mutex);
6790 map = trace_enum_maps;
6793 if (map->head.mod == mod)
6795 map = trace_enum_jmp_to_tail(map);
6796 last = &map->tail.next;
6797 map = map->tail.next;
6802 *last = trace_enum_jmp_to_tail(map)->tail.next;
6805 mutex_unlock(&trace_enum_mutex);
6808 static inline void trace_module_remove_enums(struct module *mod) { }
6809 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
6811 static int trace_module_notify(struct notifier_block *self,
6812 unsigned long val, void *data)
6814 struct module *mod = data;
6817 case MODULE_STATE_COMING:
6818 trace_module_add_enums(mod);
6820 case MODULE_STATE_GOING:
6821 trace_module_remove_enums(mod);
6828 static struct notifier_block trace_module_nb = {
6829 .notifier_call = trace_module_notify,
6832 #endif /* CONFIG_MODULES */
6834 static __init int tracer_init_tracefs(void)
6836 struct dentry *d_tracer;
6838 trace_access_lock_init();
6840 d_tracer = tracing_init_dentry();
6841 if (IS_ERR(d_tracer))
6844 init_tracer_tracefs(&global_trace, d_tracer);
6846 trace_create_file("tracing_thresh", 0644, d_tracer,
6847 &global_trace, &tracing_thresh_fops);
6849 trace_create_file("README", 0444, d_tracer,
6850 NULL, &tracing_readme_fops);
6852 trace_create_file("saved_cmdlines", 0444, d_tracer,
6853 NULL, &tracing_saved_cmdlines_fops);
6855 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
6856 NULL, &tracing_saved_cmdlines_size_fops);
6860 trace_create_enum_file(d_tracer);
6862 #ifdef CONFIG_MODULES
6863 register_module_notifier(&trace_module_nb);
6866 #ifdef CONFIG_DYNAMIC_FTRACE
6867 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6868 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
6871 create_trace_instances(d_tracer);
6873 create_trace_options_dir(&global_trace);
6875 /* If the tracer was started via cmdline, create options for it here */
6876 if (global_trace.current_trace != &nop_trace)
6877 update_tracer_options(&global_trace, global_trace.current_trace);
6882 static int trace_panic_handler(struct notifier_block *this,
6883 unsigned long event, void *unused)
6885 if (ftrace_dump_on_oops)
6886 ftrace_dump(ftrace_dump_on_oops);
6890 static struct notifier_block trace_panic_notifier = {
6891 .notifier_call = trace_panic_handler,
6893 .priority = 150 /* priority: INT_MAX >= x >= 0 */
6896 static int trace_die_handler(struct notifier_block *self,
6902 if (ftrace_dump_on_oops)
6903 ftrace_dump(ftrace_dump_on_oops);
6911 static struct notifier_block trace_die_notifier = {
6912 .notifier_call = trace_die_handler,
6917 * printk is set to max of 1024, we really don't need it that big.
6918 * Nothing should be printing 1000 characters anyway.
6920 #define TRACE_MAX_PRINT 1000
6923 * Define here KERN_TRACE so that we have one place to modify
6924 * it if we decide to change what log level the ftrace dump
6927 #define KERN_TRACE KERN_EMERG
6930 trace_printk_seq(struct trace_seq *s)
6932 /* Probably should print a warning here. */
6933 if (s->seq.len >= TRACE_MAX_PRINT)
6934 s->seq.len = TRACE_MAX_PRINT;
6937 * More paranoid code. Although the buffer size is set to
6938 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
6939 * an extra layer of protection.
6941 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
6942 s->seq.len = s->seq.size - 1;
6944 /* should be zero ended, but we are paranoid. */
6945 s->buffer[s->seq.len] = 0;
6947 printk(KERN_TRACE "%s", s->buffer);
6952 void trace_init_global_iter(struct trace_iterator *iter)
6954 iter->tr = &global_trace;
6955 iter->trace = iter->tr->current_trace;
6956 iter->cpu_file = RING_BUFFER_ALL_CPUS;
6957 iter->trace_buffer = &global_trace.trace_buffer;
6959 if (iter->trace && iter->trace->open)
6960 iter->trace->open(iter);
6962 /* Annotate start of buffers if we had overruns */
6963 if (ring_buffer_overruns(iter->trace_buffer->buffer))
6964 iter->iter_flags |= TRACE_FILE_ANNOTATE;
6966 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6967 if (trace_clocks[iter->tr->clock_id].in_ns)
6968 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6971 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6973 /* use static because iter can be a bit big for the stack */
6974 static struct trace_iterator iter;
6975 static atomic_t dump_running;
6976 unsigned int old_userobj;
6977 unsigned long flags;
6980 /* Only allow one dump user at a time. */
6981 if (atomic_inc_return(&dump_running) != 1) {
6982 atomic_dec(&dump_running);
6987 * Always turn off tracing when we dump.
6988 * We don't need to show trace output of what happens
6989 * between multiple crashes.
6991 * If the user does a sysrq-z, then they can re-enable
6992 * tracing with echo 1 > tracing_on.
6996 local_irq_save(flags);
6998 /* Simulate the iterator */
6999 trace_init_global_iter(&iter);
7001 for_each_tracing_cpu(cpu) {
7002 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
7005 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
7007 /* don't look at user memory in panic mode */
7008 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
7010 switch (oops_dump_mode) {
7012 iter.cpu_file = RING_BUFFER_ALL_CPUS;
7015 iter.cpu_file = raw_smp_processor_id();
7020 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
7021 iter.cpu_file = RING_BUFFER_ALL_CPUS;
7024 printk(KERN_TRACE "Dumping ftrace buffer:\n");
7026 /* Did function tracer already get disabled? */
7027 if (ftrace_is_dead()) {
7028 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
7029 printk("# MAY BE MISSING FUNCTION EVENTS\n");
7033 * We need to stop all tracing on all CPUS to read the
7034 * the next buffer. This is a bit expensive, but is
7035 * not done often. We fill all what we can read,
7036 * and then release the locks again.
7039 while (!trace_empty(&iter)) {
7042 printk(KERN_TRACE "---------------------------------\n");
7046 /* reset all but tr, trace, and overruns */
7047 memset(&iter.seq, 0,
7048 sizeof(struct trace_iterator) -
7049 offsetof(struct trace_iterator, seq));
7050 iter.iter_flags |= TRACE_FILE_LAT_FMT;
7053 if (trace_find_next_entry_inc(&iter) != NULL) {
7056 ret = print_trace_line(&iter);
7057 if (ret != TRACE_TYPE_NO_CONSUME)
7058 trace_consume(&iter);
7060 touch_nmi_watchdog();
7062 trace_printk_seq(&iter.seq);
7066 printk(KERN_TRACE " (ftrace buffer empty)\n");
7068 printk(KERN_TRACE "---------------------------------\n");
7071 trace_flags |= old_userobj;
7073 for_each_tracing_cpu(cpu) {
7074 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
7076 atomic_dec(&dump_running);
7077 local_irq_restore(flags);
7079 EXPORT_SYMBOL_GPL(ftrace_dump);
7081 __init static int tracer_alloc_buffers(void)
7086 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
7089 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
7090 goto out_free_buffer_mask;
7092 /* Only allocate trace_printk buffers if a trace_printk exists */
7093 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
7094 /* Must be called before global_trace.buffer is allocated */
7095 trace_printk_init_buffers();
7097 /* To save memory, keep the ring buffer size to its minimum */
7098 if (ring_buffer_expanded)
7099 ring_buf_size = trace_buf_size;
7103 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
7104 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
7106 raw_spin_lock_init(&global_trace.start_lock);
7108 /* Used for event triggers */
7109 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
7111 goto out_free_cpumask;
7113 if (trace_create_savedcmd() < 0)
7114 goto out_free_temp_buffer;
7116 /* TODO: make the number of buffers hot pluggable with CPUS */
7117 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
7118 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
7120 goto out_free_savedcmd;
7123 if (global_trace.buffer_disabled)
7126 if (trace_boot_clock) {
7127 ret = tracing_set_clock(&global_trace, trace_boot_clock);
7129 pr_warning("Trace clock %s not defined, going back to default\n",
7134 * register_tracer() might reference current_trace, so it
7135 * needs to be set before we register anything. This is
7136 * just a bootstrap of current_trace anyway.
7138 global_trace.current_trace = &nop_trace;
7140 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7142 ftrace_init_global_array_ops(&global_trace);
7144 register_tracer(&nop_trace);
7146 /* All seems OK, enable tracing */
7147 tracing_disabled = 0;
7149 atomic_notifier_chain_register(&panic_notifier_list,
7150 &trace_panic_notifier);
7152 register_die_notifier(&trace_die_notifier);
7154 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
7156 INIT_LIST_HEAD(&global_trace.systems);
7157 INIT_LIST_HEAD(&global_trace.events);
7158 list_add(&global_trace.list, &ftrace_trace_arrays);
7160 while (trace_boot_options) {
7163 option = strsep(&trace_boot_options, ",");
7164 trace_set_options(&global_trace, option);
7167 register_snapshot_cmd();
7172 free_saved_cmdlines_buffer(savedcmd);
7173 out_free_temp_buffer:
7174 ring_buffer_free(temp_buffer);
7176 free_cpumask_var(global_trace.tracing_cpumask);
7177 out_free_buffer_mask:
7178 free_cpumask_var(tracing_buffer_mask);
7183 void __init trace_init(void)
7185 if (tracepoint_printk) {
7186 tracepoint_print_iter =
7187 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
7188 if (WARN_ON(!tracepoint_print_iter))
7189 tracepoint_printk = 0;
7191 tracer_alloc_buffers();
7195 __init static int clear_boot_tracer(void)
7198 * The default tracer at boot buffer is an init section.
7199 * This function is called in lateinit. If we did not
7200 * find the boot tracer, then clear it out, to prevent
7201 * later registration from accessing the buffer that is
7202 * about to be freed.
7204 if (!default_bootup_tracer)
7207 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
7208 default_bootup_tracer);
7209 default_bootup_tracer = NULL;
7214 fs_initcall(tracer_init_tracefs);
7215 late_initcall(clear_boot_tracer);