kernel/kernel/trace/trace_events.c

   1 /*
   2  * event tracer
   3  *
   4  * Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
   5  *
   6  *  - Added format output of fields of the trace point.
   7  *    This was based off of work by Tom Zanussi <tzanussi@gmail.com>.
   8  *
   9  */
  10
  11 #define pr_fmt(fmt) fmt
  12
  13 #include <linux/workqueue.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/kthread.h>
  16 #include <linux/tracefs.h>
  17 #include <linux/uaccess.h>
  18 #include <linux/bsearch.h>
  19 #include <linux/module.h>
  20 #include <linux/ctype.h>
  21 #include <linux/sort.h>
  22 #include <linux/slab.h>
  23 #include <linux/delay.h>
  24
  25 #include <trace/events/sched.h>
  26
  27 #include <asm/setup.h>
  28
  29 #include "trace_output.h"
  30
  31 #undef TRACE_SYSTEM
  32 #define TRACE_SYSTEM "TRACE_SYSTEM"
  33
  34 DEFINE_MUTEX(event_mutex);
  35
  36 LIST_HEAD(ftrace_events);
  37 static LIST_HEAD(ftrace_generic_fields);
  38 static LIST_HEAD(ftrace_common_fields);
  39
  40 #define GFP_TRACE (GFP_KERNEL | __GFP_ZERO)
  41
  42 static struct kmem_cache *field_cachep;
  43 static struct kmem_cache *file_cachep;
  44
  45 static inline int system_refcount(struct event_subsystem *system)
  46 {
  47         return system->ref_count;
  48 }
  49
  50 static int system_refcount_inc(struct event_subsystem *system)
  51 {
  52         return system->ref_count++;
  53 }
  54
  55 static int system_refcount_dec(struct event_subsystem *system)
  56 {
  57         return --system->ref_count;
  58 }
  59
  60 /* Double loops, do not use break, only goto's work */
  61 #define do_for_each_event_file(tr, file)                        \
  62         list_for_each_entry(tr, &ftrace_trace_arrays, list) {   \
  63                 list_for_each_entry(file, &tr->events, list)
  64
  65 #define do_for_each_event_file_safe(tr, file)                   \
  66         list_for_each_entry(tr, &ftrace_trace_arrays, list) {   \
  67                 struct trace_event_file *___n;                          \
  68                 list_for_each_entry_safe(file, ___n, &tr->events, list)
  69
  70 #define while_for_each_event_file()             \
  71         }
  72
  73 static struct list_head *
  74 trace_get_fields(struct trace_event_call *event_call)
  75 {
  76         if (!event_call->class->get_fields)
  77                 return &event_call->class->fields;
  78         return event_call->class->get_fields(event_call);
  79 }
  80
  81 static struct ftrace_event_field *
  82 __find_event_field(struct list_head *head, char *name)
  83 {
  84         struct ftrace_event_field *field;
  85
  86         list_for_each_entry(field, head, link) {
  87                 if (!strcmp(field->name, name))
  88                         return field;
  89         }
  90
  91         return NULL;
  92 }
  93
  94 struct ftrace_event_field *
  95 trace_find_event_field(struct trace_event_call *call, char *name)
  96 {
  97         struct ftrace_event_field *field;
  98         struct list_head *head;
  99
 100         head = trace_get_fields(call);
 101         field = __find_event_field(head, name);
 102         if (field)
 103                 return field;
 104
 105         field = __find_event_field(&ftrace_generic_fields, name);
 106         if (field)
 107                 return field;
 108
 109         return __find_event_field(&ftrace_common_fields, name);
 110 }
 111
 112 static int __trace_define_field(struct list_head *head, const char *type,
 113                                 const char *name, int offset, int size,
 114                                 int is_signed, int filter_type)
 115 {
 116         struct ftrace_event_field *field;
 117
 118         field = kmem_cache_alloc(field_cachep, GFP_TRACE);
 119         if (!field)
 120                 return -ENOMEM;
 121
 122         field->name = name;
 123         field->type = type;
 124
 125         if (filter_type == FILTER_OTHER)
 126                 field->filter_type = filter_assign_type(type);
 127         else
 128                 field->filter_type = filter_type;
 129
 130         field->offset = offset;
 131         field->size = size;
 132         field->is_signed = is_signed;
 133
 134         list_add(&field->link, head);
 135
 136         return 0;
 137 }
 138
 139 int trace_define_field(struct trace_event_call *call, const char *type,
 140                        const char *name, int offset, int size, int is_signed,
 141                        int filter_type)
 142 {
 143         struct list_head *head;
 144
 145         if (WARN_ON(!call->class))
 146                 return 0;
 147
 148         head = trace_get_fields(call);
 149         return __trace_define_field(head, type, name, offset, size,
 150                                     is_signed, filter_type);
 151 }
 152 EXPORT_SYMBOL_GPL(trace_define_field);
 153
 154 #define __generic_field(type, item, filter_type)                        \
 155         ret = __trace_define_field(&ftrace_generic_fields, #type,       \
 156                                    #item, 0, 0, is_signed_type(type),   \
 157                                    filter_type);                        \
 158         if (ret)                                                        \
 159                 return ret;
 160
 161 #define __common_field(type, item)                                      \
 162         ret = __trace_define_field(&ftrace_common_fields, #type,        \
 163                                    "common_" #item,                     \
 164                                    offsetof(typeof(ent), item),         \
 165                                    sizeof(ent.item),                    \
 166                                    is_signed_type(type), FILTER_OTHER); \
 167         if (ret)                                                        \
 168                 return ret;
 169
 170 static int trace_define_generic_fields(void)
 171 {
 172         int ret;
 173
 174         __generic_field(int, CPU, FILTER_CPU);
 175         __generic_field(int, cpu, FILTER_CPU);
 176         __generic_field(char *, COMM, FILTER_COMM);
 177         __generic_field(char *, comm, FILTER_COMM);
 178
 179         return ret;
 180 }
 181
 182 static int trace_define_common_fields(void)
 183 {
 184         int ret;
 185         struct trace_entry ent;
 186
 187         __common_field(unsigned short, type);
 188         __common_field(unsigned char, flags);
 189         __common_field(unsigned char, preempt_count);
 190         __common_field(int, pid);
 191         __common_field(unsigned short, migrate_disable);
 192         __common_field(unsigned short, padding);
 193
 194         return ret;
 195 }
 196
 197 static void trace_destroy_fields(struct trace_event_call *call)
 198 {
 199         struct ftrace_event_field *field, *next;
 200         struct list_head *head;
 201
 202         head = trace_get_fields(call);
 203         list_for_each_entry_safe(field, next, head, link) {
 204                 list_del(&field->link);
 205                 kmem_cache_free(field_cachep, field);
 206         }
 207 }
 208
 209 int trace_event_raw_init(struct trace_event_call *call)
 210 {
 211         int id;
 212
 213         id = register_trace_event(&call->event);
 214         if (!id)
 215                 return -ENODEV;
 216
 217         return 0;
 218 }
 219 EXPORT_SYMBOL_GPL(trace_event_raw_init);
 220
 221 bool trace_event_ignore_this_pid(struct trace_event_file *trace_file)
 222 {
 223         struct trace_array *tr = trace_file->tr;
 224         struct trace_array_cpu *data;
 225         struct trace_pid_list *pid_list;
 226
 227         pid_list = rcu_dereference_sched(tr->filtered_pids);
 228         if (!pid_list)
 229                 return false;
 230
 231         data = this_cpu_ptr(tr->trace_buffer.data);
 232
 233         return data->ignore_pid;
 234 }
 235 EXPORT_SYMBOL_GPL(trace_event_ignore_this_pid);
 236
 237 void *trace_event_buffer_reserve(struct trace_event_buffer *fbuffer,
 238                                  struct trace_event_file *trace_file,
 239                                  unsigned long len)
 240 {
 241         struct trace_event_call *event_call = trace_file->event_call;
 242
 243         if ((trace_file->flags & EVENT_FILE_FL_PID_FILTER) &&
 244             trace_event_ignore_this_pid(trace_file))
 245                 return NULL;
 246
 247         local_save_flags(fbuffer->flags);
 248         fbuffer->pc = preempt_count();
 249         /*
 250          * If CONFIG_PREEMPT is enabled, then the tracepoint itself disables
 251          * preemption (adding one to the preempt_count). Since we are
 252          * interested in the preempt_count at the time the tracepoint was
 253          * hit, we need to subtract one to offset the increment.
 254          */
 255         if (IS_ENABLED(CONFIG_PREEMPT))
 256                 fbuffer->pc--;
 257         fbuffer->trace_file = trace_file;
 258
 259         fbuffer->event =
 260                 trace_event_buffer_lock_reserve(&fbuffer->buffer, trace_file,
 261                                                 event_call->event.type, len,
 262                                                 fbuffer->flags, fbuffer->pc);
 263         if (!fbuffer->event)
 264                 return NULL;
 265
 266         fbuffer->entry = ring_buffer_event_data(fbuffer->event);
 267         return fbuffer->entry;
 268 }
 269 EXPORT_SYMBOL_GPL(trace_event_buffer_reserve);
 270
 271 static DEFINE_SPINLOCK(tracepoint_iter_lock);
 272
 273 static void output_printk(struct trace_event_buffer *fbuffer)
 274 {
 275         struct trace_event_call *event_call;
 276         struct trace_event *event;
 277         unsigned long flags;
 278         struct trace_iterator *iter = tracepoint_print_iter;
 279
 280         if (!iter)
 281                 return;
 282
 283         event_call = fbuffer->trace_file->event_call;
 284         if (!event_call || !event_call->event.funcs ||
 285             !event_call->event.funcs->trace)
 286                 return;
 287
 288         event = &fbuffer->trace_file->event_call->event;
 289
 290         spin_lock_irqsave(&tracepoint_iter_lock, flags);
 291         trace_seq_init(&iter->seq);
 292         iter->ent = fbuffer->entry;
 293         event_call->event.funcs->trace(iter, 0, event);
 294         trace_seq_putc(&iter->seq, 0);
 295         printk("%s", iter->seq.buffer);
 296
 297         spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
 298 }
 299
 300 void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
 301 {
 302         if (tracepoint_printk)
 303                 output_printk(fbuffer);
 304
 305         event_trigger_unlock_commit(fbuffer->trace_file, fbuffer->buffer,
 306                                     fbuffer->event, fbuffer->entry,
 307                                     fbuffer->flags, fbuffer->pc);
 308 }
 309 EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
 310
 311 int trace_event_reg(struct trace_event_call *call,
 312                     enum trace_reg type, void *data)
 313 {
 314         struct trace_event_file *file = data;
 315
 316         WARN_ON(!(call->flags & TRACE_EVENT_FL_TRACEPOINT));
 317         switch (type) {
 318         case TRACE_REG_REGISTER:
 319                 return tracepoint_probe_register(call->tp,
 320                                                  call->class->probe,
 321                                                  file);
 322         case TRACE_REG_UNREGISTER:
 323                 tracepoint_probe_unregister(call->tp,
 324                                             call->class->probe,
 325                                             file);
 326                 return 0;
 327
 328 #ifdef CONFIG_PERF_EVENTS
 329         case TRACE_REG_PERF_REGISTER:
 330                 return tracepoint_probe_register(call->tp,
 331                                                  call->class->perf_probe,
 332                                                  call);
 333         case TRACE_REG_PERF_UNREGISTER:
 334                 tracepoint_probe_unregister(call->tp,
 335                                             call->class->perf_probe,
 336                                             call);
 337                 return 0;
 338         case TRACE_REG_PERF_OPEN:
 339         case TRACE_REG_PERF_CLOSE:
 340         case TRACE_REG_PERF_ADD:
 341         case TRACE_REG_PERF_DEL:
 342                 return 0;
 343 #endif
 344         }
 345         return 0;
 346 }
 347 EXPORT_SYMBOL_GPL(trace_event_reg);
 348
 349 void trace_event_enable_cmd_record(bool enable)
 350 {
 351         struct trace_event_file *file;
 352         struct trace_array *tr;
 353
 354         mutex_lock(&event_mutex);
 355         do_for_each_event_file(tr, file) {
 356
 357                 if (!(file->flags & EVENT_FILE_FL_ENABLED))
 358                         continue;
 359
 360                 if (enable) {
 361                         tracing_start_cmdline_record();
 362                         set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
 363                 } else {
 364                         tracing_stop_cmdline_record();
 365                         clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
 366                 }
 367         } while_for_each_event_file();
 368         mutex_unlock(&event_mutex);
 369 }
 370
 371 static int __ftrace_event_enable_disable(struct trace_event_file *file,
 372                                          int enable, int soft_disable)
 373 {
 374         struct trace_event_call *call = file->event_call;
 375         struct trace_array *tr = file->tr;
 376         int ret = 0;
 377         int disable;
 378
 379         switch (enable) {
 380         case 0:
 381                 /*
 382                  * When soft_disable is set and enable is cleared, the sm_ref
 383                  * reference counter is decremented. If it reaches 0, we want
 384                  * to clear the SOFT_DISABLED flag but leave the event in the
 385                  * state that it was. That is, if the event was enabled and
 386                  * SOFT_DISABLED isn't set, then do nothing. But if SOFT_DISABLED
 387                  * is set we do not want the event to be enabled before we
 388                  * clear the bit.
 389                  *
 390                  * When soft_disable is not set but the SOFT_MODE flag is,
 391                  * we do nothing. Do not disable the tracepoint, otherwise
 392                  * "soft enable"s (clearing the SOFT_DISABLED bit) wont work.
 393                  */
 394                 if (soft_disable) {
 395                         if (atomic_dec_return(&file->sm_ref) > 0)
 396                                 break;
 397                         disable = file->flags & EVENT_FILE_FL_SOFT_DISABLED;
 398                         clear_bit(EVENT_FILE_FL_SOFT_MODE_BIT, &file->flags);
 399                 } else
 400                         disable = !(file->flags & EVENT_FILE_FL_SOFT_MODE);
 401
 402                 if (disable && (file->flags & EVENT_FILE_FL_ENABLED)) {
 403                         clear_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
 404                         if (file->flags & EVENT_FILE_FL_RECORDED_CMD) {
 405                                 tracing_stop_cmdline_record();
 406                                 clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
 407                         }
 408                         call->class->reg(call, TRACE_REG_UNREGISTER, file);
 409                 }
 410                 /* If in SOFT_MODE, just set the SOFT_DISABLE_BIT, else clear it */
 411                 if (file->flags & EVENT_FILE_FL_SOFT_MODE)
 412                         set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
 413                 else
 414                         clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
 415                 break;
 416         case 1:
 417                 /*
 418                  * When soft_disable is set and enable is set, we want to
 419                  * register the tracepoint for the event, but leave the event
 420                  * as is. That means, if the event was already enabled, we do
 421                  * nothing (but set SOFT_MODE). If the event is disabled, we
 422                  * set SOFT_DISABLED before enabling the event tracepoint, so
 423                  * it still seems to be disabled.
 424                  */
 425                 if (!soft_disable)
 426                         clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
 427                 else {
 428                         if (atomic_inc_return(&file->sm_ref) > 1)
 429                                 break;
 430                         set_bit(EVENT_FILE_FL_SOFT_MODE_BIT, &file->flags);
 431                 }
 432
 433                 if (!(file->flags & EVENT_FILE_FL_ENABLED)) {
 434
 435                         /* Keep the event disabled, when going to SOFT_MODE. */
 436                         if (soft_disable)
 437                                 set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
 438
 439                         if (tr->trace_flags & TRACE_ITER_RECORD_CMD) {
 440                                 tracing_start_cmdline_record();
 441                                 set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
 442                         }
 443                         ret = call->class->reg(call, TRACE_REG_REGISTER, file);
 444                         if (ret) {
 445                                 tracing_stop_cmdline_record();
 446                                 pr_info("event trace: Could not enable event "
 447                                         "%s\n", trace_event_name(call));
 448                                 break;
 449                         }
 450                         set_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
 451
 452                         /* WAS_ENABLED gets set but never cleared. */
 453                         call->flags |= TRACE_EVENT_FL_WAS_ENABLED;
 454                 }
 455                 break;
 456         }
 457
 458         return ret;
 459 }
 460
 461 int trace_event_enable_disable(struct trace_event_file *file,
 462                                int enable, int soft_disable)
 463 {
 464         return __ftrace_event_enable_disable(file, enable, soft_disable);
 465 }
 466
 467 static int ftrace_event_enable_disable(struct trace_event_file *file,
 468                                        int enable)
 469 {
 470         return __ftrace_event_enable_disable(file, enable, 0);
 471 }
 472
 473 static void ftrace_clear_events(struct trace_array *tr)
 474 {
 475         struct trace_event_file *file;
 476
 477         mutex_lock(&event_mutex);
 478         list_for_each_entry(file, &tr->events, list) {
 479                 ftrace_event_enable_disable(file, 0);
 480         }
 481         mutex_unlock(&event_mutex);
 482 }
 483
 484 static int cmp_pid(const void *key, const void *elt)
 485 {
 486         const pid_t *search_pid = key;
 487         const pid_t *pid = elt;
 488
 489         if (*search_pid == *pid)
 490                 return 0;
 491         if (*search_pid < *pid)
 492                 return -1;
 493         return 1;
 494 }
 495
 496 static bool
 497 check_ignore_pid(struct trace_pid_list *filtered_pids, struct task_struct *task)
 498 {
 499         pid_t search_pid;
 500         pid_t *pid;
 501
 502         /*
 503          * Return false, because if filtered_pids does not exist,
 504          * all pids are good to trace.
 505          */
 506         if (!filtered_pids)
 507                 return false;
 508
 509         search_pid = task->pid;
 510
 511         pid = bsearch(&search_pid, filtered_pids->pids,
 512                       filtered_pids->nr_pids, sizeof(pid_t),
 513                       cmp_pid);
 514         if (!pid)
 515                 return true;
 516
 517         return false;
 518 }
 519
 520 static void
 521 event_filter_pid_sched_switch_probe_pre(void *data, bool preempt,
 522                     struct task_struct *prev, struct task_struct *next)
 523 {
 524         struct trace_array *tr = data;
 525         struct trace_pid_list *pid_list;
 526
 527         pid_list = rcu_dereference_sched(tr->filtered_pids);
 528
 529         this_cpu_write(tr->trace_buffer.data->ignore_pid,
 530                        check_ignore_pid(pid_list, prev) &&
 531                        check_ignore_pid(pid_list, next));
 532 }
 533
 534 static void
 535 event_filter_pid_sched_switch_probe_post(void *data, bool preempt,
 536                     struct task_struct *prev, struct task_struct *next)
 537 {
 538         struct trace_array *tr = data;
 539         struct trace_pid_list *pid_list;
 540
 541         pid_list = rcu_dereference_sched(tr->filtered_pids);
 542
 543         this_cpu_write(tr->trace_buffer.data->ignore_pid,
 544                        check_ignore_pid(pid_list, next));
 545 }
 546
 547 static void
 548 event_filter_pid_sched_wakeup_probe_pre(void *data, struct task_struct *task)
 549 {
 550         struct trace_array *tr = data;
 551         struct trace_pid_list *pid_list;
 552
 553         /* Nothing to do if we are already tracing */
 554         if (!this_cpu_read(tr->trace_buffer.data->ignore_pid))
 555                 return;
 556
 557         pid_list = rcu_dereference_sched(tr->filtered_pids);
 558
 559         this_cpu_write(tr->trace_buffer.data->ignore_pid,
 560                        check_ignore_pid(pid_list, task));
 561 }
 562
 563 static void
 564 event_filter_pid_sched_wakeup_probe_post(void *data, struct task_struct *task)
 565 {
 566         struct trace_array *tr = data;
 567         struct trace_pid_list *pid_list;
 568
 569         /* Nothing to do if we are not tracing */
 570         if (this_cpu_read(tr->trace_buffer.data->ignore_pid))
 571                 return;
 572
 573         pid_list = rcu_dereference_sched(tr->filtered_pids);
 574
 575         /* Set tracing if current is enabled */
 576         this_cpu_write(tr->trace_buffer.data->ignore_pid,
 577                        check_ignore_pid(pid_list, current));
 578 }
 579
 580 static void __ftrace_clear_event_pids(struct trace_array *tr)
 581 {
 582         struct trace_pid_list *pid_list;
 583         struct trace_event_file *file;
 584         int cpu;
 585
 586         pid_list = rcu_dereference_protected(tr->filtered_pids,
 587                                              lockdep_is_held(&event_mutex));
 588         if (!pid_list)
 589                 return;
 590
 591         unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_pre, tr);
 592         unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_post, tr);
 593
 594         unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre, tr);
 595         unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_post, tr);
 596
 597         unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre, tr);
 598         unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post, tr);
 599
 600         unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_pre, tr);
 601         unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_post, tr);
 602
 603         list_for_each_entry(file, &tr->events, list) {
 604                 clear_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
 605         }
 606
 607         for_each_possible_cpu(cpu)
 608                 per_cpu_ptr(tr->trace_buffer.data, cpu)->ignore_pid = false;
 609
 610         rcu_assign_pointer(tr->filtered_pids, NULL);
 611
 612         /* Wait till all users are no longer using pid filtering */
 613         synchronize_sched();
 614
 615         free_pages((unsigned long)pid_list->pids, pid_list->order);
 616         kfree(pid_list);
 617 }
 618
 619 static void ftrace_clear_event_pids(struct trace_array *tr)
 620 {
 621         mutex_lock(&event_mutex);
 622         __ftrace_clear_event_pids(tr);
 623         mutex_unlock(&event_mutex);
 624 }
 625
 626 static void __put_system(struct event_subsystem *system)
 627 {
 628         struct event_filter *filter = system->filter;
 629
 630         WARN_ON_ONCE(system_refcount(system) == 0);
 631         if (system_refcount_dec(system))
 632                 return;
 633
 634         list_del(&system->list);
 635
 636         if (filter) {
 637                 kfree(filter->filter_string);
 638                 kfree(filter);
 639         }
 640         kfree_const(system->name);
 641         kfree(system);
 642 }
 643
 644 static void __get_system(struct event_subsystem *system)
 645 {
 646         WARN_ON_ONCE(system_refcount(system) == 0);
 647         system_refcount_inc(system);
 648 }
 649
 650 static void __get_system_dir(struct trace_subsystem_dir *dir)
 651 {
 652         WARN_ON_ONCE(dir->ref_count == 0);
 653         dir->ref_count++;
 654         __get_system(dir->subsystem);
 655 }
 656
 657 static void __put_system_dir(struct trace_subsystem_dir *dir)
 658 {
 659         WARN_ON_ONCE(dir->ref_count == 0);
 660         /* If the subsystem is about to be freed, the dir must be too */
 661         WARN_ON_ONCE(system_refcount(dir->subsystem) == 1 && dir->ref_count != 1);
 662
 663         __put_system(dir->subsystem);
 664         if (!--dir->ref_count)
 665                 kfree(dir);
 666 }
 667
 668 static void put_system(struct trace_subsystem_dir *dir)
 669 {
 670         mutex_lock(&event_mutex);
 671         __put_system_dir(dir);
 672         mutex_unlock(&event_mutex);
 673 }
 674
 675 static void remove_subsystem(struct trace_subsystem_dir *dir)
 676 {
 677         if (!dir)
 678                 return;
 679
 680         if (!--dir->nr_events) {
 681                 tracefs_remove_recursive(dir->entry);
 682                 list_del(&dir->list);
 683                 __put_system_dir(dir);
 684         }
 685 }
 686
 687 static void remove_event_file_dir(struct trace_event_file *file)
 688 {
 689         struct dentry *dir = file->dir;
 690         struct dentry *child;
 691
 692         if (dir) {
 693                 spin_lock(&dir->d_lock);        /* probably unneeded */
 694                 list_for_each_entry(child, &dir->d_subdirs, d_child) {
 695                         if (d_really_is_positive(child))        /* probably unneeded */
 696                                 d_inode(child)->i_private = NULL;
 697                 }
 698                 spin_unlock(&dir->d_lock);
 699
 700                 tracefs_remove_recursive(dir);
 701         }
 702
 703         list_del(&file->list);
 704         remove_subsystem(file->system);
 705         free_event_filter(file->filter);
 706         kmem_cache_free(file_cachep, file);
 707 }
 708
 709 /*
 710  * __ftrace_set_clr_event(NULL, NULL, NULL, set) will set/unset all events.
 711  */
 712 static int
 713 __ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match,
 714                               const char *sub, const char *event, int set)
 715 {
 716         struct trace_event_file *file;
 717         struct trace_event_call *call;
 718         const char *name;
 719         int ret = -EINVAL;
 720
 721         list_for_each_entry(file, &tr->events, list) {
 722
 723                 call = file->event_call;
 724                 name = trace_event_name(call);
 725
 726                 if (!name || !call->class || !call->class->reg)
 727                         continue;
 728
 729                 if (call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
 730                         continue;
 731
 732                 if (match &&
 733                     strcmp(match, name) != 0 &&
 734                     strcmp(match, call->class->system) != 0)
 735                         continue;
 736
 737                 if (sub && strcmp(sub, call->class->system) != 0)
 738                         continue;
 739
 740                 if (event && strcmp(event, name) != 0)
 741                         continue;
 742
 743                 ftrace_event_enable_disable(file, set);
 744
 745                 ret = 0;
 746         }
 747
 748         return ret;
 749 }
 750
 751 static int __ftrace_set_clr_event(struct trace_array *tr, const char *match,
 752                                   const char *sub, const char *event, int set)
 753 {
 754         int ret;
 755
 756         mutex_lock(&event_mutex);
 757         ret = __ftrace_set_clr_event_nolock(tr, match, sub, event, set);
 758         mutex_unlock(&event_mutex);
 759
 760         return ret;
 761 }
 762
 763 static int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set)
 764 {
 765         char *event = NULL, *sub = NULL, *match;
 766         int ret;
 767
 768         /*
 769          * The buf format can be <subsystem>:<event-name>
 770          *  *:<event-name> means any event by that name.
 771          *  :<event-name> is the same.
 772          *
 773          *  <subsystem>:* means all events in that subsystem
 774          *  <subsystem>: means the same.
 775          *
 776          *  <name> (no ':') means all events in a subsystem with
 777          *  the name <name> or any event that matches <name>
 778          */
 779
 780         match = strsep(&buf, ":");
 781         if (buf) {
 782                 sub = match;
 783                 event = buf;
 784                 match = NULL;
 785
 786                 if (!strlen(sub) || strcmp(sub, "*") == 0)
 787                         sub = NULL;
 788                 if (!strlen(event) || strcmp(event, "*") == 0)
 789                         event = NULL;
 790         }
 791
 792         ret = __ftrace_set_clr_event(tr, match, sub, event, set);
 793
 794         /* Put back the colon to allow this to be called again */
 795         if (buf)
 796                 *(buf - 1) = ':';
 797
 798         return ret;
 799 }
 800
 801 /**
 802  * trace_set_clr_event - enable or disable an event
 803  * @system: system name to match (NULL for any system)
 804  * @event: event name to match (NULL for all events, within system)
 805  * @set: 1 to enable, 0 to disable
 806  *
 807  * This is a way for other parts of the kernel to enable or disable
 808  * event recording.
 809  *
 810  * Returns 0 on success, -EINVAL if the parameters do not match any
 811  * registered events.
 812  */
 813 int trace_set_clr_event(const char *system, const char *event, int set)
 814 {
 815         struct trace_array *tr = top_trace_array();
 816
 817         if (!tr)
 818                 return -ENODEV;
 819
 820         return __ftrace_set_clr_event(tr, NULL, system, event, set);
 821 }
 822 EXPORT_SYMBOL_GPL(trace_set_clr_event);
 823
 824 /* 128 should be much more than enough */
 825 #define EVENT_BUF_SIZE          127
 826
 827 static ssize_t
 828 ftrace_event_write(struct file *file, const char __user *ubuf,
 829                    size_t cnt, loff_t *ppos)
 830 {
 831         struct trace_parser parser;
 832         struct seq_file *m = file->private_data;
 833         struct trace_array *tr = m->private;
 834         ssize_t read, ret;
 835
 836         if (!cnt)
 837                 return 0;
 838
 839         ret = tracing_update_buffers();
 840         if (ret < 0)
 841                 return ret;
 842
 843         if (trace_parser_get_init(&parser, EVENT_BUF_SIZE + 1))
 844                 return -ENOMEM;
 845
 846         read = trace_get_user(&parser, ubuf, cnt, ppos);
 847
 848         if (read >= 0 && trace_parser_loaded((&parser))) {
 849                 int set = 1;
 850
 851                 if (*parser.buffer == '!')
 852                         set = 0;
 853
 854                 parser.buffer[parser.idx] = 0;
 855
 856                 ret = ftrace_set_clr_event(tr, parser.buffer + !set, set);
 857                 if (ret)
 858                         goto out_put;
 859         }
 860
 861         ret = read;
 862
 863  out_put:
 864         trace_parser_put(&parser);
 865
 866         return ret;
 867 }
 868
 869 static void *
 870 t_next(struct seq_file *m, void *v, loff_t *pos)
 871 {
 872         struct trace_event_file *file = v;
 873         struct trace_event_call *call;
 874         struct trace_array *tr = m->private;
 875
 876         (*pos)++;
 877
 878         list_for_each_entry_continue(file, &tr->events, list) {
 879                 call = file->event_call;
 880                 /*
 881                  * The ftrace subsystem is for showing formats only.
 882                  * They can not be enabled or disabled via the event files.
 883                  */
 884                 if (call->class && call->class->reg &&
 885                     !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
 886                         return file;
 887         }
 888
 889         return NULL;
 890 }
 891
 892 static void *t_start(struct seq_file *m, loff_t *pos)
 893 {
 894         struct trace_event_file *file;
 895         struct trace_array *tr = m->private;
 896         loff_t l;
 897
 898         mutex_lock(&event_mutex);
 899
 900         file = list_entry(&tr->events, struct trace_event_file, list);
 901         for (l = 0; l <= *pos; ) {
 902                 file = t_next(m, file, &l);
 903                 if (!file)
 904                         break;
 905         }
 906         return file;
 907 }
 908
 909 static void *
 910 s_next(struct seq_file *m, void *v, loff_t *pos)
 911 {
 912         struct trace_event_file *file = v;
 913         struct trace_array *tr = m->private;
 914
 915         (*pos)++;
 916
 917         list_for_each_entry_continue(file, &tr->events, list) {
 918                 if (file->flags & EVENT_FILE_FL_ENABLED)
 919                         return file;
 920         }
 921
 922         return NULL;
 923 }
 924
 925 static void *s_start(struct seq_file *m, loff_t *pos)
 926 {
 927         struct trace_event_file *file;
 928         struct trace_array *tr = m->private;
 929         loff_t l;
 930
 931         mutex_lock(&event_mutex);
 932
 933         file = list_entry(&tr->events, struct trace_event_file, list);
 934         for (l = 0; l <= *pos; ) {
 935                 file = s_next(m, file, &l);
 936                 if (!file)
 937                         break;
 938         }
 939         return file;
 940 }
 941
 942 static int t_show(struct seq_file *m, void *v)
 943 {
 944         struct trace_event_file *file = v;
 945         struct trace_event_call *call = file->event_call;
 946
 947         if (strcmp(call->class->system, TRACE_SYSTEM) != 0)
 948                 seq_printf(m, "%s:", call->class->system);
 949         seq_printf(m, "%s\n", trace_event_name(call));
 950
 951         return 0;
 952 }
 953
 954 static void t_stop(struct seq_file *m, void *p)
 955 {
 956         mutex_unlock(&event_mutex);
 957 }
 958
 959 static void *p_start(struct seq_file *m, loff_t *pos)
 960         __acquires(RCU)
 961 {
 962         struct trace_pid_list *pid_list;
 963         struct trace_array *tr = m->private;
 964
 965         /*
 966          * Grab the mutex, to keep calls to p_next() having the same
 967          * tr->filtered_pids as p_start() has.
 968          * If we just passed the tr->filtered_pids around, then RCU would
 969          * have been enough, but doing that makes things more complex.
 970          */
 971         mutex_lock(&event_mutex);
 972         rcu_read_lock_sched();
 973
 974         pid_list = rcu_dereference_sched(tr->filtered_pids);
 975
 976         if (!pid_list || *pos >= pid_list->nr_pids)
 977                 return NULL;
 978
 979         return (void *)&pid_list->pids[*pos];
 980 }
 981
 982 static void p_stop(struct seq_file *m, void *p)
 983         __releases(RCU)
 984 {
 985         rcu_read_unlock_sched();
 986         mutex_unlock(&event_mutex);
 987 }
 988
 989 static void *
 990 p_next(struct seq_file *m, void *v, loff_t *pos)
 991 {
 992         struct trace_array *tr = m->private;
 993         struct trace_pid_list *pid_list = rcu_dereference_sched(tr->filtered_pids);
 994
 995         (*pos)++;
 996
 997         if (*pos >= pid_list->nr_pids)
 998                 return NULL;
 999
1000         return (void *)&pid_list->pids[*pos];
1001 }
1002
1003 static int p_show(struct seq_file *m, void *v)
1004 {
1005         pid_t *pid = v;
1006
1007         seq_printf(m, "%d\n", *pid);
1008         return 0;
1009 }
1010
1011 static ssize_t
1012 event_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1013                   loff_t *ppos)
1014 {
1015         struct trace_event_file *file;
1016         unsigned long flags;
1017         char buf[4] = "0";
1018
1019         mutex_lock(&event_mutex);
1020         file = event_file_data(filp);
1021         if (likely(file))
1022                 flags = file->flags;
1023         mutex_unlock(&event_mutex);
1024
1025         if (!file)
1026                 return -ENODEV;
1027
1028         if (flags & EVENT_FILE_FL_ENABLED &&
1029             !(flags & EVENT_FILE_FL_SOFT_DISABLED))
1030                 strcpy(buf, "1");
1031
1032         if (flags & EVENT_FILE_FL_SOFT_DISABLED ||
1033             flags & EVENT_FILE_FL_SOFT_MODE)
1034                 strcat(buf, "*");
1035
1036         strcat(buf, "\n");
1037
1038         return simple_read_from_buffer(ubuf, cnt, ppos, buf, strlen(buf));
1039 }
1040
1041 static ssize_t
1042 event_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1043                    loff_t *ppos)
1044 {
1045         struct trace_event_file *file;
1046         unsigned long val;
1047         int ret;
1048
1049         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1050         if (ret)
1051                 return ret;
1052
1053         ret = tracing_update_buffers();
1054         if (ret < 0)
1055                 return ret;
1056
1057         switch (val) {
1058         case 0:
1059         case 1:
1060                 ret = -ENODEV;
1061                 mutex_lock(&event_mutex);
1062                 file = event_file_data(filp);
1063                 if (likely(file))
1064                         ret = ftrace_event_enable_disable(file, val);
1065                 mutex_unlock(&event_mutex);
1066                 break;
1067
1068         default:
1069                 return -EINVAL;
1070         }
1071
1072         *ppos += cnt;
1073
1074         return ret ? ret : cnt;
1075 }
1076
1077 static ssize_t
1078 system_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1079                    loff_t *ppos)
1080 {
1081         const char set_to_char[4] = { '?', '0', '1', 'X' };
1082         struct trace_subsystem_dir *dir = filp->private_data;
1083         struct event_subsystem *system = dir->subsystem;
1084         struct trace_event_call *call;
1085         struct trace_event_file *file;
1086         struct trace_array *tr = dir->tr;
1087         char buf[2];
1088         int set = 0;
1089         int ret;
1090
1091         mutex_lock(&event_mutex);
1092         list_for_each_entry(file, &tr->events, list) {
1093                 call = file->event_call;
1094                 if (!trace_event_name(call) || !call->class || !call->class->reg)
1095                         continue;
1096
1097                 if (system && strcmp(call->class->system, system->name) != 0)
1098                         continue;
1099
1100                 /*
1101                  * We need to find out if all the events are set
1102                  * or if all events or cleared, or if we have
1103                  * a mixture.
1104                  */
1105                 set |= (1 << !!(file->flags & EVENT_FILE_FL_ENABLED));
1106
1107                 /*
1108                  * If we have a mixture, no need to look further.
1109                  */
1110                 if (set == 3)
1111                         break;
1112         }
1113         mutex_unlock(&event_mutex);
1114
1115         buf[0] = set_to_char[set];
1116         buf[1] = '\n';
1117
1118         ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
1119
1120         return ret;
1121 }
1122
1123 static ssize_t
1124 system_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1125                     loff_t *ppos)
1126 {
1127         struct trace_subsystem_dir *dir = filp->private_data;
1128         struct event_subsystem *system = dir->subsystem;
1129         const char *name = NULL;
1130         unsigned long val;
1131         ssize_t ret;
1132
1133         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1134         if (ret)
1135                 return ret;
1136
1137         ret = tracing_update_buffers();
1138         if (ret < 0)
1139                 return ret;
1140
1141         if (val != 0 && val != 1)
1142                 return -EINVAL;
1143
1144         /*
1145          * Opening of "enable" adds a ref count to system,
1146          * so the name is safe to use.
1147          */
1148         if (system)
1149                 name = system->name;
1150
1151         ret = __ftrace_set_clr_event(dir->tr, NULL, name, NULL, val);
1152         if (ret)
1153                 goto out;
1154
1155         ret = cnt;
1156
1157 out:
1158         *ppos += cnt;
1159
1160         return ret;
1161 }
1162
1163 enum {
1164         FORMAT_HEADER           = 1,
1165         FORMAT_FIELD_SEPERATOR  = 2,
1166         FORMAT_PRINTFMT         = 3,
1167 };
1168
1169 static void *f_next(struct seq_file *m, void *v, loff_t *pos)
1170 {
1171         struct trace_event_call *call = event_file_data(m->private);
1172         struct list_head *common_head = &ftrace_common_fields;
1173         struct list_head *head = trace_get_fields(call);
1174         struct list_head *node = v;
1175
1176         (*pos)++;
1177
1178         switch ((unsigned long)v) {
1179         case FORMAT_HEADER:
1180                 node = common_head;
1181                 break;
1182
1183         case FORMAT_FIELD_SEPERATOR:
1184                 node = head;
1185                 break;
1186
1187         case FORMAT_PRINTFMT:
1188                 /* all done */
1189                 return NULL;
1190         }
1191
1192         node = node->prev;
1193         if (node == common_head)
1194                 return (void *)FORMAT_FIELD_SEPERATOR;
1195         else if (node == head)
1196                 return (void *)FORMAT_PRINTFMT;
1197         else
1198                 return node;
1199 }
1200
1201 static int f_show(struct seq_file *m, void *v)
1202 {
1203         struct trace_event_call *call = event_file_data(m->private);
1204         struct ftrace_event_field *field;
1205         const char *array_descriptor;
1206
1207         switch ((unsigned long)v) {
1208         case FORMAT_HEADER:
1209                 seq_printf(m, "name: %s\n", trace_event_name(call));
1210                 seq_printf(m, "ID: %d\n", call->event.type);
1211                 seq_puts(m, "format:\n");
1212                 return 0;
1213
1214         case FORMAT_FIELD_SEPERATOR:
1215                 seq_putc(m, '\n');
1216                 return 0;
1217
1218         case FORMAT_PRINTFMT:
1219                 seq_printf(m, "\nprint fmt: %s\n",
1220                            call->print_fmt);
1221                 return 0;
1222         }
1223
1224         field = list_entry(v, struct ftrace_event_field, link);
1225         /*
1226          * Smartly shows the array type(except dynamic array).
1227          * Normal:
1228          *      field:TYPE VAR
1229          * If TYPE := TYPE[LEN], it is shown:
1230          *      field:TYPE VAR[LEN]
1231          */
1232         array_descriptor = strchr(field->type, '[');
1233
1234         if (!strncmp(field->type, "__data_loc", 10))
1235                 array_descriptor = NULL;
1236
1237         if (!array_descriptor)
1238                 seq_printf(m, "\tfield:%s %s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1239                            field->type, field->name, field->offset,
1240                            field->size, !!field->is_signed);
1241         else
1242                 seq_printf(m, "\tfield:%.*s %s%s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1243                            (int)(array_descriptor - field->type),
1244                            field->type, field->name,
1245                            array_descriptor, field->offset,
1246                            field->size, !!field->is_signed);
1247
1248         return 0;
1249 }
1250
1251 static void *f_start(struct seq_file *m, loff_t *pos)
1252 {
1253         void *p = (void *)FORMAT_HEADER;
1254         loff_t l = 0;
1255
1256         /* ->stop() is called even if ->start() fails */
1257         mutex_lock(&event_mutex);
1258         if (!event_file_data(m->private))
1259                 return ERR_PTR(-ENODEV);
1260
1261         while (l < *pos && p)
1262                 p = f_next(m, p, &l);
1263
1264         return p;
1265 }
1266
1267 static void f_stop(struct seq_file *m, void *p)
1268 {
1269         mutex_unlock(&event_mutex);
1270 }
1271
1272 static const struct seq_operations trace_format_seq_ops = {
1273         .start          = f_start,
1274         .next           = f_next,
1275         .stop           = f_stop,
1276         .show           = f_show,
1277 };
1278
1279 static int trace_format_open(struct inode *inode, struct file *file)
1280 {
1281         struct seq_file *m;
1282         int ret;
1283
1284         ret = seq_open(file, &trace_format_seq_ops);
1285         if (ret < 0)
1286                 return ret;
1287
1288         m = file->private_data;
1289         m->private = file;
1290
1291         return 0;
1292 }
1293
1294 static ssize_t
1295 event_id_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
1296 {
1297         int id = (long)event_file_data(filp);
1298         char buf[32];
1299         int len;
1300
1301         if (*ppos)
1302                 return 0;
1303
1304         if (unlikely(!id))
1305                 return -ENODEV;
1306
1307         len = sprintf(buf, "%d\n", id);
1308
1309         return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
1310 }
1311
1312 static ssize_t
1313 event_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
1314                   loff_t *ppos)
1315 {
1316         struct trace_event_file *file;
1317         struct trace_seq *s;
1318         int r = -ENODEV;
1319
1320         if (*ppos)
1321                 return 0;
1322
1323         s = kmalloc(sizeof(*s), GFP_KERNEL);
1324
1325         if (!s)
1326                 return -ENOMEM;
1327
1328         trace_seq_init(s);
1329
1330         mutex_lock(&event_mutex);
1331         file = event_file_data(filp);
1332         if (file)
1333                 print_event_filter(file, s);
1334         mutex_unlock(&event_mutex);
1335
1336         if (file)
1337                 r = simple_read_from_buffer(ubuf, cnt, ppos,
1338                                             s->buffer, trace_seq_used(s));
1339
1340         kfree(s);
1341
1342         return r;
1343 }
1344
1345 static ssize_t
1346 event_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
1347                    loff_t *ppos)
1348 {
1349         struct trace_event_file *file;
1350         char *buf;
1351         int err = -ENODEV;
1352
1353         if (cnt >= PAGE_SIZE)
1354                 return -EINVAL;
1355
1356         buf = (char *)__get_free_page(GFP_TEMPORARY);
1357         if (!buf)
1358                 return -ENOMEM;
1359
1360         if (copy_from_user(buf, ubuf, cnt)) {
1361                 free_page((unsigned long) buf);
1362                 return -EFAULT;
1363         }
1364         buf[cnt] = '\0';
1365
1366         mutex_lock(&event_mutex);
1367         file = event_file_data(filp);
1368         if (file)
1369                 err = apply_event_filter(file, buf);
1370         mutex_unlock(&event_mutex);
1371
1372         free_page((unsigned long) buf);
1373         if (err < 0)
1374                 return err;
1375
1376         *ppos += cnt;
1377
1378         return cnt;
1379 }
1380
1381 static LIST_HEAD(event_subsystems);
1382
1383 static int subsystem_open(struct inode *inode, struct file *filp)
1384 {
1385         struct event_subsystem *system = NULL;
1386         struct trace_subsystem_dir *dir = NULL; /* Initialize for gcc */
1387         struct trace_array *tr;
1388         int ret;
1389
1390         if (tracing_is_disabled())
1391                 return -ENODEV;
1392
1393         /* Make sure the system still exists */
1394         mutex_lock(&trace_types_lock);
1395         mutex_lock(&event_mutex);
1396         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1397                 list_for_each_entry(dir, &tr->systems, list) {
1398                         if (dir == inode->i_private) {
1399                                 /* Don't open systems with no events */
1400                                 if (dir->nr_events) {
1401                                         __get_system_dir(dir);
1402                                         system = dir->subsystem;
1403                                 }
1404                                 goto exit_loop;
1405                         }
1406                 }
1407         }
1408  exit_loop:
1409         mutex_unlock(&event_mutex);
1410         mutex_unlock(&trace_types_lock);
1411
1412         if (!system)
1413                 return -ENODEV;
1414
1415         /* Some versions of gcc think dir can be uninitialized here */
1416         WARN_ON(!dir);
1417
1418         /* Still need to increment the ref count of the system */
1419         if (trace_array_get(tr) < 0) {
1420                 put_system(dir);
1421                 return -ENODEV;
1422         }
1423
1424         ret = tracing_open_generic(inode, filp);
1425         if (ret < 0) {
1426                 trace_array_put(tr);
1427                 put_system(dir);
1428         }
1429
1430         return ret;
1431 }
1432
1433 static int system_tr_open(struct inode *inode, struct file *filp)
1434 {
1435         struct trace_subsystem_dir *dir;
1436         struct trace_array *tr = inode->i_private;
1437         int ret;
1438
1439         if (tracing_is_disabled())
1440                 return -ENODEV;
1441
1442         if (trace_array_get(tr) < 0)
1443                 return -ENODEV;
1444
1445         /* Make a temporary dir that has no system but points to tr */
1446         dir = kzalloc(sizeof(*dir), GFP_KERNEL);
1447         if (!dir) {
1448                 trace_array_put(tr);
1449                 return -ENOMEM;
1450         }
1451
1452         dir->tr = tr;
1453
1454         ret = tracing_open_generic(inode, filp);
1455         if (ret < 0) {
1456                 trace_array_put(tr);
1457                 kfree(dir);
1458                 return ret;
1459         }
1460
1461         filp->private_data = dir;
1462
1463         return 0;
1464 }
1465
1466 static int subsystem_release(struct inode *inode, struct file *file)
1467 {
1468         struct trace_subsystem_dir *dir = file->private_data;
1469
1470         trace_array_put(dir->tr);
1471
1472         /*
1473          * If dir->subsystem is NULL, then this is a temporary
1474          * descriptor that was made for a trace_array to enable
1475          * all subsystems.
1476          */
1477         if (dir->subsystem)
1478                 put_system(dir);
1479         else
1480                 kfree(dir);
1481
1482         return 0;
1483 }
1484
1485 static ssize_t
1486 subsystem_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
1487                       loff_t *ppos)
1488 {
1489         struct trace_subsystem_dir *dir = filp->private_data;
1490         struct event_subsystem *system = dir->subsystem;
1491         struct trace_seq *s;
1492         int r;
1493
1494         if (*ppos)
1495                 return 0;
1496
1497         s = kmalloc(sizeof(*s), GFP_KERNEL);
1498         if (!s)
1499                 return -ENOMEM;
1500
1501         trace_seq_init(s);
1502
1503         print_subsystem_event_filter(system, s);
1504         r = simple_read_from_buffer(ubuf, cnt, ppos,
1505                                     s->buffer, trace_seq_used(s));
1506
1507         kfree(s);
1508
1509         return r;
1510 }
1511
1512 static ssize_t
1513 subsystem_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
1514                        loff_t *ppos)
1515 {
1516         struct trace_subsystem_dir *dir = filp->private_data;
1517         char *buf;
1518         int err;
1519
1520         if (cnt >= PAGE_SIZE)
1521                 return -EINVAL;
1522
1523         buf = (char *)__get_free_page(GFP_TEMPORARY);
1524         if (!buf)
1525                 return -ENOMEM;
1526
1527         if (copy_from_user(buf, ubuf, cnt)) {
1528                 free_page((unsigned long) buf);
1529                 return -EFAULT;
1530         }
1531         buf[cnt] = '\0';
1532
1533         err = apply_subsystem_event_filter(dir, buf);
1534         free_page((unsigned long) buf);
1535         if (err < 0)
1536                 return err;
1537
1538         *ppos += cnt;
1539
1540         return cnt;
1541 }
1542
1543 static ssize_t
1544 show_header(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
1545 {
1546         int (*func)(struct trace_seq *s) = filp->private_data;
1547         struct trace_seq *s;
1548         int r;
1549
1550         if (*ppos)
1551                 return 0;
1552
1553         s = kmalloc(sizeof(*s), GFP_KERNEL);
1554         if (!s)
1555                 return -ENOMEM;
1556
1557         trace_seq_init(s);
1558
1559         func(s);
1560         r = simple_read_from_buffer(ubuf, cnt, ppos,
1561                                     s->buffer, trace_seq_used(s));
1562
1563         kfree(s);
1564
1565         return r;
1566 }
1567
1568 static int max_pids(struct trace_pid_list *pid_list)
1569 {
1570         return (PAGE_SIZE << pid_list->order) / sizeof(pid_t);
1571 }
1572
1573 static void ignore_task_cpu(void *data)
1574 {
1575         struct trace_array *tr = data;
1576         struct trace_pid_list *pid_list;
1577
1578         /*
1579          * This function is called by on_each_cpu() while the
1580          * event_mutex is held.
1581          */
1582         pid_list = rcu_dereference_protected(tr->filtered_pids,
1583                                              mutex_is_locked(&event_mutex));
1584
1585         this_cpu_write(tr->trace_buffer.data->ignore_pid,
1586                        check_ignore_pid(pid_list, current));
1587 }
1588
1589 static ssize_t
1590 ftrace_event_pid_write(struct file *filp, const char __user *ubuf,
1591                        size_t cnt, loff_t *ppos)
1592 {
1593         struct seq_file *m = filp->private_data;
1594         struct trace_array *tr = m->private;
1595         struct trace_pid_list *filtered_pids = NULL;
1596         struct trace_pid_list *pid_list = NULL;
1597         struct trace_event_file *file;
1598         struct trace_parser parser;
1599         unsigned long val;
1600         loff_t this_pos;
1601         ssize_t read = 0;
1602         ssize_t ret = 0;
1603         pid_t pid;
1604         int i;
1605
1606         if (!cnt)
1607                 return 0;
1608
1609         ret = tracing_update_buffers();
1610         if (ret < 0)
1611                 return ret;
1612
1613         if (trace_parser_get_init(&parser, EVENT_BUF_SIZE + 1))
1614                 return -ENOMEM;
1615
1616         mutex_lock(&event_mutex);
1617         /*
1618          * Load as many pids into the array before doing a
1619          * swap from the tr->filtered_pids to the new list.
1620          */
1621         while (cnt > 0) {
1622
1623                 this_pos = 0;
1624
1625                 ret = trace_get_user(&parser, ubuf, cnt, &this_pos);
1626                 if (ret < 0 || !trace_parser_loaded(&parser))
1627                         break;
1628
1629                 read += ret;
1630                 ubuf += ret;
1631                 cnt -= ret;
1632
1633                 parser.buffer[parser.idx] = 0;
1634
1635                 ret = -EINVAL;
1636                 if (kstrtoul(parser.buffer, 0, &val))
1637                         break;
1638                 if (val > INT_MAX)
1639                         break;
1640
1641                 pid = (pid_t)val;
1642
1643                 ret = -ENOMEM;
1644                 if (!pid_list) {
1645                         pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
1646                         if (!pid_list)
1647                                 break;
1648
1649                         filtered_pids = rcu_dereference_protected(tr->filtered_pids,
1650                                                         lockdep_is_held(&event_mutex));
1651                         if (filtered_pids)
1652                                 pid_list->order = filtered_pids->order;
1653                         else
1654                                 pid_list->order = 0;
1655
1656                         pid_list->pids = (void *)__get_free_pages(GFP_KERNEL,
1657                                                                   pid_list->order);
1658                         if (!pid_list->pids)
1659                                 break;
1660
1661                         if (filtered_pids) {
1662                                 pid_list->nr_pids = filtered_pids->nr_pids;
1663                                 memcpy(pid_list->pids, filtered_pids->pids,
1664                                        pid_list->nr_pids * sizeof(pid_t));
1665                         } else
1666                                 pid_list->nr_pids = 0;
1667                 }
1668
1669                 if (pid_list->nr_pids >= max_pids(pid_list)) {
1670                         pid_t *pid_page;
1671
1672                         pid_page = (void *)__get_free_pages(GFP_KERNEL,
1673                                                             pid_list->order + 1);
1674                         if (!pid_page)
1675                                 break;
1676                         memcpy(pid_page, pid_list->pids,
1677                                pid_list->nr_pids * sizeof(pid_t));
1678                         free_pages((unsigned long)pid_list->pids, pid_list->order);
1679
1680                         pid_list->order++;
1681                         pid_list->pids = pid_page;
1682                 }
1683
1684                 pid_list->pids[pid_list->nr_pids++] = pid;
1685                 trace_parser_clear(&parser);
1686                 ret = 0;
1687         }
1688         trace_parser_put(&parser);
1689
1690         if (ret < 0) {
1691                 if (pid_list)
1692                         free_pages((unsigned long)pid_list->pids, pid_list->order);
1693                 kfree(pid_list);
1694                 mutex_unlock(&event_mutex);
1695                 return ret;
1696         }
1697
1698         if (!pid_list) {
1699                 mutex_unlock(&event_mutex);
1700                 return ret;
1701         }
1702
1703         sort(pid_list->pids, pid_list->nr_pids, sizeof(pid_t), cmp_pid, NULL);
1704
1705         /* Remove duplicates */
1706         for (i = 1; i < pid_list->nr_pids; i++) {
1707                 int start = i;
1708
1709                 while (i < pid_list->nr_pids &&
1710                        pid_list->pids[i - 1] == pid_list->pids[i])
1711                         i++;
1712
1713                 if (start != i) {
1714                         if (i < pid_list->nr_pids) {
1715                                 memmove(&pid_list->pids[start], &pid_list->pids[i],
1716                                         (pid_list->nr_pids - i) * sizeof(pid_t));
1717                                 pid_list->nr_pids -= i - start;
1718                                 i = start;
1719                         } else
1720                                 pid_list->nr_pids = start;
1721                 }
1722         }
1723
1724         rcu_assign_pointer(tr->filtered_pids, pid_list);
1725
1726         list_for_each_entry(file, &tr->events, list) {
1727                 set_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
1728         }
1729
1730         if (filtered_pids) {
1731                 synchronize_sched();
1732
1733                 free_pages((unsigned long)filtered_pids->pids, filtered_pids->order);
1734                 kfree(filtered_pids);
1735         } else {
1736                 /*
1737                  * Register a probe that is called before all other probes
1738                  * to set ignore_pid if next or prev do not match.
1739                  * Register a probe this is called after all other probes
1740                  * to only keep ignore_pid set if next pid matches.
1741                  */
1742                 register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_pre,
1743                                                  tr, INT_MAX);
1744                 register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_post,
1745                                                  tr, 0);
1746
1747                 register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre,
1748                                                  tr, INT_MAX);
1749                 register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_post,
1750                                                  tr, 0);
1751
1752                 register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre,
1753                                                      tr, INT_MAX);
1754                 register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post,
1755                                                      tr, 0);
1756
1757                 register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_pre,
1758                                                  tr, INT_MAX);
1759                 register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_post,
1760                                                  tr, 0);
1761         }
1762
1763         /*
1764          * Ignoring of pids is done at task switch. But we have to
1765          * check for those tasks that are currently running.
1766          * Always do this in case a pid was appended or removed.
1767          */
1768         on_each_cpu(ignore_task_cpu, tr, 1);
1769
1770         mutex_unlock(&event_mutex);
1771
1772         ret = read;
1773         *ppos += read;
1774
1775         return ret;
1776 }
1777
1778 static int ftrace_event_avail_open(struct inode *inode, struct file *file);
1779 static int ftrace_event_set_open(struct inode *inode, struct file *file);
1780 static int ftrace_event_set_pid_open(struct inode *inode, struct file *file);
1781 static int ftrace_event_release(struct inode *inode, struct file *file);
1782
1783 static const struct seq_operations show_event_seq_ops = {
1784         .start = t_start,
1785         .next = t_next,
1786         .show = t_show,
1787         .stop = t_stop,
1788 };
1789
1790 static const struct seq_operations show_set_event_seq_ops = {
1791         .start = s_start,
1792         .next = s_next,
1793         .show = t_show,
1794         .stop = t_stop,
1795 };
1796
1797 static const struct seq_operations show_set_pid_seq_ops = {
1798         .start = p_start,
1799         .next = p_next,
1800         .show = p_show,
1801         .stop = p_stop,
1802 };
1803
1804 static const struct file_operations ftrace_avail_fops = {
1805         .open = ftrace_event_avail_open,
1806         .read = seq_read,
1807         .llseek = seq_lseek,
1808         .release = seq_release,
1809 };
1810
1811 static const struct file_operations ftrace_set_event_fops = {
1812         .open = ftrace_event_set_open,
1813         .read = seq_read,
1814         .write = ftrace_event_write,
1815         .llseek = seq_lseek,
1816         .release = ftrace_event_release,
1817 };
1818
1819 static const struct file_operations ftrace_set_event_pid_fops = {
1820         .open = ftrace_event_set_pid_open,
1821         .read = seq_read,
1822         .write = ftrace_event_pid_write,
1823         .llseek = seq_lseek,
1824         .release = ftrace_event_release,
1825 };
1826
1827 static const struct file_operations ftrace_enable_fops = {
1828         .open = tracing_open_generic,
1829         .read = event_enable_read,
1830         .write = event_enable_write,
1831         .llseek = default_llseek,
1832 };
1833
1834 static const struct file_operations ftrace_event_format_fops = {
1835         .open = trace_format_open,
1836         .read = seq_read,
1837         .llseek = seq_lseek,
1838         .release = seq_release,
1839 };
1840
1841 static const struct file_operations ftrace_event_id_fops = {
1842         .read = event_id_read,
1843         .llseek = default_llseek,
1844 };
1845
1846 static const struct file_operations ftrace_event_filter_fops = {
1847         .open = tracing_open_generic,
1848         .read = event_filter_read,
1849         .write = event_filter_write,
1850         .llseek = default_llseek,
1851 };
1852
1853 static const struct file_operations ftrace_subsystem_filter_fops = {
1854         .open = subsystem_open,
1855         .read = subsystem_filter_read,
1856         .write = subsystem_filter_write,
1857         .llseek = default_llseek,
1858         .release = subsystem_release,
1859 };
1860
1861 static const struct file_operations ftrace_system_enable_fops = {
1862         .open = subsystem_open,
1863         .read = system_enable_read,
1864         .write = system_enable_write,
1865         .llseek = default_llseek,
1866         .release = subsystem_release,
1867 };
1868
1869 static const struct file_operations ftrace_tr_enable_fops = {
1870         .open = system_tr_open,
1871         .read = system_enable_read,
1872         .write = system_enable_write,
1873         .llseek = default_llseek,
1874         .release = subsystem_release,
1875 };
1876
1877 static const struct file_operations ftrace_show_header_fops = {
1878         .open = tracing_open_generic,
1879         .read = show_header,
1880         .llseek = default_llseek,
1881 };
1882
1883 static int
1884 ftrace_event_open(struct inode *inode, struct file *file,
1885                   const struct seq_operations *seq_ops)
1886 {
1887         struct seq_file *m;
1888         int ret;
1889
1890         ret = seq_open(file, seq_ops);
1891         if (ret < 0)
1892                 return ret;
1893         m = file->private_data;
1894         /* copy tr over to seq ops */
1895         m->private = inode->i_private;
1896
1897         return ret;
1898 }
1899
1900 static int ftrace_event_release(struct inode *inode, struct file *file)
1901 {
1902         struct trace_array *tr = inode->i_private;
1903
1904         trace_array_put(tr);
1905
1906         return seq_release(inode, file);
1907 }
1908
1909 static int
1910 ftrace_event_avail_open(struct inode *inode, struct file *file)
1911 {
1912         const struct seq_operations *seq_ops = &show_event_seq_ops;
1913
1914         return ftrace_event_open(inode, file, seq_ops);
1915 }
1916
1917 static int
1918 ftrace_event_set_open(struct inode *inode, struct file *file)
1919 {
1920         const struct seq_operations *seq_ops = &show_set_event_seq_ops;
1921         struct trace_array *tr = inode->i_private;
1922         int ret;
1923
1924         if (trace_array_get(tr) < 0)
1925                 return -ENODEV;
1926
1927         if ((file->f_mode & FMODE_WRITE) &&
1928             (file->f_flags & O_TRUNC))
1929                 ftrace_clear_events(tr);
1930
1931         ret = ftrace_event_open(inode, file, seq_ops);
1932         if (ret < 0)
1933                 trace_array_put(tr);
1934         return ret;
1935 }
1936
1937 static int
1938 ftrace_event_set_pid_open(struct inode *inode, struct file *file)
1939 {
1940         const struct seq_operations *seq_ops = &show_set_pid_seq_ops;
1941         struct trace_array *tr = inode->i_private;
1942         int ret;
1943
1944         if (trace_array_get(tr) < 0)
1945                 return -ENODEV;
1946
1947         if ((file->f_mode & FMODE_WRITE) &&
1948             (file->f_flags & O_TRUNC))
1949                 ftrace_clear_event_pids(tr);
1950
1951         ret = ftrace_event_open(inode, file, seq_ops);
1952         if (ret < 0)
1953                 trace_array_put(tr);
1954         return ret;
1955 }
1956
1957 static struct event_subsystem *
1958 create_new_subsystem(const char *name)
1959 {
1960         struct event_subsystem *system;
1961
1962         /* need to create new entry */
1963         system = kmalloc(sizeof(*system), GFP_KERNEL);
1964         if (!system)
1965                 return NULL;
1966
1967         system->ref_count = 1;
1968
1969         /* Only allocate if dynamic (kprobes and modules) */
1970         system->name = kstrdup_const(name, GFP_KERNEL);
1971         if (!system->name)
1972                 goto out_free;
1973
1974         system->filter = NULL;
1975
1976         system->filter = kzalloc(sizeof(struct event_filter), GFP_KERNEL);
1977         if (!system->filter)
1978                 goto out_free;
1979
1980         list_add(&system->list, &event_subsystems);
1981
1982         return system;
1983
1984  out_free:
1985         kfree_const(system->name);
1986         kfree(system);
1987         return NULL;
1988 }
1989
1990 static struct dentry *
1991 event_subsystem_dir(struct trace_array *tr, const char *name,
1992                     struct trace_event_file *file, struct dentry *parent)
1993 {
1994         struct trace_subsystem_dir *dir;
1995         struct event_subsystem *system;
1996         struct dentry *entry;
1997
1998         /* First see if we did not already create this dir */
1999         list_for_each_entry(dir, &tr->systems, list) {
2000                 system = dir->subsystem;
2001                 if (strcmp(system->name, name) == 0) {
2002                         dir->nr_events++;
2003                         file->system = dir;
2004                         return dir->entry;
2005                 }
2006         }
2007
2008         /* Now see if the system itself exists. */
2009         list_for_each_entry(system, &event_subsystems, list) {
2010                 if (strcmp(system->name, name) == 0)
2011                         break;
2012         }
2013         /* Reset system variable when not found */
2014         if (&system->list == &event_subsystems)
2015                 system = NULL;
2016
2017         dir = kmalloc(sizeof(*dir), GFP_KERNEL);
2018         if (!dir)
2019                 goto out_fail;
2020
2021         if (!system) {
2022                 system = create_new_subsystem(name);
2023                 if (!system)
2024                         goto out_free;
2025         } else
2026                 __get_system(system);
2027
2028         dir->entry = tracefs_create_dir(name, parent);
2029         if (!dir->entry) {
2030                 pr_warn("Failed to create system directory %s\n", name);
2031                 __put_system(system);
2032                 goto out_free;
2033         }
2034
2035         dir->tr = tr;
2036         dir->ref_count = 1;
2037         dir->nr_events = 1;
2038         dir->subsystem = system;
2039         file->system = dir;
2040
2041         entry = tracefs_create_file("filter", 0644, dir->entry, dir,
2042                                     &ftrace_subsystem_filter_fops);
2043         if (!entry) {
2044                 kfree(system->filter);
2045                 system->filter = NULL;
2046                 pr_warn("Could not create tracefs '%s/filter' entry\n", name);
2047         }
2048
2049         trace_create_file("enable", 0644, dir->entry, dir,
2050                           &ftrace_system_enable_fops);
2051
2052         list_add(&dir->list, &tr->systems);
2053
2054         return dir->entry;
2055
2056  out_free:
2057         kfree(dir);
2058  out_fail:
2059         /* Only print this message if failed on memory allocation */
2060         if (!dir || !system)
2061                 pr_warn("No memory to create event subsystem %s\n", name);
2062         return NULL;
2063 }
2064
2065 static int
2066 event_create_dir(struct dentry *parent, struct trace_event_file *file)
2067 {
2068         struct trace_event_call *call = file->event_call;
2069         struct trace_array *tr = file->tr;
2070         struct list_head *head;
2071         struct dentry *d_events;
2072         const char *name;
2073         int ret;
2074
2075         /*
2076          * If the trace point header did not define TRACE_SYSTEM
2077          * then the system would be called "TRACE_SYSTEM".
2078          */
2079         if (strcmp(call->class->system, TRACE_SYSTEM) != 0) {
2080                 d_events = event_subsystem_dir(tr, call->class->system, file, parent);
2081                 if (!d_events)
2082                         return -ENOMEM;
2083         } else
2084                 d_events = parent;
2085
2086         name = trace_event_name(call);
2087         file->dir = tracefs_create_dir(name, d_events);
2088         if (!file->dir) {
2089                 pr_warn("Could not create tracefs '%s' directory\n", name);
2090                 return -1;
2091         }
2092
2093         if (call->class->reg && !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
2094                 trace_create_file("enable", 0644, file->dir, file,
2095                                   &ftrace_enable_fops);
2096
2097 #ifdef CONFIG_PERF_EVENTS
2098         if (call->event.type && call->class->reg)
2099                 trace_create_file("id", 0444, file->dir,
2100                                   (void *)(long)call->event.type,
2101                                   &ftrace_event_id_fops);
2102 #endif
2103
2104         /*
2105          * Other events may have the same class. Only update
2106          * the fields if they are not already defined.
2107          */
2108         head = trace_get_fields(call);
2109         if (list_empty(head)) {
2110                 ret = call->class->define_fields(call);
2111                 if (ret < 0) {
2112                         pr_warn("Could not initialize trace point events/%s\n",
2113                                 name);
2114                         return -1;
2115                 }
2116         }
2117         trace_create_file("filter", 0644, file->dir, file,
2118                           &ftrace_event_filter_fops);
2119
2120         /*
2121          * Only event directories that can be enabled should have
2122          * triggers.
2123          */
2124         if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
2125                 trace_create_file("trigger", 0644, file->dir, file,
2126                                   &event_trigger_fops);
2127
2128         trace_create_file("format", 0444, file->dir, call,
2129                           &ftrace_event_format_fops);
2130
2131         return 0;
2132 }
2133
2134 static void remove_event_from_tracers(struct trace_event_call *call)
2135 {
2136         struct trace_event_file *file;
2137         struct trace_array *tr;
2138
2139         do_for_each_event_file_safe(tr, file) {
2140                 if (file->event_call != call)
2141                         continue;
2142
2143                 remove_event_file_dir(file);
2144                 /*
2145                  * The do_for_each_event_file_safe() is
2146                  * a double loop. After finding the call for this
2147                  * trace_array, we use break to jump to the next
2148                  * trace_array.
2149                  */
2150                 break;
2151         } while_for_each_event_file();
2152 }
2153
2154 static void event_remove(struct trace_event_call *call)
2155 {
2156         struct trace_array *tr;
2157         struct trace_event_file *file;
2158
2159         do_for_each_event_file(tr, file) {
2160                 if (file->event_call != call)
2161                         continue;
2162                 ftrace_event_enable_disable(file, 0);
2163                 /*
2164                  * The do_for_each_event_file() is
2165                  * a double loop. After finding the call for this
2166                  * trace_array, we use break to jump to the next
2167                  * trace_array.
2168                  */
2169                 break;
2170         } while_for_each_event_file();
2171
2172         if (call->event.funcs)
2173                 __unregister_trace_event(&call->event);
2174         remove_event_from_tracers(call);
2175         list_del(&call->list);
2176 }
2177
2178 static int event_init(struct trace_event_call *call)
2179 {
2180         int ret = 0;
2181         const char *name;
2182
2183         name = trace_event_name(call);
2184         if (WARN_ON(!name))
2185                 return -EINVAL;
2186
2187         if (call->class->raw_init) {
2188                 ret = call->class->raw_init(call);
2189                 if (ret < 0 && ret != -ENOSYS)
2190                         pr_warn("Could not initialize trace events/%s\n", name);
2191         }
2192
2193         return ret;
2194 }
2195
2196 static int
2197 __register_event(struct trace_event_call *call, struct module *mod)
2198 {
2199         int ret;
2200
2201         ret = event_init(call);
2202         if (ret < 0)
2203                 return ret;
2204
2205         list_add(&call->list, &ftrace_events);
2206         call->mod = mod;
2207
2208         return 0;
2209 }
2210
2211 static char *enum_replace(char *ptr, struct trace_enum_map *map, int len)
2212 {
2213         int rlen;
2214         int elen;
2215
2216         /* Find the length of the enum value as a string */
2217         elen = snprintf(ptr, 0, "%ld", map->enum_value);
2218         /* Make sure there's enough room to replace the string with the value */
2219         if (len < elen)
2220                 return NULL;
2221
2222         snprintf(ptr, elen + 1, "%ld", map->enum_value);
2223
2224         /* Get the rest of the string of ptr */
2225         rlen = strlen(ptr + len);
2226         memmove(ptr + elen, ptr + len, rlen);
2227         /* Make sure we end the new string */
2228         ptr[elen + rlen] = 0;
2229
2230         return ptr + elen;
2231 }
2232
2233 static void update_event_printk(struct trace_event_call *call,
2234                                 struct trace_enum_map *map)
2235 {
2236         char *ptr;
2237         int quote = 0;
2238         int len = strlen(map->enum_string);
2239
2240         for (ptr = call->print_fmt; *ptr; ptr++) {
2241                 if (*ptr == '\\') {
2242                         ptr++;
2243                         /* paranoid */
2244                         if (!*ptr)
2245                                 break;
2246                         continue;
2247                 }
2248                 if (*ptr == '"') {
2249                         quote ^= 1;
2250                         continue;
2251                 }
2252                 if (quote)
2253                         continue;
2254                 if (isdigit(*ptr)) {
2255                         /* skip numbers */
2256                         do {
2257                                 ptr++;
2258                                 /* Check for alpha chars like ULL */
2259                         } while (isalnum(*ptr));
2260                         if (!*ptr)
2261                                 break;
2262                         /*
2263                          * A number must have some kind of delimiter after
2264                          * it, and we can ignore that too.
2265                          */
2266                         continue;
2267                 }
2268                 if (isalpha(*ptr) || *ptr == '_') {
2269                         if (strncmp(map->enum_string, ptr, len) == 0 &&
2270                             !isalnum(ptr[len]) && ptr[len] != '_') {
2271                                 ptr = enum_replace(ptr, map, len);
2272                                 /* Hmm, enum string smaller than value */
2273                                 if (WARN_ON_ONCE(!ptr))
2274                                         return;
2275                                 /*
2276                                  * No need to decrement here, as enum_replace()
2277                                  * returns the pointer to the character passed
2278                                  * the enum, and two enums can not be placed
2279                                  * back to back without something in between.
2280                                  * We can skip that something in between.
2281                                  */
2282                                 continue;
2283                         }
2284                 skip_more:
2285                         do {
2286                                 ptr++;
2287                         } while (isalnum(*ptr) || *ptr == '_');
2288                         if (!*ptr)
2289                                 break;
2290                         /*
2291                          * If what comes after this variable is a '.' or
2292                          * '->' then we can continue to ignore that string.
2293                          */
2294                         if (*ptr == '.' || (ptr[0] == '-' && ptr[1] == '>')) {
2295                                 ptr += *ptr == '.' ? 1 : 2;
2296                                 if (!*ptr)
2297                                         break;
2298                                 goto skip_more;
2299                         }
2300                         /*
2301                          * Once again, we can skip the delimiter that came
2302                          * after the string.
2303                          */
2304                         continue;
2305                 }
2306         }
2307 }
2308
2309 void trace_event_enum_update(struct trace_enum_map **map, int len)
2310 {
2311         struct trace_event_call *call, *p;
2312         const char *last_system = NULL;
2313         int last_i;
2314         int i;
2315
2316         down_write(&trace_event_sem);
2317         list_for_each_entry_safe(call, p, &ftrace_events, list) {
2318                 /* events are usually grouped together with systems */
2319                 if (!last_system || call->class->system != last_system) {
2320                         last_i = 0;
2321                         last_system = call->class->system;
2322                 }
2323
2324                 for (i = last_i; i < len; i++) {
2325                         if (call->class->system == map[i]->system) {
2326                                 /* Save the first system if need be */
2327                                 if (!last_i)
2328                                         last_i = i;
2329                                 update_event_printk(call, map[i]);
2330                         }
2331                 }
2332         }
2333         up_write(&trace_event_sem);
2334 }
2335
2336 static struct trace_event_file *
2337 trace_create_new_event(struct trace_event_call *call,
2338                        struct trace_array *tr)
2339 {
2340         struct trace_event_file *file;
2341
2342         file = kmem_cache_alloc(file_cachep, GFP_TRACE);
2343         if (!file)
2344                 return NULL;
2345
2346         file->event_call = call;
2347         file->tr = tr;
2348         atomic_set(&file->sm_ref, 0);
2349         atomic_set(&file->tm_ref, 0);
2350         INIT_LIST_HEAD(&file->triggers);
2351         list_add(&file->list, &tr->events);
2352
2353         return file;
2354 }
2355
2356 /* Add an event to a trace directory */
2357 static int
2358 __trace_add_new_event(struct trace_event_call *call, struct trace_array *tr)
2359 {
2360         struct trace_event_file *file;
2361
2362         file = trace_create_new_event(call, tr);
2363         if (!file)
2364                 return -ENOMEM;
2365
2366         return event_create_dir(tr->event_dir, file);
2367 }
2368
2369 /*
2370  * Just create a decriptor for early init. A descriptor is required
2371  * for enabling events at boot. We want to enable events before
2372  * the filesystem is initialized.
2373  */
2374 static __init int
2375 __trace_early_add_new_event(struct trace_event_call *call,
2376                             struct trace_array *tr)
2377 {
2378         struct trace_event_file *file;
2379
2380         file = trace_create_new_event(call, tr);
2381         if (!file)
2382                 return -ENOMEM;
2383
2384         return 0;
2385 }
2386
2387 struct ftrace_module_file_ops;
2388 static void __add_event_to_tracers(struct trace_event_call *call);
2389
2390 /* Add an additional event_call dynamically */
2391 int trace_add_event_call(struct trace_event_call *call)
2392 {
2393         int ret;
2394         mutex_lock(&trace_types_lock);
2395         mutex_lock(&event_mutex);
2396
2397         ret = __register_event(call, NULL);
2398         if (ret >= 0)
2399                 __add_event_to_tracers(call);
2400
2401         mutex_unlock(&event_mutex);
2402         mutex_unlock(&trace_types_lock);
2403         return ret;
2404 }
2405
2406 /*
2407  * Must be called under locking of trace_types_lock, event_mutex and
2408  * trace_event_sem.
2409  */
2410 static void __trace_remove_event_call(struct trace_event_call *call)
2411 {
2412         event_remove(call);
2413         trace_destroy_fields(call);
2414         free_event_filter(call->filter);
2415         call->filter = NULL;
2416 }
2417
2418 static int probe_remove_event_call(struct trace_event_call *call)
2419 {
2420         struct trace_array *tr;
2421         struct trace_event_file *file;
2422
2423 #ifdef CONFIG_PERF_EVENTS
2424         if (call->perf_refcount)
2425                 return -EBUSY;
2426 #endif
2427         do_for_each_event_file(tr, file) {
2428                 if (file->event_call != call)
2429                         continue;
2430                 /*
2431                  * We can't rely on ftrace_event_enable_disable(enable => 0)
2432                  * we are going to do, EVENT_FILE_FL_SOFT_MODE can suppress
2433                  * TRACE_REG_UNREGISTER.
2434                  */
2435                 if (file->flags & EVENT_FILE_FL_ENABLED)
2436                         return -EBUSY;
2437                 /*
2438                  * The do_for_each_event_file_safe() is
2439                  * a double loop. After finding the call for this
2440                  * trace_array, we use break to jump to the next
2441                  * trace_array.
2442                  */
2443                 break;
2444         } while_for_each_event_file();
2445
2446         __trace_remove_event_call(call);
2447
2448         return 0;
2449 }
2450
2451 /* Remove an event_call */
2452 int trace_remove_event_call(struct trace_event_call *call)
2453 {
2454         int ret;
2455
2456         mutex_lock(&trace_types_lock);
2457         mutex_lock(&event_mutex);
2458         down_write(&trace_event_sem);
2459         ret = probe_remove_event_call(call);
2460         up_write(&trace_event_sem);
2461         mutex_unlock(&event_mutex);
2462         mutex_unlock(&trace_types_lock);
2463
2464         return ret;
2465 }
2466
2467 #define for_each_event(event, start, end)                       \
2468         for (event = start;                                     \
2469              (unsigned long)event < (unsigned long)end;         \
2470              event++)
2471
2472 #ifdef CONFIG_MODULES
2473
2474 static void trace_module_add_events(struct module *mod)
2475 {
2476         struct trace_event_call **call, **start, **end;
2477
2478         if (!mod->num_trace_events)
2479                 return;
2480
2481         /* Don't add infrastructure for mods without tracepoints */
2482         if (trace_module_has_bad_taint(mod)) {
2483                 pr_err("%s: module has bad taint, not creating trace events\n",
2484                        mod->name);
2485                 return;
2486         }
2487
2488         start = mod->trace_events;
2489         end = mod->trace_events + mod->num_trace_events;
2490
2491         for_each_event(call, start, end) {
2492                 __register_event(*call, mod);
2493                 __add_event_to_tracers(*call);
2494         }
2495 }
2496
2497 static void trace_module_remove_events(struct module *mod)
2498 {
2499         struct trace_event_call *call, *p;
2500         bool clear_trace = false;
2501
2502         down_write(&trace_event_sem);
2503         list_for_each_entry_safe(call, p, &ftrace_events, list) {
2504                 if (call->mod == mod) {
2505                         if (call->flags & TRACE_EVENT_FL_WAS_ENABLED)
2506                                 clear_trace = true;
2507                         __trace_remove_event_call(call);
2508                 }
2509         }
2510         up_write(&trace_event_sem);
2511
2512         /*
2513          * It is safest to reset the ring buffer if the module being unloaded
2514          * registered any events that were used. The only worry is if
2515          * a new module gets loaded, and takes on the same id as the events
2516          * of this module. When printing out the buffer, traced events left
2517          * over from this module may be passed to the new module events and
2518          * unexpected results may occur.
2519          */
2520         if (clear_trace)
2521                 tracing_reset_all_online_cpus();
2522 }
2523
2524 static int trace_module_notify(struct notifier_block *self,
2525                                unsigned long val, void *data)
2526 {
2527         struct module *mod = data;
2528
2529         mutex_lock(&trace_types_lock);
2530         mutex_lock(&event_mutex);
2531         switch (val) {
2532         case MODULE_STATE_COMING:
2533                 trace_module_add_events(mod);
2534                 break;
2535         case MODULE_STATE_GOING:
2536                 trace_module_remove_events(mod);
2537                 break;
2538         }
2539         mutex_unlock(&event_mutex);
2540         mutex_unlock(&trace_types_lock);
2541
2542         return 0;
2543 }
2544
2545 static struct notifier_block trace_module_nb = {
2546         .notifier_call = trace_module_notify,
2547         .priority = 1, /* higher than trace.c module notify */
2548 };
2549 #endif /* CONFIG_MODULES */
2550
2551 /* Create a new event directory structure for a trace directory. */
2552 static void
2553 __trace_add_event_dirs(struct trace_array *tr)
2554 {
2555         struct trace_event_call *call;
2556         int ret;
2557
2558         list_for_each_entry(call, &ftrace_events, list) {
2559                 ret = __trace_add_new_event(call, tr);
2560                 if (ret < 0)
2561                         pr_warn("Could not create directory for event %s\n",
2562                                 trace_event_name(call));
2563         }
2564 }
2565
2566 struct trace_event_file *
2567 find_event_file(struct trace_array *tr, const char *system,  const char *event)
2568 {
2569         struct trace_event_file *file;
2570         struct trace_event_call *call;
2571         const char *name;
2572
2573         list_for_each_entry(file, &tr->events, list) {
2574
2575                 call = file->event_call;
2576                 name = trace_event_name(call);
2577
2578                 if (!name || !call->class || !call->class->reg)
2579                         continue;
2580
2581                 if (call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
2582                         continue;
2583
2584                 if (strcmp(event, name) == 0 &&
2585                     strcmp(system, call->class->system) == 0)
2586                         return file;
2587         }
2588         return NULL;
2589 }
2590
2591 #ifdef CONFIG_DYNAMIC_FTRACE
2592
2593 /* Avoid typos */
2594 #define ENABLE_EVENT_STR        "enable_event"
2595 #define DISABLE_EVENT_STR       "disable_event"
2596
2597 struct event_probe_data {
2598         struct trace_event_file *file;
2599         unsigned long                   count;
2600         int                             ref;
2601         bool                            enable;
2602 };
2603
2604 static void
2605 event_enable_probe(unsigned long ip, unsigned long parent_ip, void **_data)
2606 {
2607         struct event_probe_data **pdata = (struct event_probe_data **)_data;
2608         struct event_probe_data *data = *pdata;
2609
2610         if (!data)
2611                 return;
2612
2613         if (data->enable)
2614                 clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
2615         else
2616                 set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
2617 }
2618
2619 static void
2620 event_enable_count_probe(unsigned long ip, unsigned long parent_ip, void **_data)
2621 {
2622         struct event_probe_data **pdata = (struct event_probe_data **)_data;
2623         struct event_probe_data *data = *pdata;
2624
2625         if (!data)
2626                 return;
2627
2628         if (!data->count)
2629                 return;
2630
2631         /* Skip if the event is in a state we want to switch to */
2632         if (data->enable == !(data->file->flags & EVENT_FILE_FL_SOFT_DISABLED))
2633                 return;
2634
2635         if (data->count != -1)
2636                 (data->count)--;
2637
2638         event_enable_probe(ip, parent_ip, _data);
2639 }
2640
2641 static int
2642 event_enable_print(struct seq_file *m, unsigned long ip,
2643                       struct ftrace_probe_ops *ops, void *_data)
2644 {
2645         struct event_probe_data *data = _data;
2646
2647         seq_printf(m, "%ps:", (void *)ip);
2648
2649         seq_printf(m, "%s:%s:%s",
2650                    data->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR,
2651                    data->file->event_call->class->system,
2652                    trace_event_name(data->file->event_call));
2653
2654         if (data->count == -1)
2655                 seq_puts(m, ":unlimited\n");
2656         else
2657                 seq_printf(m, ":count=%ld\n", data->count);
2658
2659         return 0;
2660 }
2661
2662 static int
2663 event_enable_init(struct ftrace_probe_ops *ops, unsigned long ip,
2664                   void **_data)
2665 {
2666         struct event_probe_data **pdata = (struct event_probe_data **)_data;
2667         struct event_probe_data *data = *pdata;
2668
2669         data->ref++;
2670         return 0;
2671 }
2672
2673 static void
2674 event_enable_free(struct ftrace_probe_ops *ops, unsigned long ip,
2675                   void **_data)
2676 {
2677         struct event_probe_data **pdata = (struct event_probe_data **)_data;
2678         struct event_probe_data *data = *pdata;
2679
2680         if (WARN_ON_ONCE(data->ref <= 0))
2681                 return;
2682
2683         data->ref--;
2684         if (!data->ref) {
2685                 /* Remove the SOFT_MODE flag */
2686                 __ftrace_event_enable_disable(data->file, 0, 1);
2687                 module_put(data->file->event_call->mod);
2688                 kfree(data);
2689         }
2690         *pdata = NULL;
2691 }
2692
2693 static struct ftrace_probe_ops event_enable_probe_ops = {
2694         .func                   = event_enable_probe,
2695         .print                  = event_enable_print,
2696         .init                   = event_enable_init,
2697         .free                   = event_enable_free,
2698 };
2699
2700 static struct ftrace_probe_ops event_enable_count_probe_ops = {
2701         .func                   = event_enable_count_probe,
2702         .print                  = event_enable_print,
2703         .init                   = event_enable_init,
2704         .free                   = event_enable_free,
2705 };
2706
2707 static struct ftrace_probe_ops event_disable_probe_ops = {
2708         .func                   = event_enable_probe,
2709         .print                  = event_enable_print,
2710         .init                   = event_enable_init,
2711         .free                   = event_enable_free,
2712 };
2713
2714 static struct ftrace_probe_ops event_disable_count_probe_ops = {
2715         .func                   = event_enable_count_probe,
2716         .print                  = event_enable_print,
2717         .init                   = event_enable_init,
2718         .free                   = event_enable_free,
2719 };
2720
2721 static int
2722 event_enable_func(struct ftrace_hash *hash,
2723                   char *glob, char *cmd, char *param, int enabled)
2724 {
2725         struct trace_array *tr = top_trace_array();
2726         struct trace_event_file *file;
2727         struct ftrace_probe_ops *ops;
2728         struct event_probe_data *data;
2729         const char *system;
2730         const char *event;
2731         char *number;
2732         bool enable;
2733         int ret;
2734
2735         if (!tr)
2736                 return -ENODEV;
2737
2738         /* hash funcs only work with set_ftrace_filter */
2739         if (!enabled || !param)
2740                 return -EINVAL;
2741
2742         system = strsep(&param, ":");
2743         if (!param)
2744                 return -EINVAL;
2745
2746         event = strsep(&param, ":");
2747
2748         mutex_lock(&event_mutex);
2749
2750         ret = -EINVAL;
2751         file = find_event_file(tr, system, event);
2752         if (!file)
2753                 goto out;
2754
2755         enable = strcmp(cmd, ENABLE_EVENT_STR) == 0;
2756
2757         if (enable)
2758                 ops = param ? &event_enable_count_probe_ops : &event_enable_probe_ops;
2759         else
2760                 ops = param ? &event_disable_count_probe_ops : &event_disable_probe_ops;
2761
2762         if (glob[0] == '!') {
2763                 unregister_ftrace_function_probe_func(glob+1, ops);
2764                 ret = 0;
2765                 goto out;
2766         }
2767
2768         ret = -ENOMEM;
2769         data = kzalloc(sizeof(*data), GFP_KERNEL);
2770         if (!data)
2771                 goto out;
2772
2773         data->enable = enable;
2774         data->count = -1;
2775         data->file = file;
2776
2777         if (!param)
2778                 goto out_reg;
2779
2780         number = strsep(&param, ":");
2781
2782         ret = -EINVAL;
2783         if (!strlen(number))
2784                 goto out_free;
2785
2786         /*
2787          * We use the callback data field (which is a pointer)
2788          * as our counter.
2789          */
2790         ret = kstrtoul(number, 0, &data->count);
2791         if (ret)
2792                 goto out_free;
2793
2794  out_reg:
2795         /* Don't let event modules unload while probe registered */
2796         ret = try_module_get(file->event_call->mod);
2797         if (!ret) {
2798                 ret = -EBUSY;
2799                 goto out_free;
2800         }
2801
2802         ret = __ftrace_event_enable_disable(file, 1, 1);
2803         if (ret < 0)
2804                 goto out_put;
2805         ret = register_ftrace_function_probe(glob, ops, data);
2806         /*
2807          * The above returns on success the # of functions enabled,
2808          * but if it didn't find any functions it returns zero.
2809          * Consider no functions a failure too.
2810          */
2811         if (!ret) {
2812                 ret = -ENOENT;
2813                 goto out_disable;
2814         } else if (ret < 0)
2815                 goto out_disable;
2816         /* Just return zero, not the number of enabled functions */
2817         ret = 0;
2818  out:
2819         mutex_unlock(&event_mutex);
2820         return ret;
2821
2822  out_disable:
2823         __ftrace_event_enable_disable(file, 0, 1);
2824  out_put:
2825         module_put(file->event_call->mod);
2826  out_free:
2827         kfree(data);
2828         goto out;
2829 }
2830
2831 static struct ftrace_func_command event_enable_cmd = {
2832         .name                   = ENABLE_EVENT_STR,
2833         .func                   = event_enable_func,
2834 };
2835
2836 static struct ftrace_func_command event_disable_cmd = {
2837         .name                   = DISABLE_EVENT_STR,
2838         .func                   = event_enable_func,
2839 };
2840
2841 static __init int register_event_cmds(void)
2842 {
2843         int ret;
2844
2845         ret = register_ftrace_command(&event_enable_cmd);
2846         if (WARN_ON(ret < 0))
2847                 return ret;
2848         ret = register_ftrace_command(&event_disable_cmd);
2849         if (WARN_ON(ret < 0))
2850                 unregister_ftrace_command(&event_enable_cmd);
2851         return ret;
2852 }
2853 #else
2854 static inline int register_event_cmds(void) { return 0; }
2855 #endif /* CONFIG_DYNAMIC_FTRACE */
2856
2857 /*
2858  * The top level array has already had its trace_event_file
2859  * descriptors created in order to allow for early events to
2860  * be recorded. This function is called after the tracefs has been
2861  * initialized, and we now have to create the files associated
2862  * to the events.
2863  */
2864 static __init void
2865 __trace_early_add_event_dirs(struct trace_array *tr)
2866 {
2867         struct trace_event_file *file;
2868         int ret;
2869
2870
2871         list_for_each_entry(file, &tr->events, list) {
2872                 ret = event_create_dir(tr->event_dir, file);
2873                 if (ret < 0)
2874                         pr_warn("Could not create directory for event %s\n",
2875                                 trace_event_name(file->event_call));
2876         }
2877 }
2878
2879 /*
2880  * For early boot up, the top trace array requires to have
2881  * a list of events that can be enabled. This must be done before
2882  * the filesystem is set up in order to allow events to be traced
2883  * early.
2884  */
2885 static __init void
2886 __trace_early_add_events(struct trace_array *tr)
2887 {
2888         struct trace_event_call *call;
2889         int ret;
2890
2891         list_for_each_entry(call, &ftrace_events, list) {
2892                 /* Early boot up should not have any modules loaded */
2893                 if (WARN_ON_ONCE(call->mod))
2894                         continue;
2895
2896                 ret = __trace_early_add_new_event(call, tr);
2897                 if (ret < 0)
2898                         pr_warn("Could not create early event %s\n",
2899                                 trace_event_name(call));
2900         }
2901 }
2902
2903 /* Remove the event directory structure for a trace directory. */
2904 static void
2905 __trace_remove_event_dirs(struct trace_array *tr)
2906 {
2907         struct trace_event_file *file, *next;
2908
2909         list_for_each_entry_safe(file, next, &tr->events, list)
2910                 remove_event_file_dir(file);
2911 }
2912
2913 static void __add_event_to_tracers(struct trace_event_call *call)
2914 {
2915         struct trace_array *tr;
2916
2917         list_for_each_entry(tr, &ftrace_trace_arrays, list)
2918                 __trace_add_new_event(call, tr);
2919 }
2920
2921 extern struct trace_event_call *__start_ftrace_events[];
2922 extern struct trace_event_call *__stop_ftrace_events[];
2923
2924 static char bootup_event_buf[COMMAND_LINE_SIZE] __initdata;
2925
2926 static __init int setup_trace_event(char *str)
2927 {
2928         strlcpy(bootup_event_buf, str, COMMAND_LINE_SIZE);
2929         ring_buffer_expanded = true;
2930         tracing_selftest_disabled = true;
2931
2932         return 1;
2933 }
2934 __setup("trace_event=", setup_trace_event);
2935
2936 /* Expects to have event_mutex held when called */
2937 static int
2938 create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
2939 {
2940         struct dentry *d_events;
2941         struct dentry *entry;
2942
2943         entry = tracefs_create_file("set_event", 0644, parent,
2944                                     tr, &ftrace_set_event_fops);
2945         if (!entry) {
2946                 pr_warn("Could not create tracefs 'set_event' entry\n");
2947                 return -ENOMEM;
2948         }
2949
2950         d_events = tracefs_create_dir("events", parent);
2951         if (!d_events) {
2952                 pr_warn("Could not create tracefs 'events' directory\n");
2953                 return -ENOMEM;
2954         }
2955
2956         entry = tracefs_create_file("set_event_pid", 0644, parent,
2957                                     tr, &ftrace_set_event_pid_fops);
2958
2959         /* ring buffer internal formats */
2960         trace_create_file("header_page", 0444, d_events,
2961                           ring_buffer_print_page_header,
2962                           &ftrace_show_header_fops);
2963
2964         trace_create_file("header_event", 0444, d_events,
2965                           ring_buffer_print_entry_header,
2966                           &ftrace_show_header_fops);
2967
2968         trace_create_file("enable", 0644, d_events,
2969                           tr, &ftrace_tr_enable_fops);
2970
2971         tr->event_dir = d_events;
2972
2973         return 0;
2974 }
2975
2976 /**
2977  * event_trace_add_tracer - add a instance of a trace_array to events
2978  * @parent: The parent dentry to place the files/directories for events in
2979  * @tr: The trace array associated with these events
2980  *
2981  * When a new instance is created, it needs to set up its events
2982  * directory, as well as other files associated with events. It also
2983  * creates the event hierachry in the @parent/events directory.
2984  *
2985  * Returns 0 on success.
2986  */
2987 int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr)
2988 {
2989         int ret;
2990
2991         mutex_lock(&event_mutex);
2992
2993         ret = create_event_toplevel_files(parent, tr);
2994         if (ret)
2995                 goto out_unlock;
2996
2997         down_write(&trace_event_sem);
2998         __trace_add_event_dirs(tr);
2999         up_write(&trace_event_sem);
3000
3001  out_unlock:
3002         mutex_unlock(&event_mutex);
3003
3004         return ret;
3005 }
3006
3007 /*
3008  * The top trace array already had its file descriptors created.
3009  * Now the files themselves need to be created.
3010  */
3011 static __init int
3012 early_event_add_tracer(struct dentry *parent, struct trace_array *tr)
3013 {
3014         int ret;
3015
3016         mutex_lock(&event_mutex);
3017
3018         ret = create_event_toplevel_files(parent, tr);
3019         if (ret)
3020                 goto out_unlock;
3021
3022         down_write(&trace_event_sem);
3023         __trace_early_add_event_dirs(tr);
3024         up_write(&trace_event_sem);
3025
3026  out_unlock:
3027         mutex_unlock(&event_mutex);
3028
3029         return ret;
3030 }
3031
3032 int event_trace_del_tracer(struct trace_array *tr)
3033 {
3034         mutex_lock(&event_mutex);
3035
3036         /* Disable any event triggers and associated soft-disabled events */
3037         clear_event_triggers(tr);
3038
3039         /* Clear the pid list */
3040         __ftrace_clear_event_pids(tr);
3041
3042         /* Disable any running events */
3043         __ftrace_set_clr_event_nolock(tr, NULL, NULL, NULL, 0);
3044
3045         /* Access to events are within rcu_read_lock_sched() */
3046         synchronize_sched();
3047
3048         down_write(&trace_event_sem);
3049         __trace_remove_event_dirs(tr);
3050         tracefs_remove_recursive(tr->event_dir);
3051         up_write(&trace_event_sem);
3052
3053         tr->event_dir = NULL;
3054
3055         mutex_unlock(&event_mutex);
3056
3057         return 0;
3058 }
3059
3060 static __init int event_trace_memsetup(void)
3061 {
3062         field_cachep = KMEM_CACHE(ftrace_event_field, SLAB_PANIC);
3063         file_cachep = KMEM_CACHE(trace_event_file, SLAB_PANIC);
3064         return 0;
3065 }
3066
3067 static __init void
3068 early_enable_events(struct trace_array *tr, bool disable_first)
3069 {
3070         char *buf = bootup_event_buf;
3071         char *token;
3072         int ret;
3073
3074         while (true) {
3075                 token = strsep(&buf, ",");
3076
3077                 if (!token)
3078                         break;
3079
3080                 if (*token) {
3081                         /* Restarting syscalls requires that we stop them first */
3082                         if (disable_first)
3083                                 ftrace_set_clr_event(tr, token, 0);
3084
3085                         ret = ftrace_set_clr_event(tr, token, 1);
3086                         if (ret)
3087                                 pr_warn("Failed to enable trace event: %s\n", token);
3088                 }
3089
3090                 /* Put back the comma to allow this to be called again */
3091                 if (buf)
3092                         *(buf - 1) = ',';
3093         }
3094 }
3095
3096 static __init int event_trace_enable(void)
3097 {
3098         struct trace_array *tr = top_trace_array();
3099         struct trace_event_call **iter, *call;
3100         int ret;
3101
3102         if (!tr)
3103                 return -ENODEV;
3104
3105         for_each_event(iter, __start_ftrace_events, __stop_ftrace_events) {
3106
3107                 call = *iter;
3108                 ret = event_init(call);
3109                 if (!ret)
3110                         list_add(&call->list, &ftrace_events);
3111         }
3112
3113         /*
3114          * We need the top trace array to have a working set of trace
3115          * points at early init, before the debug files and directories
3116          * are created. Create the file entries now, and attach them
3117          * to the actual file dentries later.
3118          */
3119         __trace_early_add_events(tr);
3120
3121         early_enable_events(tr, false);
3122
3123         trace_printk_start_comm();
3124
3125         register_event_cmds();
3126
3127         register_trigger_cmds();
3128
3129         return 0;
3130 }
3131
3132 /*
3133  * event_trace_enable() is called from trace_event_init() first to
3134  * initialize events and perhaps start any events that are on the
3135  * command line. Unfortunately, there are some events that will not
3136  * start this early, like the system call tracepoints that need
3137  * to set the TIF_SYSCALL_TRACEPOINT flag of pid 1. But event_trace_enable()
3138  * is called before pid 1 starts, and this flag is never set, making
3139  * the syscall tracepoint never get reached, but the event is enabled
3140  * regardless (and not doing anything).
3141  */
3142 static __init int event_trace_enable_again(void)
3143 {
3144         struct trace_array *tr;
3145
3146         tr = top_trace_array();
3147         if (!tr)
3148                 return -ENODEV;
3149
3150         early_enable_events(tr, true);
3151
3152         return 0;
3153 }
3154
3155 early_initcall(event_trace_enable_again);
3156
3157 static __init int event_trace_init(void)
3158 {
3159         struct trace_array *tr;
3160         struct dentry *d_tracer;
3161         struct dentry *entry;
3162         int ret;
3163
3164         tr = top_trace_array();
3165         if (!tr)
3166                 return -ENODEV;
3167
3168         d_tracer = tracing_init_dentry();
3169         if (IS_ERR(d_tracer))
3170                 return 0;
3171
3172         entry = tracefs_create_file("available_events", 0444, d_tracer,
3173                                     tr, &ftrace_avail_fops);
3174         if (!entry)
3175                 pr_warn("Could not create tracefs 'available_events' entry\n");
3176
3177         if (trace_define_generic_fields())
3178                 pr_warn("tracing: Failed to allocated generic fields");
3179
3180         if (trace_define_common_fields())
3181                 pr_warn("tracing: Failed to allocate common fields");
3182
3183         ret = early_event_add_tracer(d_tracer, tr);
3184         if (ret)
3185                 return ret;
3186
3187 #ifdef CONFIG_MODULES
3188         ret = register_module_notifier(&trace_module_nb);
3189         if (ret)
3190                 pr_warn("Failed to register trace events module notifier\n");
3191 #endif
3192         return 0;
3193 }
3194
3195 void __init trace_event_init(void)
3196 {
3197         event_trace_memsetup();
3198         init_ftrace_syscalls();
3199         event_trace_enable();
3200 }
3201
3202 fs_initcall(event_trace_init);
3203
3204 #ifdef CONFIG_FTRACE_STARTUP_TEST
3205
3206 static DEFINE_SPINLOCK(test_spinlock);
3207 static DEFINE_SPINLOCK(test_spinlock_irq);
3208 static DEFINE_MUTEX(test_mutex);
3209
3210 static __init void test_work(struct work_struct *dummy)
3211 {
3212         spin_lock(&test_spinlock);
3213         spin_lock_irq(&test_spinlock_irq);
3214         udelay(1);
3215         spin_unlock_irq(&test_spinlock_irq);
3216         spin_unlock(&test_spinlock);
3217
3218         mutex_lock(&test_mutex);
3219         msleep(1);
3220         mutex_unlock(&test_mutex);
3221 }
3222
3223 static __init int event_test_thread(void *unused)
3224 {
3225         void *test_malloc;
3226
3227         test_malloc = kmalloc(1234, GFP_KERNEL);
3228         if (!test_malloc)
3229                 pr_info("failed to kmalloc\n");
3230
3231         schedule_on_each_cpu(test_work);
3232
3233         kfree(test_malloc);
3234
3235         set_current_state(TASK_INTERRUPTIBLE);
3236         while (!kthread_should_stop()) {
3237                 schedule();
3238                 set_current_state(TASK_INTERRUPTIBLE);
3239         }
3240         __set_current_state(TASK_RUNNING);
3241
3242         return 0;
3243 }
3244
3245 /*
3246  * Do various things that may trigger events.
3247  */
3248 static __init void event_test_stuff(void)
3249 {
3250         struct task_struct *test_thread;
3251
3252         test_thread = kthread_run(event_test_thread, NULL, "test-events");
3253         msleep(1);
3254         kthread_stop(test_thread);
3255 }
3256
3257 /*
3258  * For every trace event defined, we will test each trace point separately,
3259  * and then by groups, and finally all trace points.
3260  */
3261 static __init void event_trace_self_tests(void)
3262 {
3263         struct trace_subsystem_dir *dir;
3264         struct trace_event_file *file;
3265         struct trace_event_call *call;
3266         struct event_subsystem *system;
3267         struct trace_array *tr;
3268         int ret;
3269
3270         tr = top_trace_array();
3271         if (!tr)
3272                 return;
3273
3274         pr_info("Running tests on trace events:\n");
3275
3276         list_for_each_entry(file, &tr->events, list) {
3277
3278                 call = file->event_call;
3279
3280                 /* Only test those that have a probe */
3281                 if (!call->class || !call->class->probe)
3282                         continue;
3283
3284 /*
3285  * Testing syscall events here is pretty useless, but
3286  * we still do it if configured. But this is time consuming.
3287  * What we really need is a user thread to perform the
3288  * syscalls as we test.
3289  */
3290 #ifndef CONFIG_EVENT_TRACE_TEST_SYSCALLS
3291                 if (call->class->system &&
3292                     strcmp(call->class->system, "syscalls") == 0)
3293                         continue;
3294 #endif
3295
3296                 pr_info("Testing event %s: ", trace_event_name(call));
3297
3298                 /*
3299                  * If an event is already enabled, someone is using
3300                  * it and the self test should not be on.
3301                  */
3302                 if (file->flags & EVENT_FILE_FL_ENABLED) {
3303                         pr_warn("Enabled event during self test!\n");
3304                         WARN_ON_ONCE(1);
3305                         continue;
3306                 }
3307
3308                 ftrace_event_enable_disable(file, 1);
3309                 event_test_stuff();
3310                 ftrace_event_enable_disable(file, 0);
3311
3312                 pr_cont("OK\n");
3313         }
3314
3315         /* Now test at the sub system level */
3316
3317         pr_info("Running tests on trace event systems:\n");
3318
3319         list_for_each_entry(dir, &tr->systems, list) {
3320
3321                 system = dir->subsystem;
3322
3323                 /* the ftrace system is special, skip it */
3324                 if (strcmp(system->name, "ftrace") == 0)
3325                         continue;
3326
3327                 pr_info("Testing event system %s: ", system->name);
3328
3329                 ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 1);
3330                 if (WARN_ON_ONCE(ret)) {
3331                         pr_warn("error enabling system %s\n",
3332                                 system->name);
3333                         continue;
3334                 }
3335
3336                 event_test_stuff();
3337
3338                 ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 0);
3339                 if (WARN_ON_ONCE(ret)) {
3340                         pr_warn("error disabling system %s\n",
3341                                 system->name);
3342                         continue;
3343                 }
3344
3345                 pr_cont("OK\n");
3346         }
3347
3348         /* Test with all events enabled */
3349
3350         pr_info("Running tests on all trace events:\n");
3351         pr_info("Testing all events: ");
3352
3353         ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 1);
3354         if (WARN_ON_ONCE(ret)) {
3355                 pr_warn("error enabling all events\n");
3356                 return;
3357         }
3358
3359         event_test_stuff();
3360
3361         /* reset sysname */
3362         ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 0);
3363         if (WARN_ON_ONCE(ret)) {
3364                 pr_warn("error disabling all events\n");
3365                 return;
3366         }
3367
3368         pr_cont("OK\n");
3369 }
3370
3371 #ifdef CONFIG_FUNCTION_TRACER
3372
3373 static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable);
3374
3375 static struct trace_array *event_tr;
3376
3377 static void __init
3378 function_test_events_call(unsigned long ip, unsigned long parent_ip,
3379                           struct ftrace_ops *op, struct pt_regs *pt_regs)
3380 {
3381         struct ring_buffer_event *event;
3382         struct ring_buffer *buffer;
3383         struct ftrace_entry *entry;
3384         unsigned long flags;
3385         long disabled;
3386         int cpu;
3387         int pc;
3388
3389         pc = preempt_count();
3390         preempt_disable_notrace();
3391         cpu = raw_smp_processor_id();
3392         disabled = atomic_inc_return(&per_cpu(ftrace_test_event_disable, cpu));
3393
3394         if (disabled != 1)
3395                 goto out;
3396
3397         local_save_flags(flags);
3398
3399         event = trace_current_buffer_lock_reserve(&buffer,
3400                                                   TRACE_FN, sizeof(*entry),
3401                                                   flags, pc);
3402         if (!event)
3403                 goto out;
3404         entry   = ring_buffer_event_data(event);
3405         entry->ip                       = ip;
3406         entry->parent_ip                = parent_ip;
3407
3408         trace_buffer_unlock_commit(event_tr, buffer, event, flags, pc);
3409
3410  out:
3411         atomic_dec(&per_cpu(ftrace_test_event_disable, cpu));
3412         preempt_enable_notrace();
3413 }
3414
3415 static struct ftrace_ops trace_ops __initdata  =
3416 {
3417         .func = function_test_events_call,
3418         .flags = FTRACE_OPS_FL_RECURSION_SAFE,
3419 };
3420
3421 static __init void event_trace_self_test_with_function(void)
3422 {
3423         int ret;
3424         event_tr = top_trace_array();
3425         if (WARN_ON(!event_tr))
3426                 return;
3427         ret = register_ftrace_function(&trace_ops);
3428         if (WARN_ON(ret < 0)) {
3429                 pr_info("Failed to enable function tracer for event tests\n");
3430                 return;
3431         }
3432         pr_info("Running tests again, along with the function tracer\n");
3433         event_trace_self_tests();
3434         unregister_ftrace_function(&trace_ops);
3435 }
3436 #else
3437 static __init void event_trace_self_test_with_function(void)
3438 {
3439 }
3440 #endif
3441
3442 static __init int event_trace_self_tests_init(void)
3443 {
3444         if (!tracing_selftest_disabled) {
3445                 event_trace_self_tests();
3446                 event_trace_self_test_with_function();
3447         }
3448
3449         return 0;
3450 }
3451
3452 late_initcall(event_trace_self_tests_init);
3453
3454 #endif