kernel/tools/perf/util/bpf-loader.c

   1 /*
   2  * bpf-loader.c
   3  *
   4  * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
   5  * Copyright (C) 2015 Huawei Inc.
   6  */
   7
   8 #include <bpf/libbpf.h>
   9 #include <linux/err.h>
  10 #include "perf.h"
  11 #include "debug.h"
  12 #include "bpf-loader.h"
  13 #include "probe-event.h"
  14 #include "probe-finder.h" // for MAX_PROBES
  15 #include "llvm-utils.h"
  16
  17 #define DEFINE_PRINT_FN(name, level) \
  18 static int libbpf_##name(const char *fmt, ...)  \
  19 {                                               \
  20         va_list args;                           \
  21         int ret;                                \
  22                                                 \
  23         va_start(args, fmt);                    \
  24         ret = veprintf(level, verbose, pr_fmt(fmt), args);\
  25         va_end(args);                           \
  26         return ret;                             \
  27 }
  28
  29 DEFINE_PRINT_FN(warning, 1)
  30 DEFINE_PRINT_FN(info, 1)
  31 DEFINE_PRINT_FN(debug, 1)
  32
  33 struct bpf_prog_priv {
  34         struct perf_probe_event pev;
  35 };
  36
  37 static bool libbpf_initialized;
  38
  39 struct bpf_object *
  40 bpf__prepare_load_buffer(void *obj_buf, size_t obj_buf_sz, const char *name)
  41 {
  42         struct bpf_object *obj;
  43
  44         if (!libbpf_initialized) {
  45                 libbpf_set_print(libbpf_warning,
  46                                  libbpf_info,
  47                                  libbpf_debug);
  48                 libbpf_initialized = true;
  49         }
  50
  51         obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, name);
  52         if (IS_ERR(obj)) {
  53                 pr_debug("bpf: failed to load buffer\n");
  54                 return ERR_PTR(-EINVAL);
  55         }
  56
  57         return obj;
  58 }
  59
  60 struct bpf_object *bpf__prepare_load(const char *filename, bool source)
  61 {
  62         struct bpf_object *obj;
  63
  64         if (!libbpf_initialized) {
  65                 libbpf_set_print(libbpf_warning,
  66                                  libbpf_info,
  67                                  libbpf_debug);
  68                 libbpf_initialized = true;
  69         }
  70
  71         if (source) {
  72                 int err;
  73                 void *obj_buf;
  74                 size_t obj_buf_sz;
  75
  76                 err = llvm__compile_bpf(filename, &obj_buf, &obj_buf_sz);
  77                 if (err)
  78                         return ERR_PTR(-BPF_LOADER_ERRNO__COMPILE);
  79                 obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, filename);
  80                 free(obj_buf);
  81         } else
  82                 obj = bpf_object__open(filename);
  83
  84         if (IS_ERR(obj)) {
  85                 pr_debug("bpf: failed to load %s\n", filename);
  86                 return obj;
  87         }
  88
  89         return obj;
  90 }
  91
  92 void bpf__clear(void)
  93 {
  94         struct bpf_object *obj, *tmp;
  95
  96         bpf_object__for_each_safe(obj, tmp) {
  97                 bpf__unprobe(obj);
  98                 bpf_object__close(obj);
  99         }
 100 }
 101
 102 static void
 103 bpf_prog_priv__clear(struct bpf_program *prog __maybe_unused,
 104                      void *_priv)
 105 {
 106         struct bpf_prog_priv *priv = _priv;
 107
 108         cleanup_perf_probe_events(&priv->pev, 1);
 109         free(priv);
 110 }
 111
 112 static int
 113 config_bpf_program(struct bpf_program *prog)
 114 {
 115         struct perf_probe_event *pev = NULL;
 116         struct bpf_prog_priv *priv = NULL;
 117         const char *config_str;
 118         int err;
 119
 120         config_str = bpf_program__title(prog, false);
 121         if (IS_ERR(config_str)) {
 122                 pr_debug("bpf: unable to get title for program\n");
 123                 return PTR_ERR(config_str);
 124         }
 125
 126         priv = calloc(sizeof(*priv), 1);
 127         if (!priv) {
 128                 pr_debug("bpf: failed to alloc priv\n");
 129                 return -ENOMEM;
 130         }
 131         pev = &priv->pev;
 132
 133         pr_debug("bpf: config program '%s'\n", config_str);
 134         err = parse_perf_probe_command(config_str, pev);
 135         if (err < 0) {
 136                 pr_debug("bpf: '%s' is not a valid config string\n",
 137                          config_str);
 138                 err = -BPF_LOADER_ERRNO__CONFIG;
 139                 goto errout;
 140         }
 141
 142         if (pev->group && strcmp(pev->group, PERF_BPF_PROBE_GROUP)) {
 143                 pr_debug("bpf: '%s': group for event is set and not '%s'.\n",
 144                          config_str, PERF_BPF_PROBE_GROUP);
 145                 err = -BPF_LOADER_ERRNO__GROUP;
 146                 goto errout;
 147         } else if (!pev->group)
 148                 pev->group = strdup(PERF_BPF_PROBE_GROUP);
 149
 150         if (!pev->group) {
 151                 pr_debug("bpf: strdup failed\n");
 152                 err = -ENOMEM;
 153                 goto errout;
 154         }
 155
 156         if (!pev->event) {
 157                 pr_debug("bpf: '%s': event name is missing. Section name should be 'key=value'\n",
 158                          config_str);
 159                 err = -BPF_LOADER_ERRNO__EVENTNAME;
 160                 goto errout;
 161         }
 162         pr_debug("bpf: config '%s' is ok\n", config_str);
 163
 164         err = bpf_program__set_private(prog, priv, bpf_prog_priv__clear);
 165         if (err) {
 166                 pr_debug("Failed to set priv for program '%s'\n", config_str);
 167                 goto errout;
 168         }
 169
 170         return 0;
 171
 172 errout:
 173         if (pev)
 174                 clear_perf_probe_event(pev);
 175         free(priv);
 176         return err;
 177 }
 178
 179 static int bpf__prepare_probe(void)
 180 {
 181         static int err = 0;
 182         static bool initialized = false;
 183
 184         /*
 185          * Make err static, so if init failed the first, bpf__prepare_probe()
 186          * fails each time without calling init_probe_symbol_maps multiple
 187          * times.
 188          */
 189         if (initialized)
 190                 return err;
 191
 192         initialized = true;
 193         err = init_probe_symbol_maps(false);
 194         if (err < 0)
 195                 pr_debug("Failed to init_probe_symbol_maps\n");
 196         probe_conf.max_probes = MAX_PROBES;
 197         return err;
 198 }
 199
 200 int bpf__probe(struct bpf_object *obj)
 201 {
 202         int err = 0;
 203         struct bpf_program *prog;
 204         struct bpf_prog_priv *priv;
 205         struct perf_probe_event *pev;
 206
 207         err = bpf__prepare_probe();
 208         if (err) {
 209                 pr_debug("bpf__prepare_probe failed\n");
 210                 return err;
 211         }
 212
 213         bpf_object__for_each_program(prog, obj) {
 214                 err = config_bpf_program(prog);
 215                 if (err)
 216                         goto out;
 217
 218                 err = bpf_program__get_private(prog, (void **)&priv);
 219                 if (err || !priv)
 220                         goto out;
 221                 pev = &priv->pev;
 222
 223                 err = convert_perf_probe_events(pev, 1);
 224                 if (err < 0) {
 225                         pr_debug("bpf_probe: failed to convert perf probe events");
 226                         goto out;
 227                 }
 228
 229                 err = apply_perf_probe_events(pev, 1);
 230                 if (err < 0) {
 231                         pr_debug("bpf_probe: failed to apply perf probe events");
 232                         goto out;
 233                 }
 234         }
 235 out:
 236         return err < 0 ? err : 0;
 237 }
 238
 239 #define EVENTS_WRITE_BUFSIZE  4096
 240 int bpf__unprobe(struct bpf_object *obj)
 241 {
 242         int err, ret = 0;
 243         struct bpf_program *prog;
 244         struct bpf_prog_priv *priv;
 245
 246         bpf_object__for_each_program(prog, obj) {
 247                 int i;
 248
 249                 err = bpf_program__get_private(prog, (void **)&priv);
 250                 if (err || !priv)
 251                         continue;
 252
 253                 for (i = 0; i < priv->pev.ntevs; i++) {
 254                         struct probe_trace_event *tev = &priv->pev.tevs[i];
 255                         char name_buf[EVENTS_WRITE_BUFSIZE];
 256                         struct strfilter *delfilter;
 257
 258                         snprintf(name_buf, EVENTS_WRITE_BUFSIZE,
 259                                  "%s:%s", tev->group, tev->event);
 260                         name_buf[EVENTS_WRITE_BUFSIZE - 1] = '\0';
 261
 262                         delfilter = strfilter__new(name_buf, NULL);
 263                         if (!delfilter) {
 264                                 pr_debug("Failed to create filter for unprobing\n");
 265                                 ret = -ENOMEM;
 266                                 continue;
 267                         }
 268
 269                         err = del_perf_probe_events(delfilter);
 270                         strfilter__delete(delfilter);
 271                         if (err) {
 272                                 pr_debug("Failed to delete %s\n", name_buf);
 273                                 ret = err;
 274                                 continue;
 275                         }
 276                 }
 277         }
 278         return ret;
 279 }
 280
 281 int bpf__load(struct bpf_object *obj)
 282 {
 283         int err;
 284
 285         err = bpf_object__load(obj);
 286         if (err) {
 287                 pr_debug("bpf: load objects failed\n");
 288                 return err;
 289         }
 290         return 0;
 291 }
 292
 293 int bpf__foreach_tev(struct bpf_object *obj,
 294                      bpf_prog_iter_callback_t func,
 295                      void *arg)
 296 {
 297         struct bpf_program *prog;
 298         int err;
 299
 300         bpf_object__for_each_program(prog, obj) {
 301                 struct probe_trace_event *tev;
 302                 struct perf_probe_event *pev;
 303                 struct bpf_prog_priv *priv;
 304                 int i, fd;
 305
 306                 err = bpf_program__get_private(prog,
 307                                 (void **)&priv);
 308                 if (err || !priv) {
 309                         pr_debug("bpf: failed to get private field\n");
 310                         return -BPF_LOADER_ERRNO__INTERNAL;
 311                 }
 312
 313                 pev = &priv->pev;
 314                 for (i = 0; i < pev->ntevs; i++) {
 315                         tev = &pev->tevs[i];
 316
 317                         fd = bpf_program__fd(prog);
 318                         if (fd < 0) {
 319                                 pr_debug("bpf: failed to get file descriptor\n");
 320                                 return fd;
 321                         }
 322
 323                         err = (*func)(tev, fd, arg);
 324                         if (err) {
 325                                 pr_debug("bpf: call back failed, stop iterate\n");
 326                                 return err;
 327                         }
 328                 }
 329         }
 330         return 0;
 331 }
 332
 333 #define ERRNO_OFFSET(e)         ((e) - __BPF_LOADER_ERRNO__START)
 334 #define ERRCODE_OFFSET(c)       ERRNO_OFFSET(BPF_LOADER_ERRNO__##c)
 335 #define NR_ERRNO        (__BPF_LOADER_ERRNO__END - __BPF_LOADER_ERRNO__START)
 336
 337 static const char *bpf_loader_strerror_table[NR_ERRNO] = {
 338         [ERRCODE_OFFSET(CONFIG)]        = "Invalid config string",
 339         [ERRCODE_OFFSET(GROUP)]         = "Invalid group name",
 340         [ERRCODE_OFFSET(EVENTNAME)]     = "No event name found in config string",
 341         [ERRCODE_OFFSET(INTERNAL)]      = "BPF loader internal error",
 342         [ERRCODE_OFFSET(COMPILE)]       = "Error when compiling BPF scriptlet",
 343 };
 344
 345 static int
 346 bpf_loader_strerror(int err, char *buf, size_t size)
 347 {
 348         char sbuf[STRERR_BUFSIZE];
 349         const char *msg;
 350
 351         if (!buf || !size)
 352                 return -1;
 353
 354         err = err > 0 ? err : -err;
 355
 356         if (err >= __LIBBPF_ERRNO__START)
 357                 return libbpf_strerror(err, buf, size);
 358
 359         if (err >= __BPF_LOADER_ERRNO__START && err < __BPF_LOADER_ERRNO__END) {
 360                 msg = bpf_loader_strerror_table[ERRNO_OFFSET(err)];
 361                 snprintf(buf, size, "%s", msg);
 362                 buf[size - 1] = '\0';
 363                 return 0;
 364         }
 365
 366         if (err >= __BPF_LOADER_ERRNO__END)
 367                 snprintf(buf, size, "Unknown bpf loader error %d", err);
 368         else
 369                 snprintf(buf, size, "%s",
 370                          strerror_r(err, sbuf, sizeof(sbuf)));
 371
 372         buf[size - 1] = '\0';
 373         return -1;
 374 }
 375
 376 #define bpf__strerror_head(err, buf, size) \
 377         char sbuf[STRERR_BUFSIZE], *emsg;\
 378         if (!size)\
 379                 return 0;\
 380         if (err < 0)\
 381                 err = -err;\
 382         bpf_loader_strerror(err, sbuf, sizeof(sbuf));\
 383         emsg = sbuf;\
 384         switch (err) {\
 385         default:\
 386                 scnprintf(buf, size, "%s", emsg);\
 387                 break;
 388
 389 #define bpf__strerror_entry(val, fmt...)\
 390         case val: {\
 391                 scnprintf(buf, size, fmt);\
 392                 break;\
 393         }
 394
 395 #define bpf__strerror_end(buf, size)\
 396         }\
 397         buf[size - 1] = '\0';
 398
 399 int bpf__strerror_prepare_load(const char *filename, bool source,
 400                                int err, char *buf, size_t size)
 401 {
 402         size_t n;
 403         int ret;
 404
 405         n = snprintf(buf, size, "Failed to load %s%s: ",
 406                          filename, source ? " from source" : "");
 407         if (n >= size) {
 408                 buf[size - 1] = '\0';
 409                 return 0;
 410         }
 411         buf += n;
 412         size -= n;
 413
 414         ret = bpf_loader_strerror(err, buf, size);
 415         buf[size - 1] = '\0';
 416         return ret;
 417 }
 418
 419 int bpf__strerror_probe(struct bpf_object *obj __maybe_unused,
 420                         int err, char *buf, size_t size)
 421 {
 422         bpf__strerror_head(err, buf, size);
 423         bpf__strerror_entry(EEXIST, "Probe point exist. Try use 'perf probe -d \"*\"'");
 424         bpf__strerror_entry(EACCES, "You need to be root");
 425         bpf__strerror_entry(EPERM, "You need to be root, and /proc/sys/kernel/kptr_restrict should be 0");
 426         bpf__strerror_entry(ENOENT, "You need to check probing points in BPF file");
 427         bpf__strerror_end(buf, size);
 428         return 0;
 429 }
 430
 431 int bpf__strerror_load(struct bpf_object *obj,
 432                        int err, char *buf, size_t size)
 433 {
 434         bpf__strerror_head(err, buf, size);
 435         case LIBBPF_ERRNO__KVER: {
 436                 unsigned int obj_kver = bpf_object__get_kversion(obj);
 437                 unsigned int real_kver;
 438
 439                 if (fetch_kernel_version(&real_kver, NULL, 0)) {
 440                         scnprintf(buf, size, "Unable to fetch kernel version");
 441                         break;
 442                 }
 443
 444                 if (obj_kver != real_kver) {
 445                         scnprintf(buf, size,
 446                                   "'version' ("KVER_FMT") doesn't match running kernel ("KVER_FMT")",
 447                                   KVER_PARAM(obj_kver),
 448                                   KVER_PARAM(real_kver));
 449                         break;
 450                 }
 451
 452                 scnprintf(buf, size, "Failed to load program for unknown reason");
 453                 break;
 454         }
 455         bpf__strerror_end(buf, size);
 456         return 0;
 457 }