kernel/tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 /*
   2  * intel_pt_decoder.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #ifndef _GNU_SOURCE
  17 #define _GNU_SOURCE
  18 #endif
  19 #include <stdlib.h>
  20 #include <stdbool.h>
  21 #include <string.h>
  22 #include <errno.h>
  23 #include <stdint.h>
  24 #include <inttypes.h>
  25
  26 #include "../cache.h"
  27 #include "../util.h"
  28
  29 #include "intel-pt-insn-decoder.h"
  30 #include "intel-pt-pkt-decoder.h"
  31 #include "intel-pt-decoder.h"
  32 #include "intel-pt-log.h"
  33
  34 #define INTEL_PT_BLK_SIZE 1024
  35
  36 #define BIT63 (((uint64_t)1 << 63))
  37
  38 #define INTEL_PT_RETURN 1
  39
  40 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
  41 #define INTEL_PT_MAX_LOOPS 10000
  42
  43 struct intel_pt_blk {
  44         struct intel_pt_blk *prev;
  45         uint64_t ip[INTEL_PT_BLK_SIZE];
  46 };
  47
  48 struct intel_pt_stack {
  49         struct intel_pt_blk *blk;
  50         struct intel_pt_blk *spare;
  51         int pos;
  52 };
  53
  54 enum intel_pt_pkt_state {
  55         INTEL_PT_STATE_NO_PSB,
  56         INTEL_PT_STATE_NO_IP,
  57         INTEL_PT_STATE_ERR_RESYNC,
  58         INTEL_PT_STATE_IN_SYNC,
  59         INTEL_PT_STATE_TNT,
  60         INTEL_PT_STATE_TIP,
  61         INTEL_PT_STATE_TIP_PGD,
  62         INTEL_PT_STATE_FUP,
  63         INTEL_PT_STATE_FUP_NO_TIP,
  64 };
  65
  66 #ifdef INTEL_PT_STRICT
  67 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
  68 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
  69 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
  70 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
  71 #else
  72 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
  73 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
  74 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
  75 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
  76 #endif
  77
  78 struct intel_pt_decoder {
  79         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
  80         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
  81                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
  82                          uint64_t max_insn_cnt, void *data);
  83         void *data;
  84         struct intel_pt_state state;
  85         const unsigned char *buf;
  86         size_t len;
  87         bool return_compression;
  88         bool mtc_insn;
  89         bool pge;
  90         bool have_tma;
  91         bool have_cyc;
  92         bool fixup_last_mtc;
  93         uint64_t pos;
  94         uint64_t last_ip;
  95         uint64_t ip;
  96         uint64_t cr3;
  97         uint64_t timestamp;
  98         uint64_t tsc_timestamp;
  99         uint64_t ref_timestamp;
 100         uint64_t ret_addr;
 101         uint64_t ctc_timestamp;
 102         uint64_t ctc_delta;
 103         uint64_t cycle_cnt;
 104         uint64_t cyc_ref_timestamp;
 105         uint32_t last_mtc;
 106         uint32_t tsc_ctc_ratio_n;
 107         uint32_t tsc_ctc_ratio_d;
 108         uint32_t tsc_ctc_mult;
 109         uint32_t tsc_slip;
 110         uint32_t ctc_rem_mask;
 111         int mtc_shift;
 112         struct intel_pt_stack stack;
 113         enum intel_pt_pkt_state pkt_state;
 114         struct intel_pt_pkt packet;
 115         struct intel_pt_pkt tnt;
 116         int pkt_step;
 117         int pkt_len;
 118         int last_packet_type;
 119         unsigned int cbr;
 120         unsigned int max_non_turbo_ratio;
 121         double max_non_turbo_ratio_fp;
 122         double cbr_cyc_to_tsc;
 123         double calc_cyc_to_tsc;
 124         bool have_calc_cyc_to_tsc;
 125         int exec_mode;
 126         unsigned int insn_bytes;
 127         uint64_t sign_bit;
 128         uint64_t sign_bits;
 129         uint64_t period;
 130         enum intel_pt_period_type period_type;
 131         uint64_t tot_insn_cnt;
 132         uint64_t period_insn_cnt;
 133         uint64_t period_mask;
 134         uint64_t period_ticks;
 135         uint64_t last_masked_timestamp;
 136         bool continuous_period;
 137         bool overflow;
 138         bool set_fup_tx_flags;
 139         unsigned int fup_tx_flags;
 140         unsigned int tx_flags;
 141         uint64_t timestamp_insn_cnt;
 142         uint64_t stuck_ip;
 143         int no_progress;
 144         int stuck_ip_prd;
 145         int stuck_ip_cnt;
 146         const unsigned char *next_buf;
 147         size_t next_len;
 148         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 149 };
 150
 151 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 152 {
 153         int i;
 154
 155         for (i = 0; x != 1; i++)
 156                 x >>= 1;
 157
 158         return x << i;
 159 }
 160
 161 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 162 {
 163         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 164                 uint64_t period;
 165
 166                 period = intel_pt_lower_power_of_2(decoder->period);
 167                 decoder->period_mask  = ~(period - 1);
 168                 decoder->period_ticks = period;
 169         }
 170 }
 171
 172 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 173 {
 174         if (!d)
 175                 return 0;
 176         return (t / d) * n + ((t % d) * n) / d;
 177 }
 178
 179 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 180 {
 181         struct intel_pt_decoder *decoder;
 182
 183         if (!params->get_trace || !params->walk_insn)
 184                 return NULL;
 185
 186         decoder = zalloc(sizeof(struct intel_pt_decoder));
 187         if (!decoder)
 188                 return NULL;
 189
 190         decoder->get_trace          = params->get_trace;
 191         decoder->walk_insn          = params->walk_insn;
 192         decoder->data               = params->data;
 193         decoder->return_compression = params->return_compression;
 194
 195         decoder->sign_bit           = (uint64_t)1 << 47;
 196         decoder->sign_bits          = ~(((uint64_t)1 << 48) - 1);
 197
 198         decoder->period             = params->period;
 199         decoder->period_type        = params->period_type;
 200
 201         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 202         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 203
 204         intel_pt_setup_period(decoder);
 205
 206         decoder->mtc_shift = params->mtc_period;
 207         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 208
 209         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 210         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 211
 212         if (!decoder->tsc_ctc_ratio_n)
 213                 decoder->tsc_ctc_ratio_d = 0;
 214
 215         if (decoder->tsc_ctc_ratio_d) {
 216                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 217                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 218                                                 decoder->tsc_ctc_ratio_d;
 219
 220                 /*
 221                  * Allow for timestamps appearing to backwards because a TSC
 222                  * packet has slipped past a MTC packet, so allow 2 MTC ticks
 223                  * or ...
 224                  */
 225                 decoder->tsc_slip = multdiv(2 << decoder->mtc_shift,
 226                                         decoder->tsc_ctc_ratio_n,
 227                                         decoder->tsc_ctc_ratio_d);
 228         }
 229         /* ... or 0x100 paranoia */
 230         if (decoder->tsc_slip < 0x100)
 231                 decoder->tsc_slip = 0x100;
 232
 233         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 234         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 235         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 236         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 237         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 238
 239         return decoder;
 240 }
 241
 242 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 243 {
 244         struct intel_pt_blk *blk = stack->blk;
 245
 246         stack->blk = blk->prev;
 247         if (!stack->spare)
 248                 stack->spare = blk;
 249         else
 250                 free(blk);
 251 }
 252
 253 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 254 {
 255         if (!stack->pos) {
 256                 if (!stack->blk)
 257                         return 0;
 258                 intel_pt_pop_blk(stack);
 259                 if (!stack->blk)
 260                         return 0;
 261                 stack->pos = INTEL_PT_BLK_SIZE;
 262         }
 263         return stack->blk->ip[--stack->pos];
 264 }
 265
 266 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 267 {
 268         struct intel_pt_blk *blk;
 269
 270         if (stack->spare) {
 271                 blk = stack->spare;
 272                 stack->spare = NULL;
 273         } else {
 274                 blk = malloc(sizeof(struct intel_pt_blk));
 275                 if (!blk)
 276                         return -ENOMEM;
 277         }
 278
 279         blk->prev = stack->blk;
 280         stack->blk = blk;
 281         stack->pos = 0;
 282         return 0;
 283 }
 284
 285 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 286 {
 287         int err;
 288
 289         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 290                 err = intel_pt_alloc_blk(stack);
 291                 if (err)
 292                         return err;
 293         }
 294
 295         stack->blk->ip[stack->pos++] = ip;
 296         return 0;
 297 }
 298
 299 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 300 {
 301         while (stack->blk)
 302                 intel_pt_pop_blk(stack);
 303         stack->pos = 0;
 304 }
 305
 306 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 307 {
 308         intel_pt_clear_stack(stack);
 309         zfree(&stack->blk);
 310         zfree(&stack->spare);
 311 }
 312
 313 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 314 {
 315         intel_pt_free_stack(&decoder->stack);
 316         free(decoder);
 317 }
 318
 319 static int intel_pt_ext_err(int code)
 320 {
 321         switch (code) {
 322         case -ENOMEM:
 323                 return INTEL_PT_ERR_NOMEM;
 324         case -ENOSYS:
 325                 return INTEL_PT_ERR_INTERN;
 326         case -EBADMSG:
 327                 return INTEL_PT_ERR_BADPKT;
 328         case -ENODATA:
 329                 return INTEL_PT_ERR_NODATA;
 330         case -EILSEQ:
 331                 return INTEL_PT_ERR_NOINSN;
 332         case -ENOENT:
 333                 return INTEL_PT_ERR_MISMAT;
 334         case -EOVERFLOW:
 335                 return INTEL_PT_ERR_OVR;
 336         case -ENOSPC:
 337                 return INTEL_PT_ERR_LOST;
 338         case -ELOOP:
 339                 return INTEL_PT_ERR_NELOOP;
 340         default:
 341                 return INTEL_PT_ERR_UNK;
 342         }
 343 }
 344
 345 static const char *intel_pt_err_msgs[] = {
 346         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 347         [INTEL_PT_ERR_INTERN] = "Internal error",
 348         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 349         [INTEL_PT_ERR_NODATA] = "No more data",
 350         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 351         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 352         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 353         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 354         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 355         [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
 356 };
 357
 358 int intel_pt__strerror(int code, char *buf, size_t buflen)
 359 {
 360         if (code < 1 || code > INTEL_PT_ERR_MAX)
 361                 code = INTEL_PT_ERR_UNK;
 362         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 363         return 0;
 364 }
 365
 366 static uint64_t intel_pt_calc_ip(struct intel_pt_decoder *decoder,
 367                                  const struct intel_pt_pkt *packet,
 368                                  uint64_t last_ip)
 369 {
 370         uint64_t ip;
 371
 372         switch (packet->count) {
 373         case 2:
 374                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 375                      packet->payload;
 376                 break;
 377         case 4:
 378                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 379                      packet->payload;
 380                 break;
 381         case 6:
 382                 ip = packet->payload;
 383                 break;
 384         default:
 385                 return 0;
 386         }
 387
 388         if (ip & decoder->sign_bit)
 389                 return ip | decoder->sign_bits;
 390
 391         return ip;
 392 }
 393
 394 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 395 {
 396         decoder->last_ip = intel_pt_calc_ip(decoder, &decoder->packet,
 397                                             decoder->last_ip);
 398 }
 399
 400 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 401 {
 402         intel_pt_set_last_ip(decoder);
 403         decoder->ip = decoder->last_ip;
 404 }
 405
 406 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 407 {
 408         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 409                             decoder->buf);
 410 }
 411
 412 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 413 {
 414         intel_pt_log("ERROR: Internal error\n");
 415         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 416         return -ENOSYS;
 417 }
 418
 419 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 420 {
 421         decoder->tx_flags = 0;
 422 }
 423
 424 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 425 {
 426         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 427 }
 428
 429 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 430 {
 431         intel_pt_clear_tx_flags(decoder);
 432         decoder->have_tma = false;
 433         decoder->pkt_len = 1;
 434         decoder->pkt_step = 1;
 435         intel_pt_decoder_log_packet(decoder);
 436         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 437                 intel_pt_log("ERROR: Bad packet\n");
 438                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 439         }
 440         return -EBADMSG;
 441 }
 442
 443 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
 444 {
 445         struct intel_pt_buffer buffer = { .buf = 0, };
 446         int ret;
 447
 448         decoder->pkt_step = 0;
 449
 450         intel_pt_log("Getting more data\n");
 451         ret = decoder->get_trace(&buffer, decoder->data);
 452         if (ret)
 453                 return ret;
 454         decoder->buf = buffer.buf;
 455         decoder->len = buffer.len;
 456         if (!decoder->len) {
 457                 intel_pt_log("No more data\n");
 458                 return -ENODATA;
 459         }
 460         if (!buffer.consecutive) {
 461                 decoder->ip = 0;
 462                 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 463                 decoder->ref_timestamp = buffer.ref_timestamp;
 464                 decoder->timestamp = 0;
 465                 decoder->have_tma = false;
 466                 decoder->state.trace_nr = buffer.trace_nr;
 467                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 468                              decoder->ref_timestamp);
 469                 return -ENOLINK;
 470         }
 471
 472         return 0;
 473 }
 474
 475 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
 476 {
 477         if (!decoder->next_buf)
 478                 return intel_pt_get_data(decoder);
 479
 480         decoder->buf = decoder->next_buf;
 481         decoder->len = decoder->next_len;
 482         decoder->next_buf = 0;
 483         decoder->next_len = 0;
 484         return 0;
 485 }
 486
 487 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 488 {
 489         unsigned char *buf = decoder->temp_buf;
 490         size_t old_len, len, n;
 491         int ret;
 492
 493         old_len = decoder->len;
 494         len = decoder->len;
 495         memcpy(buf, decoder->buf, len);
 496
 497         ret = intel_pt_get_data(decoder);
 498         if (ret) {
 499                 decoder->pos += old_len;
 500                 return ret < 0 ? ret : -EINVAL;
 501         }
 502
 503         n = INTEL_PT_PKT_MAX_SZ - len;
 504         if (n > decoder->len)
 505                 n = decoder->len;
 506         memcpy(buf + len, decoder->buf, n);
 507         len += n;
 508
 509         ret = intel_pt_get_packet(buf, len, &decoder->packet);
 510         if (ret < (int)old_len) {
 511                 decoder->next_buf = decoder->buf;
 512                 decoder->next_len = decoder->len;
 513                 decoder->buf = buf;
 514                 decoder->len = old_len;
 515                 return intel_pt_bad_packet(decoder);
 516         }
 517
 518         decoder->next_buf = decoder->buf + (ret - old_len);
 519         decoder->next_len = decoder->len - (ret - old_len);
 520
 521         decoder->buf = buf;
 522         decoder->len = ret;
 523
 524         return ret;
 525 }
 526
 527 struct intel_pt_pkt_info {
 528         struct intel_pt_decoder   *decoder;
 529         struct intel_pt_pkt       packet;
 530         uint64_t                  pos;
 531         int                       pkt_len;
 532         int                       last_packet_type;
 533         void                      *data;
 534 };
 535
 536 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 537
 538 /* Lookahead packets in current buffer */
 539 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 540                                   intel_pt_pkt_cb_t cb, void *data)
 541 {
 542         struct intel_pt_pkt_info pkt_info;
 543         const unsigned char *buf = decoder->buf;
 544         size_t len = decoder->len;
 545         int ret;
 546
 547         pkt_info.decoder          = decoder;
 548         pkt_info.pos              = decoder->pos;
 549         pkt_info.pkt_len          = decoder->pkt_step;
 550         pkt_info.last_packet_type = decoder->last_packet_type;
 551         pkt_info.data             = data;
 552
 553         while (1) {
 554                 do {
 555                         pkt_info.pos += pkt_info.pkt_len;
 556                         buf          += pkt_info.pkt_len;
 557                         len          -= pkt_info.pkt_len;
 558
 559                         if (!len)
 560                                 return INTEL_PT_NEED_MORE_BYTES;
 561
 562                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
 563                         if (!ret)
 564                                 return INTEL_PT_NEED_MORE_BYTES;
 565                         if (ret < 0)
 566                                 return ret;
 567
 568                         pkt_info.pkt_len = ret;
 569                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 570
 571                 ret = cb(&pkt_info);
 572                 if (ret)
 573                         return 0;
 574
 575                 pkt_info.last_packet_type = pkt_info.packet.type;
 576         }
 577 }
 578
 579 struct intel_pt_calc_cyc_to_tsc_info {
 580         uint64_t        cycle_cnt;
 581         unsigned int    cbr;
 582         uint32_t        last_mtc;
 583         uint64_t        ctc_timestamp;
 584         uint64_t        ctc_delta;
 585         uint64_t        tsc_timestamp;
 586         uint64_t        timestamp;
 587         bool            have_tma;
 588         bool            fixup_last_mtc;
 589         bool            from_mtc;
 590         double          cbr_cyc_to_tsc;
 591 };
 592
 593 /*
 594  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
 595  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
 596  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
 597  * packet by copying the missing bits from the current MTC assuming the least
 598  * difference between the two, and that the current MTC comes after last_mtc.
 599  */
 600 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
 601                                     uint32_t *last_mtc)
 602 {
 603         uint32_t first_missing_bit = 1U << (16 - mtc_shift);
 604         uint32_t mask = ~(first_missing_bit - 1);
 605
 606         *last_mtc |= mtc & mask;
 607         if (*last_mtc >= mtc) {
 608                 *last_mtc -= first_missing_bit;
 609                 *last_mtc &= 0xff;
 610         }
 611 }
 612
 613 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 614 {
 615         struct intel_pt_decoder *decoder = pkt_info->decoder;
 616         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 617         uint64_t timestamp;
 618         double cyc_to_tsc;
 619         unsigned int cbr;
 620         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 621
 622         switch (pkt_info->packet.type) {
 623         case INTEL_PT_TNT:
 624         case INTEL_PT_TIP_PGE:
 625         case INTEL_PT_TIP:
 626         case INTEL_PT_FUP:
 627         case INTEL_PT_PSB:
 628         case INTEL_PT_PIP:
 629         case INTEL_PT_MODE_EXEC:
 630         case INTEL_PT_MODE_TSX:
 631         case INTEL_PT_PSBEND:
 632         case INTEL_PT_PAD:
 633         case INTEL_PT_VMCS:
 634         case INTEL_PT_MNT:
 635                 return 0;
 636
 637         case INTEL_PT_MTC:
 638                 if (!data->have_tma)
 639                         return 0;
 640
 641                 mtc = pkt_info->packet.payload;
 642                 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
 643                         data->fixup_last_mtc = false;
 644                         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
 645                                                 &data->last_mtc);
 646                 }
 647                 if (mtc > data->last_mtc)
 648                         mtc_delta = mtc - data->last_mtc;
 649                 else
 650                         mtc_delta = mtc + 256 - data->last_mtc;
 651                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 652                 data->last_mtc = mtc;
 653
 654                 if (decoder->tsc_ctc_mult) {
 655                         timestamp = data->ctc_timestamp +
 656                                 data->ctc_delta * decoder->tsc_ctc_mult;
 657                 } else {
 658                         timestamp = data->ctc_timestamp +
 659                                 multdiv(data->ctc_delta,
 660                                         decoder->tsc_ctc_ratio_n,
 661                                         decoder->tsc_ctc_ratio_d);
 662                 }
 663
 664                 if (timestamp < data->timestamp)
 665                         return 1;
 666
 667                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 668                         data->timestamp = timestamp;
 669                         return 0;
 670                 }
 671
 672                 break;
 673
 674         case INTEL_PT_TSC:
 675                 timestamp = pkt_info->packet.payload |
 676                             (data->timestamp & (0xffULL << 56));
 677                 if (data->from_mtc && timestamp < data->timestamp &&
 678                     data->timestamp - timestamp < decoder->tsc_slip)
 679                         return 1;
 680                 if (timestamp < data->timestamp)
 681                         timestamp += (1ULL << 56);
 682                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 683                         if (data->from_mtc)
 684                                 return 1;
 685                         data->tsc_timestamp = timestamp;
 686                         data->timestamp = timestamp;
 687                         return 0;
 688                 }
 689                 break;
 690
 691         case INTEL_PT_TMA:
 692                 if (data->from_mtc)
 693                         return 1;
 694
 695                 if (!decoder->tsc_ctc_ratio_d)
 696                         return 0;
 697
 698                 ctc = pkt_info->packet.payload;
 699                 fc = pkt_info->packet.count;
 700                 ctc_rem = ctc & decoder->ctc_rem_mask;
 701
 702                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 703
 704                 data->ctc_timestamp = data->tsc_timestamp - fc;
 705                 if (decoder->tsc_ctc_mult) {
 706                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 707                 } else {
 708                         data->ctc_timestamp -=
 709                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 710                                         decoder->tsc_ctc_ratio_d);
 711                 }
 712
 713                 data->ctc_delta = 0;
 714                 data->have_tma = true;
 715                 data->fixup_last_mtc = true;
 716
 717                 return 0;
 718
 719         case INTEL_PT_CYC:
 720                 data->cycle_cnt += pkt_info->packet.payload;
 721                 return 0;
 722
 723         case INTEL_PT_CBR:
 724                 cbr = pkt_info->packet.payload;
 725                 if (data->cbr && data->cbr != cbr)
 726                         return 1;
 727                 data->cbr = cbr;
 728                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 729                 return 0;
 730
 731         case INTEL_PT_TIP_PGD:
 732         case INTEL_PT_TRACESTOP:
 733         case INTEL_PT_OVF:
 734         case INTEL_PT_BAD: /* Does not happen */
 735         default:
 736                 return 1;
 737         }
 738
 739         if (!data->cbr && decoder->cbr) {
 740                 data->cbr = decoder->cbr;
 741                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 742         }
 743
 744         if (!data->cycle_cnt)
 745                 return 1;
 746
 747         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 748
 749         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 750             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 751                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 752                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 753                 return 1;
 754         }
 755
 756         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 757         decoder->have_calc_cyc_to_tsc = true;
 758
 759         if (data->cbr) {
 760                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 761                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 762         } else {
 763                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 764                              cyc_to_tsc, pkt_info->pos);
 765         }
 766
 767         return 1;
 768 }
 769
 770 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 771                                      bool from_mtc)
 772 {
 773         struct intel_pt_calc_cyc_to_tsc_info data = {
 774                 .cycle_cnt      = 0,
 775                 .cbr            = 0,
 776                 .last_mtc       = decoder->last_mtc,
 777                 .ctc_timestamp  = decoder->ctc_timestamp,
 778                 .ctc_delta      = decoder->ctc_delta,
 779                 .tsc_timestamp  = decoder->tsc_timestamp,
 780                 .timestamp      = decoder->timestamp,
 781                 .have_tma       = decoder->have_tma,
 782                 .fixup_last_mtc = decoder->fixup_last_mtc,
 783                 .from_mtc       = from_mtc,
 784                 .cbr_cyc_to_tsc = 0,
 785         };
 786
 787         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
 788 }
 789
 790 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
 791 {
 792         int ret;
 793
 794         decoder->last_packet_type = decoder->packet.type;
 795
 796         do {
 797                 decoder->pos += decoder->pkt_step;
 798                 decoder->buf += decoder->pkt_step;
 799                 decoder->len -= decoder->pkt_step;
 800
 801                 if (!decoder->len) {
 802                         ret = intel_pt_get_next_data(decoder);
 803                         if (ret)
 804                                 return ret;
 805                 }
 806
 807                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
 808                                           &decoder->packet);
 809                 if (ret == INTEL_PT_NEED_MORE_BYTES &&
 810                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
 811                         ret = intel_pt_get_split_packet(decoder);
 812                         if (ret < 0)
 813                                 return ret;
 814                 }
 815                 if (ret <= 0)
 816                         return intel_pt_bad_packet(decoder);
 817
 818                 decoder->pkt_len = ret;
 819                 decoder->pkt_step = ret;
 820                 intel_pt_decoder_log_packet(decoder);
 821         } while (decoder->packet.type == INTEL_PT_PAD);
 822
 823         return 0;
 824 }
 825
 826 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
 827 {
 828         uint64_t timestamp, masked_timestamp;
 829
 830         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 831         masked_timestamp = timestamp & decoder->period_mask;
 832         if (decoder->continuous_period) {
 833                 if (masked_timestamp != decoder->last_masked_timestamp)
 834                         return 1;
 835         } else {
 836                 timestamp += 1;
 837                 masked_timestamp = timestamp & decoder->period_mask;
 838                 if (masked_timestamp != decoder->last_masked_timestamp) {
 839                         decoder->last_masked_timestamp = masked_timestamp;
 840                         decoder->continuous_period = true;
 841                 }
 842         }
 843         return decoder->period_ticks - (timestamp - masked_timestamp);
 844 }
 845
 846 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
 847 {
 848         switch (decoder->period_type) {
 849         case INTEL_PT_PERIOD_INSTRUCTIONS:
 850                 return decoder->period - decoder->period_insn_cnt;
 851         case INTEL_PT_PERIOD_TICKS:
 852                 return intel_pt_next_period(decoder);
 853         case INTEL_PT_PERIOD_NONE:
 854         case INTEL_PT_PERIOD_MTC:
 855         default:
 856                 return 0;
 857         }
 858 }
 859
 860 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
 861 {
 862         uint64_t timestamp, masked_timestamp;
 863
 864         switch (decoder->period_type) {
 865         case INTEL_PT_PERIOD_INSTRUCTIONS:
 866                 decoder->period_insn_cnt = 0;
 867                 break;
 868         case INTEL_PT_PERIOD_TICKS:
 869                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 870                 masked_timestamp = timestamp & decoder->period_mask;
 871                 decoder->last_masked_timestamp = masked_timestamp;
 872                 break;
 873         case INTEL_PT_PERIOD_NONE:
 874         case INTEL_PT_PERIOD_MTC:
 875         default:
 876                 break;
 877         }
 878
 879         decoder->state.type |= INTEL_PT_INSTRUCTION;
 880 }
 881
 882 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
 883                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
 884 {
 885         uint64_t max_insn_cnt, insn_cnt = 0;
 886         int err;
 887
 888         if (!decoder->mtc_insn)
 889                 decoder->mtc_insn = true;
 890
 891         max_insn_cnt = intel_pt_next_sample(decoder);
 892
 893         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
 894                                  max_insn_cnt, decoder->data);
 895
 896         decoder->tot_insn_cnt += insn_cnt;
 897         decoder->timestamp_insn_cnt += insn_cnt;
 898         decoder->period_insn_cnt += insn_cnt;
 899
 900         if (err) {
 901                 decoder->no_progress = 0;
 902                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
 903                 intel_pt_log_at("ERROR: Failed to get instruction",
 904                                 decoder->ip);
 905                 if (err == -ENOENT)
 906                         return -ENOLINK;
 907                 return -EILSEQ;
 908         }
 909
 910         if (ip && decoder->ip == ip) {
 911                 err = -EAGAIN;
 912                 goto out;
 913         }
 914
 915         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 916                 intel_pt_sample_insn(decoder);
 917
 918         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
 919                 decoder->state.type = INTEL_PT_INSTRUCTION;
 920                 decoder->state.from_ip = decoder->ip;
 921                 decoder->state.to_ip = 0;
 922                 decoder->ip += intel_pt_insn->length;
 923                 err = INTEL_PT_RETURN;
 924                 goto out;
 925         }
 926
 927         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
 928                 /* Zero-length calls are excluded */
 929                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
 930                     intel_pt_insn->rel) {
 931                         err = intel_pt_push(&decoder->stack, decoder->ip +
 932                                             intel_pt_insn->length);
 933                         if (err)
 934                                 goto out;
 935                 }
 936         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
 937                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
 938         }
 939
 940         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
 941                 int cnt = decoder->no_progress++;
 942
 943                 decoder->state.from_ip = decoder->ip;
 944                 decoder->ip += intel_pt_insn->length +
 945                                 intel_pt_insn->rel;
 946                 decoder->state.to_ip = decoder->ip;
 947                 err = INTEL_PT_RETURN;
 948
 949                 /*
 950                  * Check for being stuck in a loop.  This can happen if a
 951                  * decoder error results in the decoder erroneously setting the
 952                  * ip to an address that is itself in an infinite loop that
 953                  * consumes no packets.  When that happens, there must be an
 954                  * unconditional branch.
 955                  */
 956                 if (cnt) {
 957                         if (cnt == 1) {
 958                                 decoder->stuck_ip = decoder->state.to_ip;
 959                                 decoder->stuck_ip_prd = 1;
 960                                 decoder->stuck_ip_cnt = 1;
 961                         } else if (cnt > INTEL_PT_MAX_LOOPS ||
 962                                    decoder->state.to_ip == decoder->stuck_ip) {
 963                                 intel_pt_log_at("ERROR: Never-ending loop",
 964                                                 decoder->state.to_ip);
 965                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
 966                                 err = -ELOOP;
 967                                 goto out;
 968                         } else if (!--decoder->stuck_ip_cnt) {
 969                                 decoder->stuck_ip_prd += 1;
 970                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
 971                                 decoder->stuck_ip = decoder->state.to_ip;
 972                         }
 973                 }
 974                 goto out_no_progress;
 975         }
 976 out:
 977         decoder->no_progress = 0;
 978 out_no_progress:
 979         decoder->state.insn_op = intel_pt_insn->op;
 980         decoder->state.insn_len = intel_pt_insn->length;
 981
 982         if (decoder->tx_flags & INTEL_PT_IN_TX)
 983                 decoder->state.flags |= INTEL_PT_IN_TX;
 984
 985         return err;
 986 }
 987
 988 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
 989 {
 990         struct intel_pt_insn intel_pt_insn;
 991         uint64_t ip;
 992         int err;
 993
 994         ip = decoder->last_ip;
 995
 996         while (1) {
 997                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
 998                 if (err == INTEL_PT_RETURN)
 999                         return 0;
1000                 if (err == -EAGAIN) {
1001                         if (decoder->set_fup_tx_flags) {
1002                                 decoder->set_fup_tx_flags = false;
1003                                 decoder->tx_flags = decoder->fup_tx_flags;
1004                                 decoder->state.type = INTEL_PT_TRANSACTION;
1005                                 decoder->state.from_ip = decoder->ip;
1006                                 decoder->state.to_ip = 0;
1007                                 decoder->state.flags = decoder->fup_tx_flags;
1008                                 return 0;
1009                         }
1010                         return err;
1011                 }
1012                 decoder->set_fup_tx_flags = false;
1013                 if (err)
1014                         return err;
1015
1016                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1017                         intel_pt_log_at("ERROR: Unexpected indirect branch",
1018                                         decoder->ip);
1019                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1020                         return -ENOENT;
1021                 }
1022
1023                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1024                         intel_pt_log_at("ERROR: Unexpected conditional branch",
1025                                         decoder->ip);
1026                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1027                         return -ENOENT;
1028                 }
1029
1030                 intel_pt_bug(decoder);
1031         }
1032 }
1033
1034 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1035 {
1036         struct intel_pt_insn intel_pt_insn;
1037         int err;
1038
1039         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1040         if (err == INTEL_PT_RETURN)
1041                 return 0;
1042         if (err)
1043                 return err;
1044
1045         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1046                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1047                         decoder->pge = false;
1048                         decoder->continuous_period = false;
1049                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1050                         decoder->state.from_ip = decoder->ip;
1051                         decoder->state.to_ip = 0;
1052                         if (decoder->packet.count != 0)
1053                                 decoder->ip = decoder->last_ip;
1054                 } else {
1055                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1056                         decoder->state.from_ip = decoder->ip;
1057                         if (decoder->packet.count == 0) {
1058                                 decoder->state.to_ip = 0;
1059                         } else {
1060                                 decoder->state.to_ip = decoder->last_ip;
1061                                 decoder->ip = decoder->last_ip;
1062                         }
1063                 }
1064                 return 0;
1065         }
1066
1067         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1068                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1069                                 decoder->ip);
1070                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1071                 return -ENOENT;
1072         }
1073
1074         return intel_pt_bug(decoder);
1075 }
1076
1077 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1078 {
1079         struct intel_pt_insn intel_pt_insn;
1080         int err;
1081
1082         while (1) {
1083                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1084                 if (err == INTEL_PT_RETURN)
1085                         return 0;
1086                 if (err)
1087                         return err;
1088
1089                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1090                         if (!decoder->return_compression) {
1091                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1092                                                 decoder->ip);
1093                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1094                                 return -ENOENT;
1095                         }
1096                         if (!decoder->ret_addr) {
1097                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1098                                                 decoder->ip);
1099                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1100                                 return -ENOENT;
1101                         }
1102                         if (!(decoder->tnt.payload & BIT63)) {
1103                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1104                                                 decoder->ip);
1105                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1106                                 return -ENOENT;
1107                         }
1108                         decoder->tnt.count -= 1;
1109                         if (!decoder->tnt.count)
1110                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1111                         decoder->tnt.payload <<= 1;
1112                         decoder->state.from_ip = decoder->ip;
1113                         decoder->ip = decoder->ret_addr;
1114                         decoder->state.to_ip = decoder->ip;
1115                         return 0;
1116                 }
1117
1118                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1119                         /* Handle deferred TIPs */
1120                         err = intel_pt_get_next_packet(decoder);
1121                         if (err)
1122                                 return err;
1123                         if (decoder->packet.type != INTEL_PT_TIP ||
1124                             decoder->packet.count == 0) {
1125                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1126                                                 decoder->ip);
1127                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1128                                 decoder->pkt_step = 0;
1129                                 return -ENOENT;
1130                         }
1131                         intel_pt_set_last_ip(decoder);
1132                         decoder->state.from_ip = decoder->ip;
1133                         decoder->state.to_ip = decoder->last_ip;
1134                         decoder->ip = decoder->last_ip;
1135                         return 0;
1136                 }
1137
1138                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1139                         decoder->tnt.count -= 1;
1140                         if (!decoder->tnt.count)
1141                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1142                         if (decoder->tnt.payload & BIT63) {
1143                                 decoder->tnt.payload <<= 1;
1144                                 decoder->state.from_ip = decoder->ip;
1145                                 decoder->ip += intel_pt_insn.length +
1146                                                intel_pt_insn.rel;
1147                                 decoder->state.to_ip = decoder->ip;
1148                                 return 0;
1149                         }
1150                         /* Instruction sample for a non-taken branch */
1151                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1152                                 decoder->tnt.payload <<= 1;
1153                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1154                                 decoder->state.from_ip = decoder->ip;
1155                                 decoder->state.to_ip = 0;
1156                                 decoder->ip += intel_pt_insn.length;
1157                                 return 0;
1158                         }
1159                         decoder->ip += intel_pt_insn.length;
1160                         if (!decoder->tnt.count)
1161                                 return -EAGAIN;
1162                         decoder->tnt.payload <<= 1;
1163                         continue;
1164                 }
1165
1166                 return intel_pt_bug(decoder);
1167         }
1168 }
1169
1170 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1171 {
1172         unsigned int fup_tx_flags;
1173         int err;
1174
1175         fup_tx_flags = decoder->packet.payload &
1176                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1177         err = intel_pt_get_next_packet(decoder);
1178         if (err)
1179                 return err;
1180         if (decoder->packet.type == INTEL_PT_FUP) {
1181                 decoder->fup_tx_flags = fup_tx_flags;
1182                 decoder->set_fup_tx_flags = true;
1183                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1184                         *no_tip = true;
1185         } else {
1186                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1187                                 decoder->pos);
1188                 intel_pt_update_in_tx(decoder);
1189         }
1190         return 0;
1191 }
1192
1193 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1194 {
1195         uint64_t timestamp;
1196
1197         decoder->have_tma = false;
1198
1199         if (decoder->ref_timestamp) {
1200                 timestamp = decoder->packet.payload |
1201                             (decoder->ref_timestamp & (0xffULL << 56));
1202                 if (timestamp < decoder->ref_timestamp) {
1203                         if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1204                                 timestamp += (1ULL << 56);
1205                 } else {
1206                         if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1207                                 timestamp -= (1ULL << 56);
1208                 }
1209                 decoder->tsc_timestamp = timestamp;
1210                 decoder->timestamp = timestamp;
1211                 decoder->ref_timestamp = 0;
1212                 decoder->timestamp_insn_cnt = 0;
1213         } else if (decoder->timestamp) {
1214                 timestamp = decoder->packet.payload |
1215                             (decoder->timestamp & (0xffULL << 56));
1216                 decoder->tsc_timestamp = timestamp;
1217                 if (timestamp < decoder->timestamp &&
1218                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1219                         intel_pt_log_to("Suppressing backwards timestamp",
1220                                         timestamp);
1221                         timestamp = decoder->timestamp;
1222                 }
1223                 if (timestamp < decoder->timestamp) {
1224                         intel_pt_log_to("Wraparound timestamp", timestamp);
1225                         timestamp += (1ULL << 56);
1226                         decoder->tsc_timestamp = timestamp;
1227                 }
1228                 decoder->timestamp = timestamp;
1229                 decoder->timestamp_insn_cnt = 0;
1230         }
1231
1232         if (decoder->last_packet_type == INTEL_PT_CYC) {
1233                 decoder->cyc_ref_timestamp = decoder->timestamp;
1234                 decoder->cycle_cnt = 0;
1235                 decoder->have_calc_cyc_to_tsc = false;
1236                 intel_pt_calc_cyc_to_tsc(decoder, false);
1237         }
1238
1239         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1240 }
1241
1242 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1243 {
1244         intel_pt_log("ERROR: Buffer overflow\n");
1245         intel_pt_clear_tx_flags(decoder);
1246         decoder->have_tma = false;
1247         decoder->cbr = 0;
1248         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1249         decoder->overflow = true;
1250         return -EOVERFLOW;
1251 }
1252
1253 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1254 {
1255         uint32_t ctc = decoder->packet.payload;
1256         uint32_t fc = decoder->packet.count;
1257         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1258
1259         if (!decoder->tsc_ctc_ratio_d)
1260                 return;
1261
1262         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1263         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1264         if (decoder->tsc_ctc_mult) {
1265                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1266         } else {
1267                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1268                                                   decoder->tsc_ctc_ratio_n,
1269                                                   decoder->tsc_ctc_ratio_d);
1270         }
1271         decoder->ctc_delta = 0;
1272         decoder->have_tma = true;
1273         decoder->fixup_last_mtc = true;
1274         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1275                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1276 }
1277
1278 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1279 {
1280         uint64_t timestamp;
1281         uint32_t mtc, mtc_delta;
1282
1283         if (!decoder->have_tma)
1284                 return;
1285
1286         mtc = decoder->packet.payload;
1287
1288         if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1289                 decoder->fixup_last_mtc = false;
1290                 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1291                                         &decoder->last_mtc);
1292         }
1293
1294         if (mtc > decoder->last_mtc)
1295                 mtc_delta = mtc - decoder->last_mtc;
1296         else
1297                 mtc_delta = mtc + 256 - decoder->last_mtc;
1298
1299         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1300
1301         if (decoder->tsc_ctc_mult) {
1302                 timestamp = decoder->ctc_timestamp +
1303                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1304         } else {
1305                 timestamp = decoder->ctc_timestamp +
1306                             multdiv(decoder->ctc_delta,
1307                                     decoder->tsc_ctc_ratio_n,
1308                                     decoder->tsc_ctc_ratio_d);
1309         }
1310
1311         if (timestamp < decoder->timestamp)
1312                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1313                              timestamp, decoder->timestamp);
1314         else
1315                 decoder->timestamp = timestamp;
1316
1317         decoder->timestamp_insn_cnt = 0;
1318         decoder->last_mtc = mtc;
1319
1320         if (decoder->last_packet_type == INTEL_PT_CYC) {
1321                 decoder->cyc_ref_timestamp = decoder->timestamp;
1322                 decoder->cycle_cnt = 0;
1323                 decoder->have_calc_cyc_to_tsc = false;
1324                 intel_pt_calc_cyc_to_tsc(decoder, true);
1325         }
1326 }
1327
1328 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1329 {
1330         unsigned int cbr = decoder->packet.payload;
1331
1332         if (decoder->cbr == cbr)
1333                 return;
1334
1335         decoder->cbr = cbr;
1336         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1337 }
1338
1339 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1340 {
1341         uint64_t timestamp = decoder->cyc_ref_timestamp;
1342
1343         decoder->have_cyc = true;
1344
1345         decoder->cycle_cnt += decoder->packet.payload;
1346
1347         if (!decoder->cyc_ref_timestamp)
1348                 return;
1349
1350         if (decoder->have_calc_cyc_to_tsc)
1351                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1352         else if (decoder->cbr)
1353                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1354         else
1355                 return;
1356
1357         if (timestamp < decoder->timestamp)
1358                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1359                              timestamp, decoder->timestamp);
1360         else
1361                 decoder->timestamp = timestamp;
1362
1363         decoder->timestamp_insn_cnt = 0;
1364 }
1365
1366 /* Walk PSB+ packets when already in sync. */
1367 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1368 {
1369         int err;
1370
1371         while (1) {
1372                 err = intel_pt_get_next_packet(decoder);
1373                 if (err)
1374                         return err;
1375
1376                 switch (decoder->packet.type) {
1377                 case INTEL_PT_PSBEND:
1378                         return 0;
1379
1380                 case INTEL_PT_TIP_PGD:
1381                 case INTEL_PT_TIP_PGE:
1382                 case INTEL_PT_TIP:
1383                 case INTEL_PT_TNT:
1384                 case INTEL_PT_TRACESTOP:
1385                 case INTEL_PT_BAD:
1386                 case INTEL_PT_PSB:
1387                         decoder->have_tma = false;
1388                         intel_pt_log("ERROR: Unexpected packet\n");
1389                         return -EAGAIN;
1390
1391                 case INTEL_PT_OVF:
1392                         return intel_pt_overflow(decoder);
1393
1394                 case INTEL_PT_TSC:
1395                         intel_pt_calc_tsc_timestamp(decoder);
1396                         break;
1397
1398                 case INTEL_PT_TMA:
1399                         intel_pt_calc_tma(decoder);
1400                         break;
1401
1402                 case INTEL_PT_CBR:
1403                         intel_pt_calc_cbr(decoder);
1404                         break;
1405
1406                 case INTEL_PT_MODE_EXEC:
1407                         decoder->exec_mode = decoder->packet.payload;
1408                         break;
1409
1410                 case INTEL_PT_PIP:
1411                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1412                         break;
1413
1414                 case INTEL_PT_FUP:
1415                         decoder->pge = true;
1416                         intel_pt_set_last_ip(decoder);
1417                         break;
1418
1419                 case INTEL_PT_MODE_TSX:
1420                         intel_pt_update_in_tx(decoder);
1421                         break;
1422
1423                 case INTEL_PT_MTC:
1424                         intel_pt_calc_mtc_timestamp(decoder);
1425                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1426                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1427                         break;
1428
1429                 case INTEL_PT_CYC:
1430                 case INTEL_PT_VMCS:
1431                 case INTEL_PT_MNT:
1432                 case INTEL_PT_PAD:
1433                 default:
1434                         break;
1435                 }
1436         }
1437 }
1438
1439 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1440 {
1441         int err;
1442
1443         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1444                 decoder->tx_flags = 0;
1445                 decoder->state.flags &= ~INTEL_PT_IN_TX;
1446                 decoder->state.flags |= INTEL_PT_ABORT_TX;
1447         } else {
1448                 decoder->state.flags |= INTEL_PT_ASYNC;
1449         }
1450
1451         while (1) {
1452                 err = intel_pt_get_next_packet(decoder);
1453                 if (err)
1454                         return err;
1455
1456                 switch (decoder->packet.type) {
1457                 case INTEL_PT_TNT:
1458                 case INTEL_PT_FUP:
1459                 case INTEL_PT_TRACESTOP:
1460                 case INTEL_PT_PSB:
1461                 case INTEL_PT_TSC:
1462                 case INTEL_PT_TMA:
1463                 case INTEL_PT_CBR:
1464                 case INTEL_PT_MODE_TSX:
1465                 case INTEL_PT_BAD:
1466                 case INTEL_PT_PSBEND:
1467                         intel_pt_log("ERROR: Missing TIP after FUP\n");
1468                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
1469                         return -ENOENT;
1470
1471                 case INTEL_PT_OVF:
1472                         return intel_pt_overflow(decoder);
1473
1474                 case INTEL_PT_TIP_PGD:
1475                         decoder->state.from_ip = decoder->ip;
1476                         decoder->state.to_ip = 0;
1477                         if (decoder->packet.count != 0) {
1478                                 intel_pt_set_ip(decoder);
1479                                 intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1480                                              decoder->ip);
1481                         }
1482                         decoder->pge = false;
1483                         decoder->continuous_period = false;
1484                         return 0;
1485
1486                 case INTEL_PT_TIP_PGE:
1487                         decoder->pge = true;
1488                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1489                                      decoder->ip);
1490                         decoder->state.from_ip = 0;
1491                         if (decoder->packet.count == 0) {
1492                                 decoder->state.to_ip = 0;
1493                         } else {
1494                                 intel_pt_set_ip(decoder);
1495                                 decoder->state.to_ip = decoder->ip;
1496                         }
1497                         return 0;
1498
1499                 case INTEL_PT_TIP:
1500                         decoder->state.from_ip = decoder->ip;
1501                         if (decoder->packet.count == 0) {
1502                                 decoder->state.to_ip = 0;
1503                         } else {
1504                                 intel_pt_set_ip(decoder);
1505                                 decoder->state.to_ip = decoder->ip;
1506                         }
1507                         return 0;
1508
1509                 case INTEL_PT_PIP:
1510                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1511                         break;
1512
1513                 case INTEL_PT_MTC:
1514                         intel_pt_calc_mtc_timestamp(decoder);
1515                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1516                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1517                         break;
1518
1519                 case INTEL_PT_CYC:
1520                         intel_pt_calc_cyc_timestamp(decoder);
1521                         break;
1522
1523                 case INTEL_PT_MODE_EXEC:
1524                         decoder->exec_mode = decoder->packet.payload;
1525                         break;
1526
1527                 case INTEL_PT_VMCS:
1528                 case INTEL_PT_MNT:
1529                 case INTEL_PT_PAD:
1530                         break;
1531
1532                 default:
1533                         return intel_pt_bug(decoder);
1534                 }
1535         }
1536 }
1537
1538 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1539 {
1540         bool no_tip = false;
1541         int err;
1542
1543         while (1) {
1544                 err = intel_pt_get_next_packet(decoder);
1545                 if (err)
1546                         return err;
1547 next:
1548                 switch (decoder->packet.type) {
1549                 case INTEL_PT_TNT:
1550                         if (!decoder->packet.count)
1551                                 break;
1552                         decoder->tnt = decoder->packet;
1553                         decoder->pkt_state = INTEL_PT_STATE_TNT;
1554                         err = intel_pt_walk_tnt(decoder);
1555                         if (err == -EAGAIN)
1556                                 break;
1557                         return err;
1558
1559                 case INTEL_PT_TIP_PGD:
1560                         if (decoder->packet.count != 0)
1561                                 intel_pt_set_last_ip(decoder);
1562                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1563                         return intel_pt_walk_tip(decoder);
1564
1565                 case INTEL_PT_TIP_PGE: {
1566                         decoder->pge = true;
1567                         if (decoder->packet.count == 0) {
1568                                 intel_pt_log_at("Skipping zero TIP.PGE",
1569                                                 decoder->pos);
1570                                 break;
1571                         }
1572                         intel_pt_set_ip(decoder);
1573                         decoder->state.from_ip = 0;
1574                         decoder->state.to_ip = decoder->ip;
1575                         return 0;
1576                 }
1577
1578                 case INTEL_PT_OVF:
1579                         return intel_pt_overflow(decoder);
1580
1581                 case INTEL_PT_TIP:
1582                         if (decoder->packet.count != 0)
1583                                 intel_pt_set_last_ip(decoder);
1584                         decoder->pkt_state = INTEL_PT_STATE_TIP;
1585                         return intel_pt_walk_tip(decoder);
1586
1587                 case INTEL_PT_FUP:
1588                         if (decoder->packet.count == 0) {
1589                                 intel_pt_log_at("Skipping zero FUP",
1590                                                 decoder->pos);
1591                                 no_tip = false;
1592                                 break;
1593                         }
1594                         intel_pt_set_last_ip(decoder);
1595                         err = intel_pt_walk_fup(decoder);
1596                         if (err != -EAGAIN) {
1597                                 if (err)
1598                                         return err;
1599                                 if (no_tip)
1600                                         decoder->pkt_state =
1601                                                 INTEL_PT_STATE_FUP_NO_TIP;
1602                                 else
1603                                         decoder->pkt_state = INTEL_PT_STATE_FUP;
1604                                 return 0;
1605                         }
1606                         if (no_tip) {
1607                                 no_tip = false;
1608                                 break;
1609                         }
1610                         return intel_pt_walk_fup_tip(decoder);
1611
1612                 case INTEL_PT_TRACESTOP:
1613                         decoder->pge = false;
1614                         decoder->continuous_period = false;
1615                         intel_pt_clear_tx_flags(decoder);
1616                         decoder->have_tma = false;
1617                         break;
1618
1619                 case INTEL_PT_PSB:
1620                         intel_pt_clear_stack(&decoder->stack);
1621                         err = intel_pt_walk_psbend(decoder);
1622                         if (err == -EAGAIN)
1623                                 goto next;
1624                         if (err)
1625                                 return err;
1626                         break;
1627
1628                 case INTEL_PT_PIP:
1629                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1630                         break;
1631
1632                 case INTEL_PT_MTC:
1633                         intel_pt_calc_mtc_timestamp(decoder);
1634                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1635                                 break;
1636                         /*
1637                          * Ensure that there has been an instruction since the
1638                          * last MTC.
1639                          */
1640                         if (!decoder->mtc_insn)
1641                                 break;
1642                         decoder->mtc_insn = false;
1643                         /* Ensure that there is a timestamp */
1644                         if (!decoder->timestamp)
1645                                 break;
1646                         decoder->state.type = INTEL_PT_INSTRUCTION;
1647                         decoder->state.from_ip = decoder->ip;
1648                         decoder->state.to_ip = 0;
1649                         decoder->mtc_insn = false;
1650                         return 0;
1651
1652                 case INTEL_PT_TSC:
1653                         intel_pt_calc_tsc_timestamp(decoder);
1654                         break;
1655
1656                 case INTEL_PT_TMA:
1657                         intel_pt_calc_tma(decoder);
1658                         break;
1659
1660                 case INTEL_PT_CYC:
1661                         intel_pt_calc_cyc_timestamp(decoder);
1662                         break;
1663
1664                 case INTEL_PT_CBR:
1665                         intel_pt_calc_cbr(decoder);
1666                         break;
1667
1668                 case INTEL_PT_MODE_EXEC:
1669                         decoder->exec_mode = decoder->packet.payload;
1670                         break;
1671
1672                 case INTEL_PT_MODE_TSX:
1673                         /* MODE_TSX need not be followed by FUP */
1674                         if (!decoder->pge) {
1675                                 intel_pt_update_in_tx(decoder);
1676                                 break;
1677                         }
1678                         err = intel_pt_mode_tsx(decoder, &no_tip);
1679                         if (err)
1680                                 return err;
1681                         goto next;
1682
1683                 case INTEL_PT_BAD: /* Does not happen */
1684                         return intel_pt_bug(decoder);
1685
1686                 case INTEL_PT_PSBEND:
1687                 case INTEL_PT_VMCS:
1688                 case INTEL_PT_MNT:
1689                 case INTEL_PT_PAD:
1690                         break;
1691
1692                 default:
1693                         return intel_pt_bug(decoder);
1694                 }
1695         }
1696 }
1697
1698 /* Walk PSB+ packets to get in sync. */
1699 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1700 {
1701         int err;
1702
1703         while (1) {
1704                 err = intel_pt_get_next_packet(decoder);
1705                 if (err)
1706                         return err;
1707
1708                 switch (decoder->packet.type) {
1709                 case INTEL_PT_TIP_PGD:
1710                         decoder->continuous_period = false;
1711                 case INTEL_PT_TIP_PGE:
1712                 case INTEL_PT_TIP:
1713                         intel_pt_log("ERROR: Unexpected packet\n");
1714                         return -ENOENT;
1715
1716                 case INTEL_PT_FUP:
1717                         decoder->pge = true;
1718                         if (decoder->last_ip || decoder->packet.count == 6 ||
1719                             decoder->packet.count == 0) {
1720                                 uint64_t current_ip = decoder->ip;
1721
1722                                 intel_pt_set_ip(decoder);
1723                                 if (current_ip)
1724                                         intel_pt_log_to("Setting IP",
1725                                                         decoder->ip);
1726                         }
1727                         break;
1728
1729                 case INTEL_PT_MTC:
1730                         intel_pt_calc_mtc_timestamp(decoder);
1731                         break;
1732
1733                 case INTEL_PT_TSC:
1734                         intel_pt_calc_tsc_timestamp(decoder);
1735                         break;
1736
1737                 case INTEL_PT_TMA:
1738                         intel_pt_calc_tma(decoder);
1739                         break;
1740
1741                 case INTEL_PT_CYC:
1742                         intel_pt_calc_cyc_timestamp(decoder);
1743                         break;
1744
1745                 case INTEL_PT_CBR:
1746                         intel_pt_calc_cbr(decoder);
1747                         break;
1748
1749                 case INTEL_PT_PIP:
1750                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1751                         break;
1752
1753                 case INTEL_PT_MODE_EXEC:
1754                         decoder->exec_mode = decoder->packet.payload;
1755                         break;
1756
1757                 case INTEL_PT_MODE_TSX:
1758                         intel_pt_update_in_tx(decoder);
1759                         break;
1760
1761                 case INTEL_PT_TRACESTOP:
1762                         decoder->pge = false;
1763                         decoder->continuous_period = false;
1764                         intel_pt_clear_tx_flags(decoder);
1765                 case INTEL_PT_TNT:
1766                         decoder->have_tma = false;
1767                         intel_pt_log("ERROR: Unexpected packet\n");
1768                         if (decoder->ip)
1769                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
1770                         else
1771                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1772                         return -ENOENT;
1773
1774                 case INTEL_PT_BAD: /* Does not happen */
1775                         return intel_pt_bug(decoder);
1776
1777                 case INTEL_PT_OVF:
1778                         return intel_pt_overflow(decoder);
1779
1780                 case INTEL_PT_PSBEND:
1781                         return 0;
1782
1783                 case INTEL_PT_PSB:
1784                 case INTEL_PT_VMCS:
1785                 case INTEL_PT_MNT:
1786                 case INTEL_PT_PAD:
1787                 default:
1788                         break;
1789                 }
1790         }
1791 }
1792
1793 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
1794 {
1795         int err;
1796
1797         while (1) {
1798                 err = intel_pt_get_next_packet(decoder);
1799                 if (err)
1800                         return err;
1801
1802                 switch (decoder->packet.type) {
1803                 case INTEL_PT_TIP_PGD:
1804                         decoder->continuous_period = false;
1805                 case INTEL_PT_TIP_PGE:
1806                 case INTEL_PT_TIP:
1807                         decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
1808                         if (decoder->last_ip || decoder->packet.count == 6 ||
1809                             decoder->packet.count == 0)
1810                                 intel_pt_set_ip(decoder);
1811                         if (decoder->ip)
1812                                 return 0;
1813                         break;
1814
1815                 case INTEL_PT_FUP:
1816                         if (decoder->overflow) {
1817                                 if (decoder->last_ip ||
1818                                     decoder->packet.count == 6 ||
1819                                     decoder->packet.count == 0)
1820                                         intel_pt_set_ip(decoder);
1821                                 if (decoder->ip)
1822                                         return 0;
1823                         }
1824                         if (decoder->packet.count)
1825                                 intel_pt_set_last_ip(decoder);
1826                         break;
1827
1828                 case INTEL_PT_MTC:
1829                         intel_pt_calc_mtc_timestamp(decoder);
1830                         break;
1831
1832                 case INTEL_PT_TSC:
1833                         intel_pt_calc_tsc_timestamp(decoder);
1834                         break;
1835
1836                 case INTEL_PT_TMA:
1837                         intel_pt_calc_tma(decoder);
1838                         break;
1839
1840                 case INTEL_PT_CYC:
1841                         intel_pt_calc_cyc_timestamp(decoder);
1842                         break;
1843
1844                 case INTEL_PT_CBR:
1845                         intel_pt_calc_cbr(decoder);
1846                         break;
1847
1848                 case INTEL_PT_PIP:
1849                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1850                         break;
1851
1852                 case INTEL_PT_MODE_EXEC:
1853                         decoder->exec_mode = decoder->packet.payload;
1854                         break;
1855
1856                 case INTEL_PT_MODE_TSX:
1857                         intel_pt_update_in_tx(decoder);
1858                         break;
1859
1860                 case INTEL_PT_OVF:
1861                         return intel_pt_overflow(decoder);
1862
1863                 case INTEL_PT_BAD: /* Does not happen */
1864                         return intel_pt_bug(decoder);
1865
1866                 case INTEL_PT_TRACESTOP:
1867                         decoder->pge = false;
1868                         decoder->continuous_period = false;
1869                         intel_pt_clear_tx_flags(decoder);
1870                         decoder->have_tma = false;
1871                         break;
1872
1873                 case INTEL_PT_PSB:
1874                         err = intel_pt_walk_psb(decoder);
1875                         if (err)
1876                                 return err;
1877                         if (decoder->ip) {
1878                                 /* Do not have a sample */
1879                                 decoder->state.type = 0;
1880                                 return 0;
1881                         }
1882                         break;
1883
1884                 case INTEL_PT_TNT:
1885                 case INTEL_PT_PSBEND:
1886                 case INTEL_PT_VMCS:
1887                 case INTEL_PT_MNT:
1888                 case INTEL_PT_PAD:
1889                 default:
1890                         break;
1891                 }
1892         }
1893 }
1894
1895 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
1896 {
1897         int err;
1898
1899         intel_pt_log("Scanning for full IP\n");
1900         err = intel_pt_walk_to_ip(decoder);
1901         if (err)
1902                 return err;
1903
1904         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1905         decoder->overflow = false;
1906
1907         decoder->state.from_ip = 0;
1908         decoder->state.to_ip = decoder->ip;
1909         intel_pt_log_to("Setting IP", decoder->ip);
1910
1911         return 0;
1912 }
1913
1914 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
1915 {
1916         const unsigned char *end = decoder->buf + decoder->len;
1917         size_t i;
1918
1919         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
1920                 if (i > decoder->len)
1921                         continue;
1922                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
1923                         return i;
1924         }
1925         return 0;
1926 }
1927
1928 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
1929 {
1930         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
1931         const char *psb = INTEL_PT_PSB_STR;
1932
1933         if (rest_psb > decoder->len ||
1934             memcmp(decoder->buf, psb + part_psb, rest_psb))
1935                 return 0;
1936
1937         return rest_psb;
1938 }
1939
1940 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
1941                                   int part_psb)
1942 {
1943         int rest_psb, ret;
1944
1945         decoder->pos += decoder->len;
1946         decoder->len = 0;
1947
1948         ret = intel_pt_get_next_data(decoder);
1949         if (ret)
1950                 return ret;
1951
1952         rest_psb = intel_pt_rest_psb(decoder, part_psb);
1953         if (!rest_psb)
1954                 return 0;
1955
1956         decoder->pos -= part_psb;
1957         decoder->next_buf = decoder->buf + rest_psb;
1958         decoder->next_len = decoder->len - rest_psb;
1959         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
1960         decoder->buf = decoder->temp_buf;
1961         decoder->len = INTEL_PT_PSB_LEN;
1962
1963         return 0;
1964 }
1965
1966 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
1967 {
1968         unsigned char *next;
1969         int ret;
1970
1971         intel_pt_log("Scanning for PSB\n");
1972         while (1) {
1973                 if (!decoder->len) {
1974                         ret = intel_pt_get_next_data(decoder);
1975                         if (ret)
1976                                 return ret;
1977                 }
1978
1979                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
1980                               INTEL_PT_PSB_LEN);
1981                 if (!next) {
1982                         int part_psb;
1983
1984                         part_psb = intel_pt_part_psb(decoder);
1985                         if (part_psb) {
1986                                 ret = intel_pt_get_split_psb(decoder, part_psb);
1987                                 if (ret)
1988                                         return ret;
1989                         } else {
1990                                 decoder->pos += decoder->len;
1991                                 decoder->len = 0;
1992                         }
1993                         continue;
1994                 }
1995
1996                 decoder->pkt_step = next - decoder->buf;
1997                 return intel_pt_get_next_packet(decoder);
1998         }
1999 }
2000
2001 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2002 {
2003         int err;
2004
2005         decoder->pge = false;
2006         decoder->continuous_period = false;
2007         decoder->last_ip = 0;
2008         decoder->ip = 0;
2009         intel_pt_clear_stack(&decoder->stack);
2010
2011         err = intel_pt_scan_for_psb(decoder);
2012         if (err)
2013                 return err;
2014
2015         decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2016
2017         err = intel_pt_walk_psb(decoder);
2018         if (err)
2019                 return err;
2020
2021         if (decoder->ip) {
2022                 decoder->state.type = 0; /* Do not have a sample */
2023                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2024         } else {
2025                 return intel_pt_sync_ip(decoder);
2026         }
2027
2028         return 0;
2029 }
2030
2031 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2032 {
2033         uint64_t est = decoder->timestamp_insn_cnt << 1;
2034
2035         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2036                 goto out;
2037
2038         est *= decoder->max_non_turbo_ratio;
2039         est /= decoder->cbr;
2040 out:
2041         return decoder->timestamp + est;
2042 }
2043
2044 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2045 {
2046         int err;
2047
2048         do {
2049                 decoder->state.type = INTEL_PT_BRANCH;
2050                 decoder->state.flags = 0;
2051
2052                 switch (decoder->pkt_state) {
2053                 case INTEL_PT_STATE_NO_PSB:
2054                         err = intel_pt_sync(decoder);
2055                         break;
2056                 case INTEL_PT_STATE_NO_IP:
2057                         decoder->last_ip = 0;
2058                         /* Fall through */
2059                 case INTEL_PT_STATE_ERR_RESYNC:
2060                         err = intel_pt_sync_ip(decoder);
2061                         break;
2062                 case INTEL_PT_STATE_IN_SYNC:
2063                         err = intel_pt_walk_trace(decoder);
2064                         break;
2065                 case INTEL_PT_STATE_TNT:
2066                         err = intel_pt_walk_tnt(decoder);
2067                         if (err == -EAGAIN)
2068                                 err = intel_pt_walk_trace(decoder);
2069                         break;
2070                 case INTEL_PT_STATE_TIP:
2071                 case INTEL_PT_STATE_TIP_PGD:
2072                         err = intel_pt_walk_tip(decoder);
2073                         break;
2074                 case INTEL_PT_STATE_FUP:
2075                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2076                         err = intel_pt_walk_fup(decoder);
2077                         if (err == -EAGAIN)
2078                                 err = intel_pt_walk_fup_tip(decoder);
2079                         else if (!err)
2080                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
2081                         break;
2082                 case INTEL_PT_STATE_FUP_NO_TIP:
2083                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2084                         err = intel_pt_walk_fup(decoder);
2085                         if (err == -EAGAIN)
2086                                 err = intel_pt_walk_trace(decoder);
2087                         break;
2088                 default:
2089                         err = intel_pt_bug(decoder);
2090                         break;
2091                 }
2092         } while (err == -ENOLINK);
2093
2094         decoder->state.err = err ? intel_pt_ext_err(err) : 0;
2095         decoder->state.timestamp = decoder->timestamp;
2096         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2097         decoder->state.cr3 = decoder->cr3;
2098         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2099
2100         if (err)
2101                 decoder->state.from_ip = decoder->ip;
2102
2103         return &decoder->state;
2104 }
2105
2106 static bool intel_pt_at_psb(unsigned char *buf, size_t len)
2107 {
2108         if (len < INTEL_PT_PSB_LEN)
2109                 return false;
2110         return memmem(buf, INTEL_PT_PSB_LEN, INTEL_PT_PSB_STR,
2111                       INTEL_PT_PSB_LEN);
2112 }
2113
2114 /**
2115  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2116  * @buf: pointer to buffer pointer
2117  * @len: size of buffer
2118  *
2119  * Updates the buffer pointer to point to the start of the next PSB packet if
2120  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
2121  * @len is adjusted accordingly.
2122  *
2123  * Return: %true if a PSB packet is found, %false otherwise.
2124  */
2125 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2126 {
2127         unsigned char *next;
2128
2129         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2130         if (next) {
2131                 *len -= next - *buf;
2132                 *buf = next;
2133                 return true;
2134         }
2135         return false;
2136 }
2137
2138 /**
2139  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2140  *                     packet.
2141  * @buf: pointer to buffer pointer
2142  * @len: size of buffer
2143  *
2144  * Updates the buffer pointer to point to the start of the following PSB packet
2145  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2146  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
2147  *
2148  * Return: %true if a PSB packet is found, %false otherwise.
2149  */
2150 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2151 {
2152         unsigned char *next;
2153
2154         if (!*len)
2155                 return false;
2156
2157         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2158         if (next) {
2159                 *len -= next - *buf;
2160                 *buf = next;
2161                 return true;
2162         }
2163         return false;
2164 }
2165
2166 /**
2167  * intel_pt_last_psb - find the last PSB packet in a buffer.
2168  * @buf: buffer
2169  * @len: size of buffer
2170  *
2171  * This function finds the last PSB in a buffer.
2172  *
2173  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2174  */
2175 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2176 {
2177         const char *n = INTEL_PT_PSB_STR;
2178         unsigned char *p;
2179         size_t k;
2180
2181         if (len < INTEL_PT_PSB_LEN)
2182                 return NULL;
2183
2184         k = len - INTEL_PT_PSB_LEN + 1;
2185         while (1) {
2186                 p = memrchr(buf, n[0], k);
2187                 if (!p)
2188                         return NULL;
2189                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2190                         return p;
2191                 k = p - buf;
2192                 if (!k)
2193                         return NULL;
2194         }
2195 }
2196
2197 /**
2198  * intel_pt_next_tsc - find and return next TSC.
2199  * @buf: buffer
2200  * @len: size of buffer
2201  * @tsc: TSC value returned
2202  *
2203  * Find a TSC packet in @buf and return the TSC value.  This function assumes
2204  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2205  * PSBEND packet is found.
2206  *
2207  * Return: %true if TSC is found, false otherwise.
2208  */
2209 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc)
2210 {
2211         struct intel_pt_pkt packet;
2212         int ret;
2213
2214         while (len) {
2215                 ret = intel_pt_get_packet(buf, len, &packet);
2216                 if (ret <= 0)
2217                         return false;
2218                 if (packet.type == INTEL_PT_TSC) {
2219                         *tsc = packet.payload;
2220                         return true;
2221                 }
2222                 if (packet.type == INTEL_PT_PSBEND)
2223                         return false;
2224                 buf += ret;
2225                 len -= ret;
2226         }
2227         return false;
2228 }
2229
2230 /**
2231  * intel_pt_tsc_cmp - compare 7-byte TSCs.
2232  * @tsc1: first TSC to compare
2233  * @tsc2: second TSC to compare
2234  *
2235  * This function compares 7-byte TSC values allowing for the possibility that
2236  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
2237  * around so for that purpose this function assumes the absolute difference is
2238  * less than half the maximum difference.
2239  *
2240  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2241  * after @tsc2.
2242  */
2243 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2244 {
2245         const uint64_t halfway = (1ULL << 55);
2246
2247         if (tsc1 == tsc2)
2248                 return 0;
2249
2250         if (tsc1 < tsc2) {
2251                 if (tsc2 - tsc1 < halfway)
2252                         return -1;
2253                 else
2254                         return 1;
2255         } else {
2256                 if (tsc1 - tsc2 < halfway)
2257                         return 1;
2258                 else
2259                         return -1;
2260         }
2261 }
2262
2263 /**
2264  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2265  *                             using TSC.
2266  * @buf_a: first buffer
2267  * @len_a: size of first buffer
2268  * @buf_b: second buffer
2269  * @len_b: size of second buffer
2270  *
2271  * If the trace contains TSC we can look at the last TSC of @buf_a and the
2272  * first TSC of @buf_b in order to determine if the buffers overlap, and then
2273  * walk forward in @buf_b until a later TSC is found.  A precondition is that
2274  * @buf_a and @buf_b are positioned at a PSB.
2275  *
2276  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2277  * @buf_b + @len_b if there is no non-overlapped data.
2278  */
2279 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2280                                                 size_t len_a,
2281                                                 unsigned char *buf_b,
2282                                                 size_t len_b)
2283 {
2284         uint64_t tsc_a, tsc_b;
2285         unsigned char *p;
2286         size_t len;
2287
2288         p = intel_pt_last_psb(buf_a, len_a);
2289         if (!p)
2290                 return buf_b; /* No PSB in buf_a => no overlap */
2291
2292         len = len_a - (p - buf_a);
2293         if (!intel_pt_next_tsc(p, len, &tsc_a)) {
2294                 /* The last PSB+ in buf_a is incomplete, so go back one more */
2295                 len_a -= len;
2296                 p = intel_pt_last_psb(buf_a, len_a);
2297                 if (!p)
2298                         return buf_b; /* No full PSB+ => assume no overlap */
2299                 len = len_a - (p - buf_a);
2300                 if (!intel_pt_next_tsc(p, len, &tsc_a))
2301                         return buf_b; /* No TSC in buf_a => assume no overlap */
2302         }
2303
2304         while (1) {
2305                 /* Ignore PSB+ with no TSC */
2306                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b) &&
2307                     intel_pt_tsc_cmp(tsc_a, tsc_b) < 0)
2308                         return buf_b; /* tsc_a < tsc_b => no overlap */
2309
2310                 if (!intel_pt_step_psb(&buf_b, &len_b))
2311                         return buf_b + len_b; /* No PSB in buf_b => no data */
2312         }
2313 }
2314
2315 /**
2316  * intel_pt_find_overlap - determine start of non-overlapped trace data.
2317  * @buf_a: first buffer
2318  * @len_a: size of first buffer
2319  * @buf_b: second buffer
2320  * @len_b: size of second buffer
2321  * @have_tsc: can use TSC packets to detect overlap
2322  *
2323  * When trace samples or snapshots are recorded there is the possibility that
2324  * the data overlaps.  Note that, for the purposes of decoding, data is only
2325  * useful if it begins with a PSB packet.
2326  *
2327  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2328  * @buf_b + @len_b if there is no non-overlapped data.
2329  */
2330 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2331                                      unsigned char *buf_b, size_t len_b,
2332                                      bool have_tsc)
2333 {
2334         unsigned char *found;
2335
2336         /* Buffer 'b' must start at PSB so throw away everything before that */
2337         if (!intel_pt_next_psb(&buf_b, &len_b))
2338                 return buf_b + len_b; /* No PSB */
2339
2340         if (!intel_pt_next_psb(&buf_a, &len_a))
2341                 return buf_b; /* No overlap */
2342
2343         if (have_tsc) {
2344                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b);
2345                 if (found)
2346                         return found;
2347         }
2348
2349         /*
2350          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2351          * we can ignore the first part of buffer 'a'.
2352          */
2353         while (len_b < len_a) {
2354                 if (!intel_pt_step_psb(&buf_a, &len_a))
2355                         return buf_b; /* No overlap */
2356         }
2357
2358         /* Now len_b >= len_a */
2359         if (len_b > len_a) {
2360                 /* The leftover buffer 'b' must start at a PSB */
2361                 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2362                         if (!intel_pt_step_psb(&buf_a, &len_a))
2363                                 return buf_b; /* No overlap */
2364                 }
2365         }
2366
2367         while (1) {
2368                 /* Potential overlap so check the bytes */
2369                 found = memmem(buf_a, len_a, buf_b, len_a);
2370                 if (found)
2371                         return buf_b + len_a;
2372
2373                 /* Try again at next PSB in buffer 'a' */
2374                 if (!intel_pt_step_psb(&buf_a, &len_a))
2375                         return buf_b; /* No overlap */
2376
2377                 /* The leftover buffer 'b' must start at a PSB */
2378                 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2379                         if (!intel_pt_step_psb(&buf_a, &len_a))
2380                                 return buf_b; /* No overlap */
2381                 }
2382         }
2383 }