2 * intel_pt_decoder.c: Intel Processor Trace support
3 * Copyright (c) 2013-2014, Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
29 #include "intel-pt-insn-decoder.h"
30 #include "intel-pt-pkt-decoder.h"
31 #include "intel-pt-decoder.h"
32 #include "intel-pt-log.h"
34 #define INTEL_PT_BLK_SIZE 1024
36 #define BIT63 (((uint64_t)1 << 63))
38 #define INTEL_PT_RETURN 1
40 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
41 #define INTEL_PT_MAX_LOOPS 10000
44 struct intel_pt_blk *prev;
45 uint64_t ip[INTEL_PT_BLK_SIZE];
48 struct intel_pt_stack {
49 struct intel_pt_blk *blk;
50 struct intel_pt_blk *spare;
54 enum intel_pt_pkt_state {
55 INTEL_PT_STATE_NO_PSB,
57 INTEL_PT_STATE_ERR_RESYNC,
58 INTEL_PT_STATE_IN_SYNC,
61 INTEL_PT_STATE_TIP_PGD,
63 INTEL_PT_STATE_FUP_NO_TIP,
66 #ifdef INTEL_PT_STRICT
67 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB
68 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB
69 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB
70 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB
72 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
73 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP
74 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC
75 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC
78 struct intel_pt_decoder {
79 int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
80 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
81 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
82 uint64_t max_insn_cnt, void *data);
84 struct intel_pt_state state;
85 const unsigned char *buf;
87 bool return_compression;
98 uint64_t tsc_timestamp;
99 uint64_t ref_timestamp;
101 uint64_t ctc_timestamp;
104 uint64_t cyc_ref_timestamp;
106 uint32_t tsc_ctc_ratio_n;
107 uint32_t tsc_ctc_ratio_d;
108 uint32_t tsc_ctc_mult;
110 uint32_t ctc_rem_mask;
112 struct intel_pt_stack stack;
113 enum intel_pt_pkt_state pkt_state;
114 struct intel_pt_pkt packet;
115 struct intel_pt_pkt tnt;
118 int last_packet_type;
120 unsigned int max_non_turbo_ratio;
121 double max_non_turbo_ratio_fp;
122 double cbr_cyc_to_tsc;
123 double calc_cyc_to_tsc;
124 bool have_calc_cyc_to_tsc;
126 unsigned int insn_bytes;
130 enum intel_pt_period_type period_type;
131 uint64_t tot_insn_cnt;
132 uint64_t period_insn_cnt;
133 uint64_t period_mask;
134 uint64_t period_ticks;
135 uint64_t last_masked_timestamp;
136 bool continuous_period;
138 bool set_fup_tx_flags;
139 unsigned int fup_tx_flags;
140 unsigned int tx_flags;
141 uint64_t timestamp_insn_cnt;
146 const unsigned char *next_buf;
148 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
151 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
155 for (i = 0; x != 1; i++)
161 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
163 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
166 period = intel_pt_lower_power_of_2(decoder->period);
167 decoder->period_mask = ~(period - 1);
168 decoder->period_ticks = period;
172 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
176 return (t / d) * n + ((t % d) * n) / d;
179 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
181 struct intel_pt_decoder *decoder;
183 if (!params->get_trace || !params->walk_insn)
186 decoder = zalloc(sizeof(struct intel_pt_decoder));
190 decoder->get_trace = params->get_trace;
191 decoder->walk_insn = params->walk_insn;
192 decoder->data = params->data;
193 decoder->return_compression = params->return_compression;
195 decoder->sign_bit = (uint64_t)1 << 47;
196 decoder->sign_bits = ~(((uint64_t)1 << 48) - 1);
198 decoder->period = params->period;
199 decoder->period_type = params->period_type;
201 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio;
202 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
204 intel_pt_setup_period(decoder);
206 decoder->mtc_shift = params->mtc_period;
207 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
209 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
210 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
212 if (!decoder->tsc_ctc_ratio_n)
213 decoder->tsc_ctc_ratio_d = 0;
215 if (decoder->tsc_ctc_ratio_d) {
216 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
217 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
218 decoder->tsc_ctc_ratio_d;
221 * Allow for timestamps appearing to backwards because a TSC
222 * packet has slipped past a MTC packet, so allow 2 MTC ticks
225 decoder->tsc_slip = multdiv(2 << decoder->mtc_shift,
226 decoder->tsc_ctc_ratio_n,
227 decoder->tsc_ctc_ratio_d);
229 /* ... or 0x100 paranoia */
230 if (decoder->tsc_slip < 0x100)
231 decoder->tsc_slip = 0x100;
233 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
234 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
235 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
236 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
237 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
242 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
244 struct intel_pt_blk *blk = stack->blk;
246 stack->blk = blk->prev;
253 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
258 intel_pt_pop_blk(stack);
261 stack->pos = INTEL_PT_BLK_SIZE;
263 return stack->blk->ip[--stack->pos];
266 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
268 struct intel_pt_blk *blk;
274 blk = malloc(sizeof(struct intel_pt_blk));
279 blk->prev = stack->blk;
285 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
289 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
290 err = intel_pt_alloc_blk(stack);
295 stack->blk->ip[stack->pos++] = ip;
299 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
302 intel_pt_pop_blk(stack);
306 static void intel_pt_free_stack(struct intel_pt_stack *stack)
308 intel_pt_clear_stack(stack);
310 zfree(&stack->spare);
313 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
315 intel_pt_free_stack(&decoder->stack);
319 static int intel_pt_ext_err(int code)
323 return INTEL_PT_ERR_NOMEM;
325 return INTEL_PT_ERR_INTERN;
327 return INTEL_PT_ERR_BADPKT;
329 return INTEL_PT_ERR_NODATA;
331 return INTEL_PT_ERR_NOINSN;
333 return INTEL_PT_ERR_MISMAT;
335 return INTEL_PT_ERR_OVR;
337 return INTEL_PT_ERR_LOST;
339 return INTEL_PT_ERR_NELOOP;
341 return INTEL_PT_ERR_UNK;
345 static const char *intel_pt_err_msgs[] = {
346 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed",
347 [INTEL_PT_ERR_INTERN] = "Internal error",
348 [INTEL_PT_ERR_BADPKT] = "Bad packet",
349 [INTEL_PT_ERR_NODATA] = "No more data",
350 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
351 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
352 [INTEL_PT_ERR_OVR] = "Overflow packet",
353 [INTEL_PT_ERR_LOST] = "Lost trace data",
354 [INTEL_PT_ERR_UNK] = "Unknown error!",
355 [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
358 int intel_pt__strerror(int code, char *buf, size_t buflen)
360 if (code < 1 || code > INTEL_PT_ERR_MAX)
361 code = INTEL_PT_ERR_UNK;
362 strlcpy(buf, intel_pt_err_msgs[code], buflen);
366 static uint64_t intel_pt_calc_ip(struct intel_pt_decoder *decoder,
367 const struct intel_pt_pkt *packet,
372 switch (packet->count) {
374 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
378 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
382 ip = packet->payload;
388 if (ip & decoder->sign_bit)
389 return ip | decoder->sign_bits;
394 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
396 decoder->last_ip = intel_pt_calc_ip(decoder, &decoder->packet,
400 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
402 intel_pt_set_last_ip(decoder);
403 decoder->ip = decoder->last_ip;
406 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
408 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
412 static int intel_pt_bug(struct intel_pt_decoder *decoder)
414 intel_pt_log("ERROR: Internal error\n");
415 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
419 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
421 decoder->tx_flags = 0;
424 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
426 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
429 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
431 intel_pt_clear_tx_flags(decoder);
432 decoder->have_tma = false;
433 decoder->pkt_len = 1;
434 decoder->pkt_step = 1;
435 intel_pt_decoder_log_packet(decoder);
436 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
437 intel_pt_log("ERROR: Bad packet\n");
438 decoder->pkt_state = INTEL_PT_STATE_ERR1;
443 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
445 struct intel_pt_buffer buffer = { .buf = 0, };
448 decoder->pkt_step = 0;
450 intel_pt_log("Getting more data\n");
451 ret = decoder->get_trace(&buffer, decoder->data);
454 decoder->buf = buffer.buf;
455 decoder->len = buffer.len;
457 intel_pt_log("No more data\n");
460 if (!buffer.consecutive) {
462 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
463 decoder->ref_timestamp = buffer.ref_timestamp;
464 decoder->timestamp = 0;
465 decoder->have_tma = false;
466 decoder->state.trace_nr = buffer.trace_nr;
467 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
468 decoder->ref_timestamp);
475 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
477 if (!decoder->next_buf)
478 return intel_pt_get_data(decoder);
480 decoder->buf = decoder->next_buf;
481 decoder->len = decoder->next_len;
482 decoder->next_buf = 0;
483 decoder->next_len = 0;
487 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
489 unsigned char *buf = decoder->temp_buf;
490 size_t old_len, len, n;
493 old_len = decoder->len;
495 memcpy(buf, decoder->buf, len);
497 ret = intel_pt_get_data(decoder);
499 decoder->pos += old_len;
500 return ret < 0 ? ret : -EINVAL;
503 n = INTEL_PT_PKT_MAX_SZ - len;
504 if (n > decoder->len)
506 memcpy(buf + len, decoder->buf, n);
509 ret = intel_pt_get_packet(buf, len, &decoder->packet);
510 if (ret < (int)old_len) {
511 decoder->next_buf = decoder->buf;
512 decoder->next_len = decoder->len;
514 decoder->len = old_len;
515 return intel_pt_bad_packet(decoder);
518 decoder->next_buf = decoder->buf + (ret - old_len);
519 decoder->next_len = decoder->len - (ret - old_len);
527 struct intel_pt_pkt_info {
528 struct intel_pt_decoder *decoder;
529 struct intel_pt_pkt packet;
532 int last_packet_type;
536 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
538 /* Lookahead packets in current buffer */
539 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
540 intel_pt_pkt_cb_t cb, void *data)
542 struct intel_pt_pkt_info pkt_info;
543 const unsigned char *buf = decoder->buf;
544 size_t len = decoder->len;
547 pkt_info.decoder = decoder;
548 pkt_info.pos = decoder->pos;
549 pkt_info.pkt_len = decoder->pkt_step;
550 pkt_info.last_packet_type = decoder->last_packet_type;
551 pkt_info.data = data;
555 pkt_info.pos += pkt_info.pkt_len;
556 buf += pkt_info.pkt_len;
557 len -= pkt_info.pkt_len;
560 return INTEL_PT_NEED_MORE_BYTES;
562 ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
564 return INTEL_PT_NEED_MORE_BYTES;
568 pkt_info.pkt_len = ret;
569 } while (pkt_info.packet.type == INTEL_PT_PAD);
575 pkt_info.last_packet_type = pkt_info.packet.type;
579 struct intel_pt_calc_cyc_to_tsc_info {
583 uint64_t ctc_timestamp;
585 uint64_t tsc_timestamp;
590 double cbr_cyc_to_tsc;
594 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
595 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
596 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
597 * packet by copying the missing bits from the current MTC assuming the least
598 * difference between the two, and that the current MTC comes after last_mtc.
600 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
603 uint32_t first_missing_bit = 1U << (16 - mtc_shift);
604 uint32_t mask = ~(first_missing_bit - 1);
606 *last_mtc |= mtc & mask;
607 if (*last_mtc >= mtc) {
608 *last_mtc -= first_missing_bit;
613 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
615 struct intel_pt_decoder *decoder = pkt_info->decoder;
616 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
620 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
622 switch (pkt_info->packet.type) {
624 case INTEL_PT_TIP_PGE:
629 case INTEL_PT_MODE_EXEC:
630 case INTEL_PT_MODE_TSX:
631 case INTEL_PT_PSBEND:
641 mtc = pkt_info->packet.payload;
642 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
643 data->fixup_last_mtc = false;
644 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
647 if (mtc > data->last_mtc)
648 mtc_delta = mtc - data->last_mtc;
650 mtc_delta = mtc + 256 - data->last_mtc;
651 data->ctc_delta += mtc_delta << decoder->mtc_shift;
652 data->last_mtc = mtc;
654 if (decoder->tsc_ctc_mult) {
655 timestamp = data->ctc_timestamp +
656 data->ctc_delta * decoder->tsc_ctc_mult;
658 timestamp = data->ctc_timestamp +
659 multdiv(data->ctc_delta,
660 decoder->tsc_ctc_ratio_n,
661 decoder->tsc_ctc_ratio_d);
664 if (timestamp < data->timestamp)
667 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
668 data->timestamp = timestamp;
675 timestamp = pkt_info->packet.payload |
676 (data->timestamp & (0xffULL << 56));
677 if (data->from_mtc && timestamp < data->timestamp &&
678 data->timestamp - timestamp < decoder->tsc_slip)
680 if (timestamp < data->timestamp)
681 timestamp += (1ULL << 56);
682 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
685 data->tsc_timestamp = timestamp;
686 data->timestamp = timestamp;
695 if (!decoder->tsc_ctc_ratio_d)
698 ctc = pkt_info->packet.payload;
699 fc = pkt_info->packet.count;
700 ctc_rem = ctc & decoder->ctc_rem_mask;
702 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
704 data->ctc_timestamp = data->tsc_timestamp - fc;
705 if (decoder->tsc_ctc_mult) {
706 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
708 data->ctc_timestamp -=
709 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
710 decoder->tsc_ctc_ratio_d);
714 data->have_tma = true;
715 data->fixup_last_mtc = true;
720 data->cycle_cnt += pkt_info->packet.payload;
724 cbr = pkt_info->packet.payload;
725 if (data->cbr && data->cbr != cbr)
728 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
731 case INTEL_PT_TIP_PGD:
732 case INTEL_PT_TRACESTOP:
734 case INTEL_PT_BAD: /* Does not happen */
739 if (!data->cbr && decoder->cbr) {
740 data->cbr = decoder->cbr;
741 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
744 if (!data->cycle_cnt)
747 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
749 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
750 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
751 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
752 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
756 decoder->calc_cyc_to_tsc = cyc_to_tsc;
757 decoder->have_calc_cyc_to_tsc = true;
760 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
761 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
763 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
764 cyc_to_tsc, pkt_info->pos);
770 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
773 struct intel_pt_calc_cyc_to_tsc_info data = {
776 .last_mtc = decoder->last_mtc,
777 .ctc_timestamp = decoder->ctc_timestamp,
778 .ctc_delta = decoder->ctc_delta,
779 .tsc_timestamp = decoder->tsc_timestamp,
780 .timestamp = decoder->timestamp,
781 .have_tma = decoder->have_tma,
782 .fixup_last_mtc = decoder->fixup_last_mtc,
783 .from_mtc = from_mtc,
787 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
790 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
794 decoder->last_packet_type = decoder->packet.type;
797 decoder->pos += decoder->pkt_step;
798 decoder->buf += decoder->pkt_step;
799 decoder->len -= decoder->pkt_step;
802 ret = intel_pt_get_next_data(decoder);
807 ret = intel_pt_get_packet(decoder->buf, decoder->len,
809 if (ret == INTEL_PT_NEED_MORE_BYTES &&
810 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
811 ret = intel_pt_get_split_packet(decoder);
816 return intel_pt_bad_packet(decoder);
818 decoder->pkt_len = ret;
819 decoder->pkt_step = ret;
820 intel_pt_decoder_log_packet(decoder);
821 } while (decoder->packet.type == INTEL_PT_PAD);
826 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
828 uint64_t timestamp, masked_timestamp;
830 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
831 masked_timestamp = timestamp & decoder->period_mask;
832 if (decoder->continuous_period) {
833 if (masked_timestamp != decoder->last_masked_timestamp)
837 masked_timestamp = timestamp & decoder->period_mask;
838 if (masked_timestamp != decoder->last_masked_timestamp) {
839 decoder->last_masked_timestamp = masked_timestamp;
840 decoder->continuous_period = true;
843 return decoder->period_ticks - (timestamp - masked_timestamp);
846 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
848 switch (decoder->period_type) {
849 case INTEL_PT_PERIOD_INSTRUCTIONS:
850 return decoder->period - decoder->period_insn_cnt;
851 case INTEL_PT_PERIOD_TICKS:
852 return intel_pt_next_period(decoder);
853 case INTEL_PT_PERIOD_NONE:
854 case INTEL_PT_PERIOD_MTC:
860 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
862 uint64_t timestamp, masked_timestamp;
864 switch (decoder->period_type) {
865 case INTEL_PT_PERIOD_INSTRUCTIONS:
866 decoder->period_insn_cnt = 0;
868 case INTEL_PT_PERIOD_TICKS:
869 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
870 masked_timestamp = timestamp & decoder->period_mask;
871 decoder->last_masked_timestamp = masked_timestamp;
873 case INTEL_PT_PERIOD_NONE:
874 case INTEL_PT_PERIOD_MTC:
879 decoder->state.type |= INTEL_PT_INSTRUCTION;
882 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
883 struct intel_pt_insn *intel_pt_insn, uint64_t ip)
885 uint64_t max_insn_cnt, insn_cnt = 0;
888 if (!decoder->mtc_insn)
889 decoder->mtc_insn = true;
891 max_insn_cnt = intel_pt_next_sample(decoder);
893 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
894 max_insn_cnt, decoder->data);
896 decoder->tot_insn_cnt += insn_cnt;
897 decoder->timestamp_insn_cnt += insn_cnt;
898 decoder->period_insn_cnt += insn_cnt;
901 decoder->no_progress = 0;
902 decoder->pkt_state = INTEL_PT_STATE_ERR2;
903 intel_pt_log_at("ERROR: Failed to get instruction",
910 if (ip && decoder->ip == ip) {
915 if (max_insn_cnt && insn_cnt >= max_insn_cnt)
916 intel_pt_sample_insn(decoder);
918 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
919 decoder->state.type = INTEL_PT_INSTRUCTION;
920 decoder->state.from_ip = decoder->ip;
921 decoder->state.to_ip = 0;
922 decoder->ip += intel_pt_insn->length;
923 err = INTEL_PT_RETURN;
927 if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
928 /* Zero-length calls are excluded */
929 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
930 intel_pt_insn->rel) {
931 err = intel_pt_push(&decoder->stack, decoder->ip +
932 intel_pt_insn->length);
936 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
937 decoder->ret_addr = intel_pt_pop(&decoder->stack);
940 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
941 int cnt = decoder->no_progress++;
943 decoder->state.from_ip = decoder->ip;
944 decoder->ip += intel_pt_insn->length +
946 decoder->state.to_ip = decoder->ip;
947 err = INTEL_PT_RETURN;
950 * Check for being stuck in a loop. This can happen if a
951 * decoder error results in the decoder erroneously setting the
952 * ip to an address that is itself in an infinite loop that
953 * consumes no packets. When that happens, there must be an
954 * unconditional branch.
958 decoder->stuck_ip = decoder->state.to_ip;
959 decoder->stuck_ip_prd = 1;
960 decoder->stuck_ip_cnt = 1;
961 } else if (cnt > INTEL_PT_MAX_LOOPS ||
962 decoder->state.to_ip == decoder->stuck_ip) {
963 intel_pt_log_at("ERROR: Never-ending loop",
964 decoder->state.to_ip);
965 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
968 } else if (!--decoder->stuck_ip_cnt) {
969 decoder->stuck_ip_prd += 1;
970 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
971 decoder->stuck_ip = decoder->state.to_ip;
974 goto out_no_progress;
977 decoder->no_progress = 0;
979 decoder->state.insn_op = intel_pt_insn->op;
980 decoder->state.insn_len = intel_pt_insn->length;
982 if (decoder->tx_flags & INTEL_PT_IN_TX)
983 decoder->state.flags |= INTEL_PT_IN_TX;
988 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
990 struct intel_pt_insn intel_pt_insn;
994 ip = decoder->last_ip;
997 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
998 if (err == INTEL_PT_RETURN)
1000 if (err == -EAGAIN) {
1001 if (decoder->set_fup_tx_flags) {
1002 decoder->set_fup_tx_flags = false;
1003 decoder->tx_flags = decoder->fup_tx_flags;
1004 decoder->state.type = INTEL_PT_TRANSACTION;
1005 decoder->state.from_ip = decoder->ip;
1006 decoder->state.to_ip = 0;
1007 decoder->state.flags = decoder->fup_tx_flags;
1012 decoder->set_fup_tx_flags = false;
1016 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1017 intel_pt_log_at("ERROR: Unexpected indirect branch",
1019 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1023 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1024 intel_pt_log_at("ERROR: Unexpected conditional branch",
1026 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1030 intel_pt_bug(decoder);
1034 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1036 struct intel_pt_insn intel_pt_insn;
1039 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1040 if (err == INTEL_PT_RETURN)
1045 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1046 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1047 decoder->pge = false;
1048 decoder->continuous_period = false;
1049 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1050 decoder->state.from_ip = decoder->ip;
1051 decoder->state.to_ip = 0;
1052 if (decoder->packet.count != 0)
1053 decoder->ip = decoder->last_ip;
1055 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1056 decoder->state.from_ip = decoder->ip;
1057 if (decoder->packet.count == 0) {
1058 decoder->state.to_ip = 0;
1060 decoder->state.to_ip = decoder->last_ip;
1061 decoder->ip = decoder->last_ip;
1067 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1068 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1070 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1074 return intel_pt_bug(decoder);
1077 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1079 struct intel_pt_insn intel_pt_insn;
1083 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1084 if (err == INTEL_PT_RETURN)
1089 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1090 if (!decoder->return_compression) {
1091 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1093 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1096 if (!decoder->ret_addr) {
1097 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1099 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1102 if (!(decoder->tnt.payload & BIT63)) {
1103 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1105 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1108 decoder->tnt.count -= 1;
1109 if (!decoder->tnt.count)
1110 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1111 decoder->tnt.payload <<= 1;
1112 decoder->state.from_ip = decoder->ip;
1113 decoder->ip = decoder->ret_addr;
1114 decoder->state.to_ip = decoder->ip;
1118 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1119 /* Handle deferred TIPs */
1120 err = intel_pt_get_next_packet(decoder);
1123 if (decoder->packet.type != INTEL_PT_TIP ||
1124 decoder->packet.count == 0) {
1125 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1127 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1128 decoder->pkt_step = 0;
1131 intel_pt_set_last_ip(decoder);
1132 decoder->state.from_ip = decoder->ip;
1133 decoder->state.to_ip = decoder->last_ip;
1134 decoder->ip = decoder->last_ip;
1138 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1139 decoder->tnt.count -= 1;
1140 if (!decoder->tnt.count)
1141 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1142 if (decoder->tnt.payload & BIT63) {
1143 decoder->tnt.payload <<= 1;
1144 decoder->state.from_ip = decoder->ip;
1145 decoder->ip += intel_pt_insn.length +
1147 decoder->state.to_ip = decoder->ip;
1150 /* Instruction sample for a non-taken branch */
1151 if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1152 decoder->tnt.payload <<= 1;
1153 decoder->state.type = INTEL_PT_INSTRUCTION;
1154 decoder->state.from_ip = decoder->ip;
1155 decoder->state.to_ip = 0;
1156 decoder->ip += intel_pt_insn.length;
1159 decoder->ip += intel_pt_insn.length;
1160 if (!decoder->tnt.count)
1162 decoder->tnt.payload <<= 1;
1166 return intel_pt_bug(decoder);
1170 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1172 unsigned int fup_tx_flags;
1175 fup_tx_flags = decoder->packet.payload &
1176 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1177 err = intel_pt_get_next_packet(decoder);
1180 if (decoder->packet.type == INTEL_PT_FUP) {
1181 decoder->fup_tx_flags = fup_tx_flags;
1182 decoder->set_fup_tx_flags = true;
1183 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1186 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1188 intel_pt_update_in_tx(decoder);
1193 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1197 decoder->have_tma = false;
1199 if (decoder->ref_timestamp) {
1200 timestamp = decoder->packet.payload |
1201 (decoder->ref_timestamp & (0xffULL << 56));
1202 if (timestamp < decoder->ref_timestamp) {
1203 if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1204 timestamp += (1ULL << 56);
1206 if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1207 timestamp -= (1ULL << 56);
1209 decoder->tsc_timestamp = timestamp;
1210 decoder->timestamp = timestamp;
1211 decoder->ref_timestamp = 0;
1212 decoder->timestamp_insn_cnt = 0;
1213 } else if (decoder->timestamp) {
1214 timestamp = decoder->packet.payload |
1215 (decoder->timestamp & (0xffULL << 56));
1216 decoder->tsc_timestamp = timestamp;
1217 if (timestamp < decoder->timestamp &&
1218 decoder->timestamp - timestamp < decoder->tsc_slip) {
1219 intel_pt_log_to("Suppressing backwards timestamp",
1221 timestamp = decoder->timestamp;
1223 if (timestamp < decoder->timestamp) {
1224 intel_pt_log_to("Wraparound timestamp", timestamp);
1225 timestamp += (1ULL << 56);
1226 decoder->tsc_timestamp = timestamp;
1228 decoder->timestamp = timestamp;
1229 decoder->timestamp_insn_cnt = 0;
1232 if (decoder->last_packet_type == INTEL_PT_CYC) {
1233 decoder->cyc_ref_timestamp = decoder->timestamp;
1234 decoder->cycle_cnt = 0;
1235 decoder->have_calc_cyc_to_tsc = false;
1236 intel_pt_calc_cyc_to_tsc(decoder, false);
1239 intel_pt_log_to("Setting timestamp", decoder->timestamp);
1242 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1244 intel_pt_log("ERROR: Buffer overflow\n");
1245 intel_pt_clear_tx_flags(decoder);
1246 decoder->have_tma = false;
1248 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1249 decoder->overflow = true;
1253 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1255 uint32_t ctc = decoder->packet.payload;
1256 uint32_t fc = decoder->packet.count;
1257 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1259 if (!decoder->tsc_ctc_ratio_d)
1262 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1263 decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1264 if (decoder->tsc_ctc_mult) {
1265 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1267 decoder->ctc_timestamp -= multdiv(ctc_rem,
1268 decoder->tsc_ctc_ratio_n,
1269 decoder->tsc_ctc_ratio_d);
1271 decoder->ctc_delta = 0;
1272 decoder->have_tma = true;
1273 decoder->fixup_last_mtc = true;
1274 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n",
1275 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1278 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1281 uint32_t mtc, mtc_delta;
1283 if (!decoder->have_tma)
1286 mtc = decoder->packet.payload;
1288 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1289 decoder->fixup_last_mtc = false;
1290 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1291 &decoder->last_mtc);
1294 if (mtc > decoder->last_mtc)
1295 mtc_delta = mtc - decoder->last_mtc;
1297 mtc_delta = mtc + 256 - decoder->last_mtc;
1299 decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1301 if (decoder->tsc_ctc_mult) {
1302 timestamp = decoder->ctc_timestamp +
1303 decoder->ctc_delta * decoder->tsc_ctc_mult;
1305 timestamp = decoder->ctc_timestamp +
1306 multdiv(decoder->ctc_delta,
1307 decoder->tsc_ctc_ratio_n,
1308 decoder->tsc_ctc_ratio_d);
1311 if (timestamp < decoder->timestamp)
1312 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1313 timestamp, decoder->timestamp);
1315 decoder->timestamp = timestamp;
1317 decoder->timestamp_insn_cnt = 0;
1318 decoder->last_mtc = mtc;
1320 if (decoder->last_packet_type == INTEL_PT_CYC) {
1321 decoder->cyc_ref_timestamp = decoder->timestamp;
1322 decoder->cycle_cnt = 0;
1323 decoder->have_calc_cyc_to_tsc = false;
1324 intel_pt_calc_cyc_to_tsc(decoder, true);
1328 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1330 unsigned int cbr = decoder->packet.payload;
1332 if (decoder->cbr == cbr)
1336 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1339 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1341 uint64_t timestamp = decoder->cyc_ref_timestamp;
1343 decoder->have_cyc = true;
1345 decoder->cycle_cnt += decoder->packet.payload;
1347 if (!decoder->cyc_ref_timestamp)
1350 if (decoder->have_calc_cyc_to_tsc)
1351 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1352 else if (decoder->cbr)
1353 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1357 if (timestamp < decoder->timestamp)
1358 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1359 timestamp, decoder->timestamp);
1361 decoder->timestamp = timestamp;
1363 decoder->timestamp_insn_cnt = 0;
1366 /* Walk PSB+ packets when already in sync. */
1367 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1372 err = intel_pt_get_next_packet(decoder);
1376 switch (decoder->packet.type) {
1377 case INTEL_PT_PSBEND:
1380 case INTEL_PT_TIP_PGD:
1381 case INTEL_PT_TIP_PGE:
1384 case INTEL_PT_TRACESTOP:
1387 decoder->have_tma = false;
1388 intel_pt_log("ERROR: Unexpected packet\n");
1392 return intel_pt_overflow(decoder);
1395 intel_pt_calc_tsc_timestamp(decoder);
1399 intel_pt_calc_tma(decoder);
1403 intel_pt_calc_cbr(decoder);
1406 case INTEL_PT_MODE_EXEC:
1407 decoder->exec_mode = decoder->packet.payload;
1411 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1415 decoder->pge = true;
1416 intel_pt_set_last_ip(decoder);
1419 case INTEL_PT_MODE_TSX:
1420 intel_pt_update_in_tx(decoder);
1424 intel_pt_calc_mtc_timestamp(decoder);
1425 if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1426 decoder->state.type |= INTEL_PT_INSTRUCTION;
1439 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1443 if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1444 decoder->tx_flags = 0;
1445 decoder->state.flags &= ~INTEL_PT_IN_TX;
1446 decoder->state.flags |= INTEL_PT_ABORT_TX;
1448 decoder->state.flags |= INTEL_PT_ASYNC;
1452 err = intel_pt_get_next_packet(decoder);
1456 switch (decoder->packet.type) {
1459 case INTEL_PT_TRACESTOP:
1464 case INTEL_PT_MODE_TSX:
1466 case INTEL_PT_PSBEND:
1467 intel_pt_log("ERROR: Missing TIP after FUP\n");
1468 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1472 return intel_pt_overflow(decoder);
1474 case INTEL_PT_TIP_PGD:
1475 decoder->state.from_ip = decoder->ip;
1476 decoder->state.to_ip = 0;
1477 if (decoder->packet.count != 0) {
1478 intel_pt_set_ip(decoder);
1479 intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1482 decoder->pge = false;
1483 decoder->continuous_period = false;
1486 case INTEL_PT_TIP_PGE:
1487 decoder->pge = true;
1488 intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1490 decoder->state.from_ip = 0;
1491 if (decoder->packet.count == 0) {
1492 decoder->state.to_ip = 0;
1494 intel_pt_set_ip(decoder);
1495 decoder->state.to_ip = decoder->ip;
1500 decoder->state.from_ip = decoder->ip;
1501 if (decoder->packet.count == 0) {
1502 decoder->state.to_ip = 0;
1504 intel_pt_set_ip(decoder);
1505 decoder->state.to_ip = decoder->ip;
1510 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1514 intel_pt_calc_mtc_timestamp(decoder);
1515 if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1516 decoder->state.type |= INTEL_PT_INSTRUCTION;
1520 intel_pt_calc_cyc_timestamp(decoder);
1523 case INTEL_PT_MODE_EXEC:
1524 decoder->exec_mode = decoder->packet.payload;
1533 return intel_pt_bug(decoder);
1538 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1540 bool no_tip = false;
1544 err = intel_pt_get_next_packet(decoder);
1548 switch (decoder->packet.type) {
1550 if (!decoder->packet.count)
1552 decoder->tnt = decoder->packet;
1553 decoder->pkt_state = INTEL_PT_STATE_TNT;
1554 err = intel_pt_walk_tnt(decoder);
1559 case INTEL_PT_TIP_PGD:
1560 if (decoder->packet.count != 0)
1561 intel_pt_set_last_ip(decoder);
1562 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1563 return intel_pt_walk_tip(decoder);
1565 case INTEL_PT_TIP_PGE: {
1566 decoder->pge = true;
1567 if (decoder->packet.count == 0) {
1568 intel_pt_log_at("Skipping zero TIP.PGE",
1572 intel_pt_set_ip(decoder);
1573 decoder->state.from_ip = 0;
1574 decoder->state.to_ip = decoder->ip;
1579 return intel_pt_overflow(decoder);
1582 if (decoder->packet.count != 0)
1583 intel_pt_set_last_ip(decoder);
1584 decoder->pkt_state = INTEL_PT_STATE_TIP;
1585 return intel_pt_walk_tip(decoder);
1588 if (decoder->packet.count == 0) {
1589 intel_pt_log_at("Skipping zero FUP",
1594 intel_pt_set_last_ip(decoder);
1595 err = intel_pt_walk_fup(decoder);
1596 if (err != -EAGAIN) {
1600 decoder->pkt_state =
1601 INTEL_PT_STATE_FUP_NO_TIP;
1603 decoder->pkt_state = INTEL_PT_STATE_FUP;
1610 return intel_pt_walk_fup_tip(decoder);
1612 case INTEL_PT_TRACESTOP:
1613 decoder->pge = false;
1614 decoder->continuous_period = false;
1615 intel_pt_clear_tx_flags(decoder);
1616 decoder->have_tma = false;
1620 intel_pt_clear_stack(&decoder->stack);
1621 err = intel_pt_walk_psbend(decoder);
1629 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1633 intel_pt_calc_mtc_timestamp(decoder);
1634 if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1637 * Ensure that there has been an instruction since the
1640 if (!decoder->mtc_insn)
1642 decoder->mtc_insn = false;
1643 /* Ensure that there is a timestamp */
1644 if (!decoder->timestamp)
1646 decoder->state.type = INTEL_PT_INSTRUCTION;
1647 decoder->state.from_ip = decoder->ip;
1648 decoder->state.to_ip = 0;
1649 decoder->mtc_insn = false;
1653 intel_pt_calc_tsc_timestamp(decoder);
1657 intel_pt_calc_tma(decoder);
1661 intel_pt_calc_cyc_timestamp(decoder);
1665 intel_pt_calc_cbr(decoder);
1668 case INTEL_PT_MODE_EXEC:
1669 decoder->exec_mode = decoder->packet.payload;
1672 case INTEL_PT_MODE_TSX:
1673 /* MODE_TSX need not be followed by FUP */
1674 if (!decoder->pge) {
1675 intel_pt_update_in_tx(decoder);
1678 err = intel_pt_mode_tsx(decoder, &no_tip);
1683 case INTEL_PT_BAD: /* Does not happen */
1684 return intel_pt_bug(decoder);
1686 case INTEL_PT_PSBEND:
1693 return intel_pt_bug(decoder);
1698 /* Walk PSB+ packets to get in sync. */
1699 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1704 err = intel_pt_get_next_packet(decoder);
1708 switch (decoder->packet.type) {
1709 case INTEL_PT_TIP_PGD:
1710 decoder->continuous_period = false;
1711 case INTEL_PT_TIP_PGE:
1713 intel_pt_log("ERROR: Unexpected packet\n");
1717 decoder->pge = true;
1718 if (decoder->last_ip || decoder->packet.count == 6 ||
1719 decoder->packet.count == 0) {
1720 uint64_t current_ip = decoder->ip;
1722 intel_pt_set_ip(decoder);
1724 intel_pt_log_to("Setting IP",
1730 intel_pt_calc_mtc_timestamp(decoder);
1734 intel_pt_calc_tsc_timestamp(decoder);
1738 intel_pt_calc_tma(decoder);
1742 intel_pt_calc_cyc_timestamp(decoder);
1746 intel_pt_calc_cbr(decoder);
1750 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1753 case INTEL_PT_MODE_EXEC:
1754 decoder->exec_mode = decoder->packet.payload;
1757 case INTEL_PT_MODE_TSX:
1758 intel_pt_update_in_tx(decoder);
1761 case INTEL_PT_TRACESTOP:
1762 decoder->pge = false;
1763 decoder->continuous_period = false;
1764 intel_pt_clear_tx_flags(decoder);
1766 decoder->have_tma = false;
1767 intel_pt_log("ERROR: Unexpected packet\n");
1769 decoder->pkt_state = INTEL_PT_STATE_ERR4;
1771 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1774 case INTEL_PT_BAD: /* Does not happen */
1775 return intel_pt_bug(decoder);
1778 return intel_pt_overflow(decoder);
1780 case INTEL_PT_PSBEND:
1793 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
1798 err = intel_pt_get_next_packet(decoder);
1802 switch (decoder->packet.type) {
1803 case INTEL_PT_TIP_PGD:
1804 decoder->continuous_period = false;
1805 case INTEL_PT_TIP_PGE:
1807 decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
1808 if (decoder->last_ip || decoder->packet.count == 6 ||
1809 decoder->packet.count == 0)
1810 intel_pt_set_ip(decoder);
1816 if (decoder->overflow) {
1817 if (decoder->last_ip ||
1818 decoder->packet.count == 6 ||
1819 decoder->packet.count == 0)
1820 intel_pt_set_ip(decoder);
1824 if (decoder->packet.count)
1825 intel_pt_set_last_ip(decoder);
1829 intel_pt_calc_mtc_timestamp(decoder);
1833 intel_pt_calc_tsc_timestamp(decoder);
1837 intel_pt_calc_tma(decoder);
1841 intel_pt_calc_cyc_timestamp(decoder);
1845 intel_pt_calc_cbr(decoder);
1849 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1852 case INTEL_PT_MODE_EXEC:
1853 decoder->exec_mode = decoder->packet.payload;
1856 case INTEL_PT_MODE_TSX:
1857 intel_pt_update_in_tx(decoder);
1861 return intel_pt_overflow(decoder);
1863 case INTEL_PT_BAD: /* Does not happen */
1864 return intel_pt_bug(decoder);
1866 case INTEL_PT_TRACESTOP:
1867 decoder->pge = false;
1868 decoder->continuous_period = false;
1869 intel_pt_clear_tx_flags(decoder);
1870 decoder->have_tma = false;
1874 err = intel_pt_walk_psb(decoder);
1878 /* Do not have a sample */
1879 decoder->state.type = 0;
1885 case INTEL_PT_PSBEND:
1895 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
1899 intel_pt_log("Scanning for full IP\n");
1900 err = intel_pt_walk_to_ip(decoder);
1904 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1905 decoder->overflow = false;
1907 decoder->state.from_ip = 0;
1908 decoder->state.to_ip = decoder->ip;
1909 intel_pt_log_to("Setting IP", decoder->ip);
1914 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
1916 const unsigned char *end = decoder->buf + decoder->len;
1919 for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
1920 if (i > decoder->len)
1922 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
1928 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
1930 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
1931 const char *psb = INTEL_PT_PSB_STR;
1933 if (rest_psb > decoder->len ||
1934 memcmp(decoder->buf, psb + part_psb, rest_psb))
1940 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
1945 decoder->pos += decoder->len;
1948 ret = intel_pt_get_next_data(decoder);
1952 rest_psb = intel_pt_rest_psb(decoder, part_psb);
1956 decoder->pos -= part_psb;
1957 decoder->next_buf = decoder->buf + rest_psb;
1958 decoder->next_len = decoder->len - rest_psb;
1959 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
1960 decoder->buf = decoder->temp_buf;
1961 decoder->len = INTEL_PT_PSB_LEN;
1966 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
1968 unsigned char *next;
1971 intel_pt_log("Scanning for PSB\n");
1973 if (!decoder->len) {
1974 ret = intel_pt_get_next_data(decoder);
1979 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
1984 part_psb = intel_pt_part_psb(decoder);
1986 ret = intel_pt_get_split_psb(decoder, part_psb);
1990 decoder->pos += decoder->len;
1996 decoder->pkt_step = next - decoder->buf;
1997 return intel_pt_get_next_packet(decoder);
2001 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2005 decoder->pge = false;
2006 decoder->continuous_period = false;
2007 decoder->last_ip = 0;
2009 intel_pt_clear_stack(&decoder->stack);
2011 err = intel_pt_scan_for_psb(decoder);
2015 decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2017 err = intel_pt_walk_psb(decoder);
2022 decoder->state.type = 0; /* Do not have a sample */
2023 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2025 return intel_pt_sync_ip(decoder);
2031 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2033 uint64_t est = decoder->timestamp_insn_cnt << 1;
2035 if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2038 est *= decoder->max_non_turbo_ratio;
2039 est /= decoder->cbr;
2041 return decoder->timestamp + est;
2044 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2049 decoder->state.type = INTEL_PT_BRANCH;
2050 decoder->state.flags = 0;
2052 switch (decoder->pkt_state) {
2053 case INTEL_PT_STATE_NO_PSB:
2054 err = intel_pt_sync(decoder);
2056 case INTEL_PT_STATE_NO_IP:
2057 decoder->last_ip = 0;
2059 case INTEL_PT_STATE_ERR_RESYNC:
2060 err = intel_pt_sync_ip(decoder);
2062 case INTEL_PT_STATE_IN_SYNC:
2063 err = intel_pt_walk_trace(decoder);
2065 case INTEL_PT_STATE_TNT:
2066 err = intel_pt_walk_tnt(decoder);
2068 err = intel_pt_walk_trace(decoder);
2070 case INTEL_PT_STATE_TIP:
2071 case INTEL_PT_STATE_TIP_PGD:
2072 err = intel_pt_walk_tip(decoder);
2074 case INTEL_PT_STATE_FUP:
2075 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2076 err = intel_pt_walk_fup(decoder);
2078 err = intel_pt_walk_fup_tip(decoder);
2080 decoder->pkt_state = INTEL_PT_STATE_FUP;
2082 case INTEL_PT_STATE_FUP_NO_TIP:
2083 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2084 err = intel_pt_walk_fup(decoder);
2086 err = intel_pt_walk_trace(decoder);
2089 err = intel_pt_bug(decoder);
2092 } while (err == -ENOLINK);
2094 decoder->state.err = err ? intel_pt_ext_err(err) : 0;
2095 decoder->state.timestamp = decoder->timestamp;
2096 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2097 decoder->state.cr3 = decoder->cr3;
2098 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2101 decoder->state.from_ip = decoder->ip;
2103 return &decoder->state;
2106 static bool intel_pt_at_psb(unsigned char *buf, size_t len)
2108 if (len < INTEL_PT_PSB_LEN)
2110 return memmem(buf, INTEL_PT_PSB_LEN, INTEL_PT_PSB_STR,
2115 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2116 * @buf: pointer to buffer pointer
2117 * @len: size of buffer
2119 * Updates the buffer pointer to point to the start of the next PSB packet if
2120 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated,
2121 * @len is adjusted accordingly.
2123 * Return: %true if a PSB packet is found, %false otherwise.
2125 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2127 unsigned char *next;
2129 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2131 *len -= next - *buf;
2139 * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2141 * @buf: pointer to buffer pointer
2142 * @len: size of buffer
2144 * Updates the buffer pointer to point to the start of the following PSB packet
2145 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2146 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly.
2148 * Return: %true if a PSB packet is found, %false otherwise.
2150 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2152 unsigned char *next;
2157 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2159 *len -= next - *buf;
2167 * intel_pt_last_psb - find the last PSB packet in a buffer.
2169 * @len: size of buffer
2171 * This function finds the last PSB in a buffer.
2173 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2175 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2177 const char *n = INTEL_PT_PSB_STR;
2181 if (len < INTEL_PT_PSB_LEN)
2184 k = len - INTEL_PT_PSB_LEN + 1;
2186 p = memrchr(buf, n[0], k);
2189 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2198 * intel_pt_next_tsc - find and return next TSC.
2200 * @len: size of buffer
2201 * @tsc: TSC value returned
2203 * Find a TSC packet in @buf and return the TSC value. This function assumes
2204 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2205 * PSBEND packet is found.
2207 * Return: %true if TSC is found, false otherwise.
2209 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc)
2211 struct intel_pt_pkt packet;
2215 ret = intel_pt_get_packet(buf, len, &packet);
2218 if (packet.type == INTEL_PT_TSC) {
2219 *tsc = packet.payload;
2222 if (packet.type == INTEL_PT_PSBEND)
2231 * intel_pt_tsc_cmp - compare 7-byte TSCs.
2232 * @tsc1: first TSC to compare
2233 * @tsc2: second TSC to compare
2235 * This function compares 7-byte TSC values allowing for the possibility that
2236 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped
2237 * around so for that purpose this function assumes the absolute difference is
2238 * less than half the maximum difference.
2240 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2243 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2245 const uint64_t halfway = (1ULL << 55);
2251 if (tsc2 - tsc1 < halfway)
2256 if (tsc1 - tsc2 < halfway)
2264 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2266 * @buf_a: first buffer
2267 * @len_a: size of first buffer
2268 * @buf_b: second buffer
2269 * @len_b: size of second buffer
2271 * If the trace contains TSC we can look at the last TSC of @buf_a and the
2272 * first TSC of @buf_b in order to determine if the buffers overlap, and then
2273 * walk forward in @buf_b until a later TSC is found. A precondition is that
2274 * @buf_a and @buf_b are positioned at a PSB.
2276 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2277 * @buf_b + @len_b if there is no non-overlapped data.
2279 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2281 unsigned char *buf_b,
2284 uint64_t tsc_a, tsc_b;
2288 p = intel_pt_last_psb(buf_a, len_a);
2290 return buf_b; /* No PSB in buf_a => no overlap */
2292 len = len_a - (p - buf_a);
2293 if (!intel_pt_next_tsc(p, len, &tsc_a)) {
2294 /* The last PSB+ in buf_a is incomplete, so go back one more */
2296 p = intel_pt_last_psb(buf_a, len_a);
2298 return buf_b; /* No full PSB+ => assume no overlap */
2299 len = len_a - (p - buf_a);
2300 if (!intel_pt_next_tsc(p, len, &tsc_a))
2301 return buf_b; /* No TSC in buf_a => assume no overlap */
2305 /* Ignore PSB+ with no TSC */
2306 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b) &&
2307 intel_pt_tsc_cmp(tsc_a, tsc_b) < 0)
2308 return buf_b; /* tsc_a < tsc_b => no overlap */
2310 if (!intel_pt_step_psb(&buf_b, &len_b))
2311 return buf_b + len_b; /* No PSB in buf_b => no data */
2316 * intel_pt_find_overlap - determine start of non-overlapped trace data.
2317 * @buf_a: first buffer
2318 * @len_a: size of first buffer
2319 * @buf_b: second buffer
2320 * @len_b: size of second buffer
2321 * @have_tsc: can use TSC packets to detect overlap
2323 * When trace samples or snapshots are recorded there is the possibility that
2324 * the data overlaps. Note that, for the purposes of decoding, data is only
2325 * useful if it begins with a PSB packet.
2327 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2328 * @buf_b + @len_b if there is no non-overlapped data.
2330 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2331 unsigned char *buf_b, size_t len_b,
2334 unsigned char *found;
2336 /* Buffer 'b' must start at PSB so throw away everything before that */
2337 if (!intel_pt_next_psb(&buf_b, &len_b))
2338 return buf_b + len_b; /* No PSB */
2340 if (!intel_pt_next_psb(&buf_a, &len_a))
2341 return buf_b; /* No overlap */
2344 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b);
2350 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2351 * we can ignore the first part of buffer 'a'.
2353 while (len_b < len_a) {
2354 if (!intel_pt_step_psb(&buf_a, &len_a))
2355 return buf_b; /* No overlap */
2358 /* Now len_b >= len_a */
2359 if (len_b > len_a) {
2360 /* The leftover buffer 'b' must start at a PSB */
2361 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2362 if (!intel_pt_step_psb(&buf_a, &len_a))
2363 return buf_b; /* No overlap */
2368 /* Potential overlap so check the bytes */
2369 found = memmem(buf_a, len_a, buf_b, len_a);
2371 return buf_b + len_a;
2373 /* Try again at next PSB in buffer 'a' */
2374 if (!intel_pt_step_psb(&buf_a, &len_a))
2375 return buf_b; /* No overlap */
2377 /* The leftover buffer 'b' must start at a PSB */
2378 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2379 if (!intel_pt_step_psb(&buf_a, &len_a))
2380 return buf_b; /* No overlap */