~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/tools/perf/util/intel-pt.c

Version: ~ [ linux-5.2-rc5 ] ~ [ linux-5.1.11 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.52 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.127 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.182 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.182 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.68 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * intel_pt.c: Intel Processor Trace support
  3  * Copyright (c) 2013-2015, Intel Corporation.
  4  *
  5  * This program is free software; you can redistribute it and/or modify it
  6  * under the terms and conditions of the GNU General Public License,
  7  * version 2, as published by the Free Software Foundation.
  8  *
  9  * This program is distributed in the hope it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 12  * more details.
 13  *
 14  */
 15 
 16 #include <stdio.h>
 17 #include <stdbool.h>
 18 #include <errno.h>
 19 #include <linux/kernel.h>
 20 #include <linux/types.h>
 21 
 22 #include "../perf.h"
 23 #include "session.h"
 24 #include "machine.h"
 25 #include "sort.h"
 26 #include "tool.h"
 27 #include "event.h"
 28 #include "evlist.h"
 29 #include "evsel.h"
 30 #include "map.h"
 31 #include "color.h"
 32 #include "util.h"
 33 #include "thread.h"
 34 #include "thread-stack.h"
 35 #include "symbol.h"
 36 #include "callchain.h"
 37 #include "dso.h"
 38 #include "debug.h"
 39 #include "auxtrace.h"
 40 #include "tsc.h"
 41 #include "intel-pt.h"
 42 #include "config.h"
 43 
 44 #include "intel-pt-decoder/intel-pt-log.h"
 45 #include "intel-pt-decoder/intel-pt-decoder.h"
 46 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
 47 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
 48 
 49 #define MAX_TIMESTAMP (~0ULL)
 50 
 51 struct intel_pt {
 52         struct auxtrace auxtrace;
 53         struct auxtrace_queues queues;
 54         struct auxtrace_heap heap;
 55         u32 auxtrace_type;
 56         struct perf_session *session;
 57         struct machine *machine;
 58         struct perf_evsel *switch_evsel;
 59         struct thread *unknown_thread;
 60         bool timeless_decoding;
 61         bool sampling_mode;
 62         bool snapshot_mode;
 63         bool per_cpu_mmaps;
 64         bool have_tsc;
 65         bool data_queued;
 66         bool est_tsc;
 67         bool sync_switch;
 68         bool mispred_all;
 69         int have_sched_switch;
 70         u32 pmu_type;
 71         u64 kernel_start;
 72         u64 switch_ip;
 73         u64 ptss_ip;
 74 
 75         struct perf_tsc_conversion tc;
 76         bool cap_user_time_zero;
 77 
 78         struct itrace_synth_opts synth_opts;
 79 
 80         bool sample_instructions;
 81         u64 instructions_sample_type;
 82         u64 instructions_sample_period;
 83         u64 instructions_id;
 84 
 85         bool sample_branches;
 86         u32 branches_filter;
 87         u64 branches_sample_type;
 88         u64 branches_id;
 89 
 90         bool sample_transactions;
 91         u64 transactions_sample_type;
 92         u64 transactions_id;
 93 
 94         bool synth_needs_swap;
 95 
 96         u64 tsc_bit;
 97         u64 mtc_bit;
 98         u64 mtc_freq_bits;
 99         u32 tsc_ctc_ratio_n;
100         u32 tsc_ctc_ratio_d;
101         u64 cyc_bit;
102         u64 noretcomp_bit;
103         unsigned max_non_turbo_ratio;
104 
105         unsigned long num_events;
106 
107         char *filter;
108         struct addr_filters filts;
109 };
110 
111 enum switch_state {
112         INTEL_PT_SS_NOT_TRACING,
113         INTEL_PT_SS_UNKNOWN,
114         INTEL_PT_SS_TRACING,
115         INTEL_PT_SS_EXPECTING_SWITCH_EVENT,
116         INTEL_PT_SS_EXPECTING_SWITCH_IP,
117 };
118 
119 struct intel_pt_queue {
120         struct intel_pt *pt;
121         unsigned int queue_nr;
122         struct auxtrace_buffer *buffer;
123         void *decoder;
124         const struct intel_pt_state *state;
125         struct ip_callchain *chain;
126         struct branch_stack *last_branch;
127         struct branch_stack *last_branch_rb;
128         size_t last_branch_pos;
129         union perf_event *event_buf;
130         bool on_heap;
131         bool stop;
132         bool step_through_buffers;
133         bool use_buffer_pid_tid;
134         pid_t pid, tid;
135         int cpu;
136         int switch_state;
137         pid_t next_tid;
138         struct thread *thread;
139         bool exclude_kernel;
140         bool have_sample;
141         u64 time;
142         u64 timestamp;
143         u32 flags;
144         u16 insn_len;
145         u64 last_insn_cnt;
146         char insn[INTEL_PT_INSN_BUF_SZ];
147 };
148 
149 static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
150                           unsigned char *buf, size_t len)
151 {
152         struct intel_pt_pkt packet;
153         size_t pos = 0;
154         int ret, pkt_len, i;
155         char desc[INTEL_PT_PKT_DESC_MAX];
156         const char *color = PERF_COLOR_BLUE;
157 
158         color_fprintf(stdout, color,
159                       ". ... Intel Processor Trace data: size %zu bytes\n",
160                       len);
161 
162         while (len) {
163                 ret = intel_pt_get_packet(buf, len, &packet);
164                 if (ret > 0)
165                         pkt_len = ret;
166                 else
167                         pkt_len = 1;
168                 printf(".");
169                 color_fprintf(stdout, color, "  %08x: ", pos);
170                 for (i = 0; i < pkt_len; i++)
171                         color_fprintf(stdout, color, " %02x", buf[i]);
172                 for (; i < 16; i++)
173                         color_fprintf(stdout, color, "   ");
174                 if (ret > 0) {
175                         ret = intel_pt_pkt_desc(&packet, desc,
176                                                 INTEL_PT_PKT_DESC_MAX);
177                         if (ret > 0)
178                                 color_fprintf(stdout, color, " %s\n", desc);
179                 } else {
180                         color_fprintf(stdout, color, " Bad packet!\n");
181                 }
182                 pos += pkt_len;
183                 buf += pkt_len;
184                 len -= pkt_len;
185         }
186 }
187 
188 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf,
189                                 size_t len)
190 {
191         printf(".\n");
192         intel_pt_dump(pt, buf, len);
193 }
194 
195 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a,
196                                    struct auxtrace_buffer *b)
197 {
198         void *start;
199 
200         start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,
201                                       pt->have_tsc);
202         if (!start)
203                 return -EINVAL;
204         b->use_size = b->data + b->size - start;
205         b->use_data = start;
206         return 0;
207 }
208 
209 static void intel_pt_use_buffer_pid_tid(struct intel_pt_queue *ptq,
210                                         struct auxtrace_queue *queue,
211                                         struct auxtrace_buffer *buffer)
212 {
213         if (queue->cpu == -1 && buffer->cpu != -1)
214                 ptq->cpu = buffer->cpu;
215 
216         ptq->pid = buffer->pid;
217         ptq->tid = buffer->tid;
218 
219         intel_pt_log("queue %u cpu %d pid %d tid %d\n",
220                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
221 
222         thread__zput(ptq->thread);
223 
224         if (ptq->tid != -1) {
225                 if (ptq->pid != -1)
226                         ptq->thread = machine__findnew_thread(ptq->pt->machine,
227                                                               ptq->pid,
228                                                               ptq->tid);
229                 else
230                         ptq->thread = machine__find_thread(ptq->pt->machine, -1,
231                                                            ptq->tid);
232         }
233 }
234 
235 /* This function assumes data is processed sequentially only */
236 static int intel_pt_get_trace(struct intel_pt_buffer *b, void *data)
237 {
238         struct intel_pt_queue *ptq = data;
239         struct auxtrace_buffer *buffer = ptq->buffer, *old_buffer = buffer;
240         struct auxtrace_queue *queue;
241 
242         if (ptq->stop) {
243                 b->len = 0;
244                 return 0;
245         }
246 
247         queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
248 next:
249         buffer = auxtrace_buffer__next(queue, buffer);
250         if (!buffer) {
251                 if (old_buffer)
252                         auxtrace_buffer__drop_data(old_buffer);
253                 b->len = 0;
254                 return 0;
255         }
256 
257         ptq->buffer = buffer;
258 
259         if (!buffer->data) {
260                 int fd = perf_data_file__fd(ptq->pt->session->file);
261 
262                 buffer->data = auxtrace_buffer__get_data(buffer, fd);
263                 if (!buffer->data)
264                         return -ENOMEM;
265         }
266 
267         if (ptq->pt->snapshot_mode && !buffer->consecutive && old_buffer &&
268             intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
269                 return -ENOMEM;
270 
271         if (buffer->use_data) {
272                 b->len = buffer->use_size;
273                 b->buf = buffer->use_data;
274         } else {
275                 b->len = buffer->size;
276                 b->buf = buffer->data;
277         }
278         b->ref_timestamp = buffer->reference;
279 
280         /*
281          * If in snapshot mode and the buffer has no usable data, get next
282          * buffer and again check overlap against old_buffer.
283          */
284         if (ptq->pt->snapshot_mode && !b->len)
285                 goto next;
286 
287         if (old_buffer)
288                 auxtrace_buffer__drop_data(old_buffer);
289 
290         if (!old_buffer || ptq->pt->sampling_mode || (ptq->pt->snapshot_mode &&
291                                                       !buffer->consecutive)) {
292                 b->consecutive = false;
293                 b->trace_nr = buffer->buffer_nr + 1;
294         } else {
295                 b->consecutive = true;
296         }
297 
298         if (ptq->use_buffer_pid_tid && (ptq->pid != buffer->pid ||
299                                         ptq->tid != buffer->tid))
300                 intel_pt_use_buffer_pid_tid(ptq, queue, buffer);
301 
302         if (ptq->step_through_buffers)
303                 ptq->stop = true;
304 
305         if (!b->len)
306                 return intel_pt_get_trace(b, data);
307 
308         return 0;
309 }
310 
311 struct intel_pt_cache_entry {
312         struct auxtrace_cache_entry     entry;
313         u64                             insn_cnt;
314         u64                             byte_cnt;
315         enum intel_pt_insn_op           op;
316         enum intel_pt_insn_branch       branch;
317         int                             length;
318         int32_t                         rel;
319         char                            insn[INTEL_PT_INSN_BUF_SZ];
320 };
321 
322 static int intel_pt_config_div(const char *var, const char *value, void *data)
323 {
324         int *d = data;
325         long val;
326 
327         if (!strcmp(var, "intel-pt.cache-divisor")) {
328                 val = strtol(value, NULL, 0);
329                 if (val > 0 && val <= INT_MAX)
330                         *d = val;
331         }
332 
333         return 0;
334 }
335 
336 static int intel_pt_cache_divisor(void)
337 {
338         static int d;
339 
340         if (d)
341                 return d;
342 
343         perf_config(intel_pt_config_div, &d);
344 
345         if (!d)
346                 d = 64;
347 
348         return d;
349 }
350 
351 static unsigned int intel_pt_cache_size(struct dso *dso,
352                                         struct machine *machine)
353 {
354         off_t size;
355 
356         size = dso__data_size(dso, machine);
357         size /= intel_pt_cache_divisor();
358         if (size < 1000)
359                 return 10;
360         if (size > (1 << 21))
361                 return 21;
362         return 32 - __builtin_clz(size);
363 }
364 
365 static struct auxtrace_cache *intel_pt_cache(struct dso *dso,
366                                              struct machine *machine)
367 {
368         struct auxtrace_cache *c;
369         unsigned int bits;
370 
371         if (dso->auxtrace_cache)
372                 return dso->auxtrace_cache;
373 
374         bits = intel_pt_cache_size(dso, machine);
375 
376         /* Ignoring cache creation failure */
377         c = auxtrace_cache__new(bits, sizeof(struct intel_pt_cache_entry), 200);
378 
379         dso->auxtrace_cache = c;
380 
381         return c;
382 }
383 
384 static int intel_pt_cache_add(struct dso *dso, struct machine *machine,
385                               u64 offset, u64 insn_cnt, u64 byte_cnt,
386                               struct intel_pt_insn *intel_pt_insn)
387 {
388         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
389         struct intel_pt_cache_entry *e;
390         int err;
391 
392         if (!c)
393                 return -ENOMEM;
394 
395         e = auxtrace_cache__alloc_entry(c);
396         if (!e)
397                 return -ENOMEM;
398 
399         e->insn_cnt = insn_cnt;
400         e->byte_cnt = byte_cnt;
401         e->op = intel_pt_insn->op;
402         e->branch = intel_pt_insn->branch;
403         e->length = intel_pt_insn->length;
404         e->rel = intel_pt_insn->rel;
405         memcpy(e->insn, intel_pt_insn->buf, INTEL_PT_INSN_BUF_SZ);
406 
407         err = auxtrace_cache__add(c, offset, &e->entry);
408         if (err)
409                 auxtrace_cache__free_entry(c, e);
410 
411         return err;
412 }
413 
414 static struct intel_pt_cache_entry *
415 intel_pt_cache_lookup(struct dso *dso, struct machine *machine, u64 offset)
416 {
417         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
418 
419         if (!c)
420                 return NULL;
421 
422         return auxtrace_cache__lookup(dso->auxtrace_cache, offset);
423 }
424 
425 static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn,
426                                    uint64_t *insn_cnt_ptr, uint64_t *ip,
427                                    uint64_t to_ip, uint64_t max_insn_cnt,
428                                    void *data)
429 {
430         struct intel_pt_queue *ptq = data;
431         struct machine *machine = ptq->pt->machine;
432         struct thread *thread;
433         struct addr_location al;
434         unsigned char buf[INTEL_PT_INSN_BUF_SZ];
435         ssize_t len;
436         int x86_64;
437         u8 cpumode;
438         u64 offset, start_offset, start_ip;
439         u64 insn_cnt = 0;
440         bool one_map = true;
441 
442         intel_pt_insn->length = 0;
443 
444         if (to_ip && *ip == to_ip)
445                 goto out_no_cache;
446 
447         if (*ip >= ptq->pt->kernel_start)
448                 cpumode = PERF_RECORD_MISC_KERNEL;
449         else
450                 cpumode = PERF_RECORD_MISC_USER;
451 
452         thread = ptq->thread;
453         if (!thread) {
454                 if (cpumode != PERF_RECORD_MISC_KERNEL)
455                         return -EINVAL;
456                 thread = ptq->pt->unknown_thread;
457         }
458 
459         while (1) {
460                 thread__find_addr_map(thread, cpumode, MAP__FUNCTION, *ip, &al);
461                 if (!al.map || !al.map->dso)
462                         return -EINVAL;
463 
464                 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
465                     dso__data_status_seen(al.map->dso,
466                                           DSO_DATA_STATUS_SEEN_ITRACE))
467                         return -ENOENT;
468 
469                 offset = al.map->map_ip(al.map, *ip);
470 
471                 if (!to_ip && one_map) {
472                         struct intel_pt_cache_entry *e;
473 
474                         e = intel_pt_cache_lookup(al.map->dso, machine, offset);
475                         if (e &&
476                             (!max_insn_cnt || e->insn_cnt <= max_insn_cnt)) {
477                                 *insn_cnt_ptr = e->insn_cnt;
478                                 *ip += e->byte_cnt;
479                                 intel_pt_insn->op = e->op;
480                                 intel_pt_insn->branch = e->branch;
481                                 intel_pt_insn->length = e->length;
482                                 intel_pt_insn->rel = e->rel;
483                                 memcpy(intel_pt_insn->buf, e->insn,
484                                        INTEL_PT_INSN_BUF_SZ);
485                                 intel_pt_log_insn_no_data(intel_pt_insn, *ip);
486                                 return 0;
487                         }
488                 }
489 
490                 start_offset = offset;
491                 start_ip = *ip;
492 
493                 /* Load maps to ensure dso->is_64_bit has been updated */
494                 map__load(al.map);
495 
496                 x86_64 = al.map->dso->is_64_bit;
497 
498                 while (1) {
499                         len = dso__data_read_offset(al.map->dso, machine,
500                                                     offset, buf,
501                                                     INTEL_PT_INSN_BUF_SZ);
502                         if (len <= 0)
503                                 return -EINVAL;
504 
505                         if (intel_pt_get_insn(buf, len, x86_64, intel_pt_insn))
506                                 return -EINVAL;
507 
508                         intel_pt_log_insn(intel_pt_insn, *ip);
509 
510                         insn_cnt += 1;
511 
512                         if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH)
513                                 goto out;
514 
515                         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
516                                 goto out_no_cache;
517 
518                         *ip += intel_pt_insn->length;
519 
520                         if (to_ip && *ip == to_ip)
521                                 goto out_no_cache;
522 
523                         if (*ip >= al.map->end)
524                                 break;
525 
526                         offset += intel_pt_insn->length;
527                 }
528                 one_map = false;
529         }
530 out:
531         *insn_cnt_ptr = insn_cnt;
532 
533         if (!one_map)
534                 goto out_no_cache;
535 
536         /*
537          * Didn't lookup in the 'to_ip' case, so do it now to prevent duplicate
538          * entries.
539          */
540         if (to_ip) {
541                 struct intel_pt_cache_entry *e;
542 
543                 e = intel_pt_cache_lookup(al.map->dso, machine, start_offset);
544                 if (e)
545                         return 0;
546         }
547 
548         /* Ignore cache errors */
549         intel_pt_cache_add(al.map->dso, machine, start_offset, insn_cnt,
550                            *ip - start_ip, intel_pt_insn);
551 
552         return 0;
553 
554 out_no_cache:
555         *insn_cnt_ptr = insn_cnt;
556         return 0;
557 }
558 
559 static bool intel_pt_match_pgd_ip(struct intel_pt *pt, uint64_t ip,
560                                   uint64_t offset, const char *filename)
561 {
562         struct addr_filter *filt;
563         bool have_filter   = false;
564         bool hit_tracestop = false;
565         bool hit_filter    = false;
566 
567         list_for_each_entry(filt, &pt->filts.head, list) {
568                 if (filt->start)
569                         have_filter = true;
570 
571                 if ((filename && !filt->filename) ||
572                     (!filename && filt->filename) ||
573                     (filename && strcmp(filename, filt->filename)))
574                         continue;
575 
576                 if (!(offset >= filt->addr && offset < filt->addr + filt->size))
577                         continue;
578 
579                 intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s hit filter: %s offset %#"PRIx64" size %#"PRIx64"\n",
580                              ip, offset, filename ? filename : "[kernel]",
581                              filt->start ? "filter" : "stop",
582                              filt->addr, filt->size);
583 
584                 if (filt->start)
585                         hit_filter = true;
586                 else
587                         hit_tracestop = true;
588         }
589 
590         if (!hit_tracestop && !hit_filter)
591                 intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s is not in a filter region\n",
592                              ip, offset, filename ? filename : "[kernel]");
593 
594         return hit_tracestop || (have_filter && !hit_filter);
595 }
596 
597 static int __intel_pt_pgd_ip(uint64_t ip, void *data)
598 {
599         struct intel_pt_queue *ptq = data;
600         struct thread *thread;
601         struct addr_location al;
602         u8 cpumode;
603         u64 offset;
604 
605         if (ip >= ptq->pt->kernel_start)
606                 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);
607 
608         cpumode = PERF_RECORD_MISC_USER;
609 
610         thread = ptq->thread;
611         if (!thread)
612                 return -EINVAL;
613 
614         thread__find_addr_map(thread, cpumode, MAP__FUNCTION, ip, &al);
615         if (!al.map || !al.map->dso)
616                 return -EINVAL;
617 
618         offset = al.map->map_ip(al.map, ip);
619 
620         return intel_pt_match_pgd_ip(ptq->pt, ip, offset,
621                                      al.map->dso->long_name);
622 }
623 
624 static bool intel_pt_pgd_ip(uint64_t ip, void *data)
625 {
626         return __intel_pt_pgd_ip(ip, data) > 0;
627 }
628 
629 static bool intel_pt_get_config(struct intel_pt *pt,
630                                 struct perf_event_attr *attr, u64 *config)
631 {
632         if (attr->type == pt->pmu_type) {
633                 if (config)
634                         *config = attr->config;
635                 return true;
636         }
637 
638         return false;
639 }
640 
641 static bool intel_pt_exclude_kernel(struct intel_pt *pt)
642 {
643         struct perf_evsel *evsel;
644 
645         evlist__for_each_entry(pt->session->evlist, evsel) {
646                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
647                     !evsel->attr.exclude_kernel)
648                         return false;
649         }
650         return true;
651 }
652 
653 static bool intel_pt_return_compression(struct intel_pt *pt)
654 {
655         struct perf_evsel *evsel;
656         u64 config;
657 
658         if (!pt->noretcomp_bit)
659                 return true;
660 
661         evlist__for_each_entry(pt->session->evlist, evsel) {
662                 if (intel_pt_get_config(pt, &evsel->attr, &config) &&
663                     (config & pt->noretcomp_bit))
664                         return false;
665         }
666         return true;
667 }
668 
669 static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
670 {
671         struct perf_evsel *evsel;
672         unsigned int shift;
673         u64 config;
674 
675         if (!pt->mtc_freq_bits)
676                 return 0;
677 
678         for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
679                 config >>= 1;
680 
681         evlist__for_each_entry(pt->session->evlist, evsel) {
682                 if (intel_pt_get_config(pt, &evsel->attr, &config))
683                         return (config & pt->mtc_freq_bits) >> shift;
684         }
685         return 0;
686 }
687 
688 static bool intel_pt_timeless_decoding(struct intel_pt *pt)
689 {
690         struct perf_evsel *evsel;
691         bool timeless_decoding = true;
692         u64 config;
693 
694         if (!pt->tsc_bit || !pt->cap_user_time_zero)
695                 return true;
696 
697         evlist__for_each_entry(pt->session->evlist, evsel) {
698                 if (!(evsel->attr.sample_type & PERF_SAMPLE_TIME))
699                         return true;
700                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
701                         if (config & pt->tsc_bit)
702                                 timeless_decoding = false;
703                         else
704                                 return true;
705                 }
706         }
707         return timeless_decoding;
708 }
709 
710 static bool intel_pt_tracing_kernel(struct intel_pt *pt)
711 {
712         struct perf_evsel *evsel;
713 
714         evlist__for_each_entry(pt->session->evlist, evsel) {
715                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
716                     !evsel->attr.exclude_kernel)
717                         return true;
718         }
719         return false;
720 }
721 
722 static bool intel_pt_have_tsc(struct intel_pt *pt)
723 {
724         struct perf_evsel *evsel;
725         bool have_tsc = false;
726         u64 config;
727 
728         if (!pt->tsc_bit)
729                 return false;
730 
731         evlist__for_each_entry(pt->session->evlist, evsel) {
732                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
733                         if (config & pt->tsc_bit)
734                                 have_tsc = true;
735                         else
736                                 return false;
737                 }
738         }
739         return have_tsc;
740 }
741 
742 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
743 {
744         u64 quot, rem;
745 
746         quot = ns / pt->tc.time_mult;
747         rem  = ns % pt->tc.time_mult;
748         return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
749                 pt->tc.time_mult;
750 }
751 
752 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt,
753                                                    unsigned int queue_nr)
754 {
755         struct intel_pt_params params = { .get_trace = 0, };
756         struct intel_pt_queue *ptq;
757 
758         ptq = zalloc(sizeof(struct intel_pt_queue));
759         if (!ptq)
760                 return NULL;
761 
762         if (pt->synth_opts.callchain) {
763                 size_t sz = sizeof(struct ip_callchain);
764 
765                 sz += pt->synth_opts.callchain_sz * sizeof(u64);
766                 ptq->chain = zalloc(sz);
767                 if (!ptq->chain)
768                         goto out_free;
769         }
770 
771         if (pt->synth_opts.last_branch) {
772                 size_t sz = sizeof(struct branch_stack);
773 
774                 sz += pt->synth_opts.last_branch_sz *
775                       sizeof(struct branch_entry);
776                 ptq->last_branch = zalloc(sz);
777                 if (!ptq->last_branch)
778                         goto out_free;
779                 ptq->last_branch_rb = zalloc(sz);
780                 if (!ptq->last_branch_rb)
781                         goto out_free;
782         }
783 
784         ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
785         if (!ptq->event_buf)
786                 goto out_free;
787 
788         ptq->pt = pt;
789         ptq->queue_nr = queue_nr;
790         ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
791         ptq->pid = -1;
792         ptq->tid = -1;
793         ptq->cpu = -1;
794         ptq->next_tid = -1;
795 
796         params.get_trace = intel_pt_get_trace;
797         params.walk_insn = intel_pt_walk_next_insn;
798         params.data = ptq;
799         params.return_compression = intel_pt_return_compression(pt);
800         params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
801         params.mtc_period = intel_pt_mtc_period(pt);
802         params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
803         params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
804 
805         if (pt->filts.cnt > 0)
806                 params.pgd_ip = intel_pt_pgd_ip;
807 
808         if (pt->synth_opts.instructions) {
809                 if (pt->synth_opts.period) {
810                         switch (pt->synth_opts.period_type) {
811                         case PERF_ITRACE_PERIOD_INSTRUCTIONS:
812                                 params.period_type =
813                                                 INTEL_PT_PERIOD_INSTRUCTIONS;
814                                 params.period = pt->synth_opts.period;
815                                 break;
816                         case PERF_ITRACE_PERIOD_TICKS:
817                                 params.period_type = INTEL_PT_PERIOD_TICKS;
818                                 params.period = pt->synth_opts.period;
819                                 break;
820                         case PERF_ITRACE_PERIOD_NANOSECS:
821                                 params.period_type = INTEL_PT_PERIOD_TICKS;
822                                 params.period = intel_pt_ns_to_ticks(pt,
823                                                         pt->synth_opts.period);
824                                 break;
825                         default:
826                                 break;
827                         }
828                 }
829 
830                 if (!params.period) {
831                         params.period_type = INTEL_PT_PERIOD_INSTRUCTIONS;
832                         params.period = 1;
833                 }
834         }
835 
836         ptq->decoder = intel_pt_decoder_new(&params);
837         if (!ptq->decoder)
838                 goto out_free;
839 
840         return ptq;
841 
842 out_free:
843         zfree(&ptq->event_buf);
844         zfree(&ptq->last_branch);
845         zfree(&ptq->last_branch_rb);
846         zfree(&ptq->chain);
847         free(ptq);
848         return NULL;
849 }
850 
851 static void intel_pt_free_queue(void *priv)
852 {
853         struct intel_pt_queue *ptq = priv;
854 
855         if (!ptq)
856                 return;
857         thread__zput(ptq->thread);
858         intel_pt_decoder_free(ptq->decoder);
859         zfree(&ptq->event_buf);
860         zfree(&ptq->last_branch);
861         zfree(&ptq->last_branch_rb);
862         zfree(&ptq->chain);
863         free(ptq);
864 }
865 
866 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
867                                      struct auxtrace_queue *queue)
868 {
869         struct intel_pt_queue *ptq = queue->priv;
870 
871         if (queue->tid == -1 || pt->have_sched_switch) {
872                 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
873                 thread__zput(ptq->thread);
874         }
875 
876         if (!ptq->thread && ptq->tid != -1)
877                 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
878 
879         if (ptq->thread) {
880                 ptq->pid = ptq->thread->pid_;
881                 if (queue->cpu == -1)
882                         ptq->cpu = ptq->thread->cpu;
883         }
884 }
885 
886 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
887 {
888         if (ptq->state->flags & INTEL_PT_ABORT_TX) {
889                 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;
890         } else if (ptq->state->flags & INTEL_PT_ASYNC) {
891                 if (ptq->state->to_ip)
892                         ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |
893                                      PERF_IP_FLAG_ASYNC |
894                                      PERF_IP_FLAG_INTERRUPT;
895                 else
896                         ptq->flags = PERF_IP_FLAG_BRANCH |
897                                      PERF_IP_FLAG_TRACE_END;
898                 ptq->insn_len = 0;
899         } else {
900                 if (ptq->state->from_ip)
901                         ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
902                 else
903                         ptq->flags = PERF_IP_FLAG_BRANCH |
904                                      PERF_IP_FLAG_TRACE_BEGIN;
905                 if (ptq->state->flags & INTEL_PT_IN_TX)
906                         ptq->flags |= PERF_IP_FLAG_IN_TX;
907                 ptq->insn_len = ptq->state->insn_len;
908                 memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ);
909         }
910 }
911 
912 static int intel_pt_setup_queue(struct intel_pt *pt,
913                                 struct auxtrace_queue *queue,
914                                 unsigned int queue_nr)
915 {
916         struct intel_pt_queue *ptq = queue->priv;
917 
918         if (list_empty(&queue->head))
919                 return 0;
920 
921         if (!ptq) {
922                 ptq = intel_pt_alloc_queue(pt, queue_nr);
923                 if (!ptq)
924                         return -ENOMEM;
925                 queue->priv = ptq;
926 
927                 if (queue->cpu != -1)
928                         ptq->cpu = queue->cpu;
929                 ptq->tid = queue->tid;
930 
931                 if (pt->sampling_mode) {
932                         if (pt->timeless_decoding)
933                                 ptq->step_through_buffers = true;
934                         if (pt->timeless_decoding || !pt->have_sched_switch)
935                                 ptq->use_buffer_pid_tid = true;
936                 }
937         }
938 
939         if (!ptq->on_heap &&
940             (!pt->sync_switch ||
941              ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
942                 const struct intel_pt_state *state;
943                 int ret;
944 
945                 if (pt->timeless_decoding)
946                         return 0;
947 
948                 intel_pt_log("queue %u getting timestamp\n", queue_nr);
949                 intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
950                              queue_nr, ptq->cpu, ptq->pid, ptq->tid);
951                 while (1) {
952                         state = intel_pt_decode(ptq->decoder);
953                         if (state->err) {
954                                 if (state->err == INTEL_PT_ERR_NODATA) {
955                                         intel_pt_log("queue %u has no timestamp\n",
956                                                      queue_nr);
957                                         return 0;
958                                 }
959                                 continue;
960                         }
961                         if (state->timestamp)
962                                 break;
963                 }
964 
965                 ptq->timestamp = state->timestamp;
966                 intel_pt_log("queue %u timestamp 0x%" PRIx64 "\n",
967                              queue_nr, ptq->timestamp);
968                 ptq->state = state;
969                 ptq->have_sample = true;
970                 intel_pt_sample_flags(ptq);
971                 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
972                 if (ret)
973                         return ret;
974                 ptq->on_heap = true;
975         }
976 
977         return 0;
978 }
979 
980 static int intel_pt_setup_queues(struct intel_pt *pt)
981 {
982         unsigned int i;
983         int ret;
984 
985         for (i = 0; i < pt->queues.nr_queues; i++) {
986                 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
987                 if (ret)
988                         return ret;
989         }
990         return 0;
991 }
992 
993 static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue *ptq)
994 {
995         struct branch_stack *bs_src = ptq->last_branch_rb;
996         struct branch_stack *bs_dst = ptq->last_branch;
997         size_t nr = 0;
998 
999         bs_dst->nr = bs_src->nr;
1000 
1001         if (!bs_src->nr)
1002                 return;
1003 
1004         nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos;
1005         memcpy(&bs_dst->entries[0],
1006                &bs_src->entries[ptq->last_branch_pos],
1007                sizeof(struct branch_entry) * nr);
1008 
1009         if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) {
1010                 memcpy(&bs_dst->entries[nr],
1011                        &bs_src->entries[0],
1012                        sizeof(struct branch_entry) * ptq->last_branch_pos);
1013         }
1014 }
1015 
1016 static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue *ptq)
1017 {
1018         ptq->last_branch_pos = 0;
1019         ptq->last_branch_rb->nr = 0;
1020 }
1021 
1022 static void intel_pt_update_last_branch_rb(struct intel_pt_queue *ptq)
1023 {
1024         const struct intel_pt_state *state = ptq->state;
1025         struct branch_stack *bs = ptq->last_branch_rb;
1026         struct branch_entry *be;
1027 
1028         if (!ptq->last_branch_pos)
1029                 ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz;
1030 
1031         ptq->last_branch_pos -= 1;
1032 
1033         be              = &bs->entries[ptq->last_branch_pos];
1034         be->from        = state->from_ip;
1035         be->to          = state->to_ip;
1036         be->flags.abort = !!(state->flags & INTEL_PT_ABORT_TX);
1037         be->flags.in_tx = !!(state->flags & INTEL_PT_IN_TX);
1038         /* No support for mispredict */
1039         be->flags.mispred = ptq->pt->mispred_all;
1040 
1041         if (bs->nr < ptq->pt->synth_opts.last_branch_sz)
1042                 bs->nr += 1;
1043 }
1044 
1045 static int intel_pt_inject_event(union perf_event *event,
1046                                  struct perf_sample *sample, u64 type,
1047                                  bool swapped)
1048 {
1049         event->header.size = perf_event__sample_event_size(sample, type, 0);
1050         return perf_event__synthesize_sample(event, type, 0, sample, swapped);
1051 }
1052 
1053 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
1054 {
1055         int ret;
1056         struct intel_pt *pt = ptq->pt;
1057         union perf_event *event = ptq->event_buf;
1058         struct perf_sample sample = { .ip = 0, };
1059         struct dummy_branch_stack {
1060                 u64                     nr;
1061                 struct branch_entry     entries;
1062         } dummy_bs;
1063 
1064         if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
1065                 return 0;
1066 
1067         if (pt->synth_opts.initial_skip &&
1068             pt->num_events++ < pt->synth_opts.initial_skip)
1069                 return 0;
1070 
1071         event->sample.header.type = PERF_RECORD_SAMPLE;
1072         event->sample.header.misc = PERF_RECORD_MISC_USER;
1073         event->sample.header.size = sizeof(struct perf_event_header);
1074 
1075         if (!pt->timeless_decoding)
1076                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1077 
1078         sample.cpumode = PERF_RECORD_MISC_USER;
1079         sample.ip = ptq->state->from_ip;
1080         sample.pid = ptq->pid;
1081         sample.tid = ptq->tid;
1082         sample.addr = ptq->state->to_ip;
1083         sample.id = ptq->pt->branches_id;
1084         sample.stream_id = ptq->pt->branches_id;
1085         sample.period = 1;
1086         sample.cpu = ptq->cpu;
1087         sample.flags = ptq->flags;
1088         sample.insn_len = ptq->insn_len;
1089         memcpy(sample.insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);
1090 
1091         /*
1092          * perf report cannot handle events without a branch stack when using
1093          * SORT_MODE__BRANCH so make a dummy one.
1094          */
1095         if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {
1096                 dummy_bs = (struct dummy_branch_stack){
1097                         .nr = 1,
1098                         .entries = {
1099                                 .from = sample.ip,
1100                                 .to = sample.addr,
1101                         },
1102                 };
1103                 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1104         }
1105 
1106         if (pt->synth_opts.inject) {
1107                 ret = intel_pt_inject_event(event, &sample,
1108                                             pt->branches_sample_type,
1109                                             pt->synth_needs_swap);
1110                 if (ret)
1111                         return ret;
1112         }
1113 
1114         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1115         if (ret)
1116                 pr_err("Intel Processor Trace: failed to deliver branch event, error %d\n",
1117                        ret);
1118 
1119         return ret;
1120 }
1121 
1122 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
1123 {
1124         int ret;
1125         struct intel_pt *pt = ptq->pt;
1126         union perf_event *event = ptq->event_buf;
1127         struct perf_sample sample = { .ip = 0, };
1128 
1129         if (pt->synth_opts.initial_skip &&
1130             pt->num_events++ < pt->synth_opts.initial_skip)
1131                 return 0;
1132 
1133         event->sample.header.type = PERF_RECORD_SAMPLE;
1134         event->sample.header.misc = PERF_RECORD_MISC_USER;
1135         event->sample.header.size = sizeof(struct perf_event_header);
1136 
1137         if (!pt->timeless_decoding)
1138                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1139 
1140         sample.cpumode = PERF_RECORD_MISC_USER;
1141         sample.ip = ptq->state->from_ip;
1142         sample.pid = ptq->pid;
1143         sample.tid = ptq->tid;
1144         sample.addr = ptq->state->to_ip;
1145         sample.id = ptq->pt->instructions_id;
1146         sample.stream_id = ptq->pt->instructions_id;
1147         sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
1148         sample.cpu = ptq->cpu;
1149         sample.flags = ptq->flags;
1150         sample.insn_len = ptq->insn_len;
1151         memcpy(sample.insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);
1152 
1153         ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
1154 
1155         if (pt->synth_opts.callchain) {
1156                 thread_stack__sample(ptq->thread, ptq->chain,
1157                                      pt->synth_opts.callchain_sz, sample.ip);
1158                 sample.callchain = ptq->chain;
1159         }
1160 
1161         if (pt->synth_opts.last_branch) {
1162                 intel_pt_copy_last_branch_rb(ptq);
1163                 sample.branch_stack = ptq->last_branch;
1164         }
1165 
1166         if (pt->synth_opts.inject) {
1167                 ret = intel_pt_inject_event(event, &sample,
1168                                             pt->instructions_sample_type,
1169                                             pt->synth_needs_swap);
1170                 if (ret)
1171                         return ret;
1172         }
1173 
1174         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1175         if (ret)
1176                 pr_err("Intel Processor Trace: failed to deliver instruction event, error %d\n",
1177                        ret);
1178 
1179         if (pt->synth_opts.last_branch)
1180                 intel_pt_reset_last_branch_rb(ptq);
1181 
1182         return ret;
1183 }
1184 
1185 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
1186 {
1187         int ret;
1188         struct intel_pt *pt = ptq->pt;
1189         union perf_event *event = ptq->event_buf;
1190         struct perf_sample sample = { .ip = 0, };
1191 
1192         if (pt->synth_opts.initial_skip &&
1193             pt->num_events++ < pt->synth_opts.initial_skip)
1194                 return 0;
1195 
1196         event->sample.header.type = PERF_RECORD_SAMPLE;
1197         event->sample.header.misc = PERF_RECORD_MISC_USER;
1198         event->sample.header.size = sizeof(struct perf_event_header);
1199 
1200         if (!pt->timeless_decoding)
1201                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1202 
1203         sample.cpumode = PERF_RECORD_MISC_USER;
1204         sample.ip = ptq->state->from_ip;
1205         sample.pid = ptq->pid;
1206         sample.tid = ptq->tid;
1207         sample.addr = ptq->state->to_ip;
1208         sample.id = ptq->pt->transactions_id;
1209         sample.stream_id = ptq->pt->transactions_id;
1210         sample.period = 1;
1211         sample.cpu = ptq->cpu;
1212         sample.flags = ptq->flags;
1213         sample.insn_len = ptq->insn_len;
1214         memcpy(sample.insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);
1215 
1216         if (pt->synth_opts.callchain) {
1217                 thread_stack__sample(ptq->thread, ptq->chain,
1218                                      pt->synth_opts.callchain_sz, sample.ip);
1219                 sample.callchain = ptq->chain;
1220         }
1221 
1222         if (pt->synth_opts.last_branch) {
1223                 intel_pt_copy_last_branch_rb(ptq);
1224                 sample.branch_stack = ptq->last_branch;
1225         }
1226 
1227         if (pt->synth_opts.inject) {
1228                 ret = intel_pt_inject_event(event, &sample,
1229                                             pt->transactions_sample_type,
1230                                             pt->synth_needs_swap);
1231                 if (ret)
1232                         return ret;
1233         }
1234 
1235         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1236         if (ret)
1237                 pr_err("Intel Processor Trace: failed to deliver transaction event, error %d\n",
1238                        ret);
1239 
1240         if (pt->synth_opts.last_branch)
1241                 intel_pt_reset_last_branch_rb(ptq);
1242 
1243         return ret;
1244 }
1245 
1246 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
1247                                 pid_t pid, pid_t tid, u64 ip)
1248 {
1249         union perf_event event;
1250         char msg[MAX_AUXTRACE_ERROR_MSG];
1251         int err;
1252 
1253         intel_pt__strerror(code, msg, MAX_AUXTRACE_ERROR_MSG);
1254 
1255         auxtrace_synth_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
1256                              code, cpu, pid, tid, ip, msg);
1257 
1258         err = perf_session__deliver_synth_event(pt->session, &event, NULL);
1259         if (err)
1260                 pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
1261                        err);
1262 
1263         return err;
1264 }
1265 
1266 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
1267 {
1268         struct auxtrace_queue *queue;
1269         pid_t tid = ptq->next_tid;
1270         int err;
1271 
1272         if (tid == -1)
1273                 return 0;
1274 
1275         intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
1276 
1277         err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
1278 
1279         queue = &pt->queues.queue_array[ptq->queue_nr];
1280         intel_pt_set_pid_tid_cpu(pt, queue);
1281 
1282         ptq->next_tid = -1;
1283 
1284         return err;
1285 }
1286 
1287 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
1288 {
1289         struct intel_pt *pt = ptq->pt;
1290 
1291         return ip == pt->switch_ip &&
1292                (ptq->flags & PERF_IP_FLAG_BRANCH) &&
1293                !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |
1294                                PERF_IP_FLAG_INTERRUPT | PERF_IP_FLAG_TX_ABORT));
1295 }
1296 
1297 static int intel_pt_sample(struct intel_pt_queue *ptq)
1298 {
1299         const struct intel_pt_state *state = ptq->state;
1300         struct intel_pt *pt = ptq->pt;
1301         int err;
1302 
1303         if (!ptq->have_sample)
1304                 return 0;
1305 
1306         ptq->have_sample = false;
1307 
1308         if (pt->sample_instructions &&
1309             (state->type & INTEL_PT_INSTRUCTION) &&
1310             (!pt->synth_opts.initial_skip ||
1311              pt->num_events++ >= pt->synth_opts.initial_skip)) {
1312                 err = intel_pt_synth_instruction_sample(ptq);
1313                 if (err)
1314                         return err;
1315         }
1316 
1317         if (pt->sample_transactions &&
1318             (state->type & INTEL_PT_TRANSACTION) &&
1319             (!pt->synth_opts.initial_skip ||
1320              pt->num_events++ >= pt->synth_opts.initial_skip)) {
1321                 err = intel_pt_synth_transaction_sample(ptq);
1322                 if (err)
1323                         return err;
1324         }
1325 
1326         if (!(state->type & INTEL_PT_BRANCH))
1327                 return 0;
1328 
1329         if (pt->synth_opts.callchain || pt->synth_opts.thread_stack)
1330                 thread_stack__event(ptq->thread, ptq->flags, state->from_ip,
1331                                     state->to_ip, ptq->insn_len,
1332                                     state->trace_nr);
1333         else
1334                 thread_stack__set_trace_nr(ptq->thread, state->trace_nr);
1335 
1336         if (pt->sample_branches) {
1337                 err = intel_pt_synth_branch_sample(ptq);
1338                 if (err)
1339                         return err;
1340         }
1341 
1342         if (pt->synth_opts.last_branch)
1343                 intel_pt_update_last_branch_rb(ptq);
1344 
1345         if (!pt->sync_switch)
1346                 return 0;
1347 
1348         if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
1349                 switch (ptq->switch_state) {
1350                 case INTEL_PT_SS_UNKNOWN:
1351                 case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1352                         err = intel_pt_next_tid(pt, ptq);
1353                         if (err)
1354                                 return err;
1355                         ptq->switch_state = INTEL_PT_SS_TRACING;
1356                         break;
1357                 default:
1358                         ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
1359                         return 1;
1360                 }
1361         } else if (!state->to_ip) {
1362                 ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
1363         } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
1364                 ptq->switch_state = INTEL_PT_SS_UNKNOWN;
1365         } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1366                    state->to_ip == pt->ptss_ip &&
1367                    (ptq->flags & PERF_IP_FLAG_CALL)) {
1368                 ptq->switch_state = INTEL_PT_SS_TRACING;
1369         }
1370 
1371         return 0;
1372 }
1373 
1374 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip)
1375 {
1376         struct machine *machine = pt->machine;
1377         struct map *map;
1378         struct symbol *sym, *start;
1379         u64 ip, switch_ip = 0;
1380         const char *ptss;
1381 
1382         if (ptss_ip)
1383                 *ptss_ip = 0;
1384 
1385         map = machine__kernel_map(machine);
1386         if (!map)
1387                 return 0;
1388 
1389         if (map__load(map))
1390                 return 0;
1391 
1392         start = dso__first_symbol(map->dso, MAP__FUNCTION);
1393 
1394         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1395                 if (sym->binding == STB_GLOBAL &&
1396                     !strcmp(sym->name, "__switch_to")) {
1397                         ip = map->unmap_ip(map, sym->start);
1398                         if (ip >= map->start && ip < map->end) {
1399                                 switch_ip = ip;
1400                                 break;
1401                         }
1402                 }
1403         }
1404 
1405         if (!switch_ip || !ptss_ip)
1406                 return 0;
1407 
1408         if (pt->have_sched_switch == 1)
1409                 ptss = "perf_trace_sched_switch";
1410         else
1411                 ptss = "__perf_event_task_sched_out";
1412 
1413         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1414                 if (!strcmp(sym->name, ptss)) {
1415                         ip = map->unmap_ip(map, sym->start);
1416                         if (ip >= map->start && ip < map->end) {
1417                                 *ptss_ip = ip;
1418                                 break;
1419                         }
1420                 }
1421         }
1422 
1423         return switch_ip;
1424 }
1425 
1426 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
1427 {
1428         const struct intel_pt_state *state = ptq->state;
1429         struct intel_pt *pt = ptq->pt;
1430         int err;
1431 
1432         if (!pt->kernel_start) {
1433                 pt->kernel_start = machine__kernel_start(pt->machine);
1434                 if (pt->per_cpu_mmaps &&
1435                     (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&
1436                     !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
1437                     !pt->sampling_mode) {
1438                         pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);
1439                         if (pt->switch_ip) {
1440                                 intel_pt_log("switch_ip: %"PRIx64" ptss_ip: %"PRIx64"\n",
1441                                              pt->switch_ip, pt->ptss_ip);
1442                                 pt->sync_switch = true;
1443                         }
1444                 }
1445         }
1446 
1447         intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
1448                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
1449         while (1) {
1450                 err = intel_pt_sample(ptq);
1451                 if (err)
1452                         return err;
1453 
1454                 state = intel_pt_decode(ptq->decoder);
1455                 if (state->err) {
1456                         if (state->err == INTEL_PT_ERR_NODATA)
1457                                 return 1;
1458                         if (pt->sync_switch &&
1459                             state->from_ip >= pt->kernel_start) {
1460                                 pt->sync_switch = false;
1461                                 intel_pt_next_tid(pt, ptq);
1462                         }
1463                         if (pt->synth_opts.errors) {
1464                                 err = intel_pt_synth_error(pt, state->err,
1465                                                            ptq->cpu, ptq->pid,
1466                                                            ptq->tid,
1467                                                            state->from_ip);
1468                                 if (err)
1469                                         return err;
1470                         }
1471                         continue;
1472                 }
1473 
1474                 ptq->state = state;
1475                 ptq->have_sample = true;
1476                 intel_pt_sample_flags(ptq);
1477 
1478                 /* Use estimated TSC upon return to user space */
1479                 if (pt->est_tsc &&
1480                     (state->from_ip >= pt->kernel_start || !state->from_ip) &&
1481                     state->to_ip && state->to_ip < pt->kernel_start) {
1482                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1483                                      state->timestamp, state->est_timestamp);
1484                         ptq->timestamp = state->est_timestamp;
1485                 /* Use estimated TSC in unknown switch state */
1486                 } else if (pt->sync_switch &&
1487                            ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1488                            intel_pt_is_switch_ip(ptq, state->to_ip) &&
1489                            ptq->next_tid == -1) {
1490                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1491                                      state->timestamp, state->est_timestamp);
1492                         ptq->timestamp = state->est_timestamp;
1493                 } else if (state->timestamp > ptq->timestamp) {
1494                         ptq->timestamp = state->timestamp;
1495                 }
1496 
1497                 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
1498                         *timestamp = ptq->timestamp;
1499                         return 0;
1500                 }
1501         }
1502         return 0;
1503 }
1504 
1505 static inline int intel_pt_update_queues(struct intel_pt *pt)
1506 {
1507         if (pt->queues.new_data) {
1508                 pt->queues.new_data = false;
1509                 return intel_pt_setup_queues(pt);
1510         }
1511         return 0;
1512 }
1513 
1514 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
1515 {
1516         unsigned int queue_nr;
1517         u64 ts;
1518         int ret;
1519 
1520         while (1) {
1521                 struct auxtrace_queue *queue;
1522                 struct intel_pt_queue *ptq;
1523 
1524                 if (!pt->heap.heap_cnt)
1525                         return 0;
1526 
1527                 if (pt->heap.heap_array[0].ordinal >= timestamp)
1528                         return 0;
1529 
1530                 queue_nr = pt->heap.heap_array[0].queue_nr;
1531                 queue = &pt->queues.queue_array[queue_nr];
1532                 ptq = queue->priv;
1533 
1534                 intel_pt_log("queue %u processing 0x%" PRIx64 " to 0x%" PRIx64 "\n",
1535                              queue_nr, pt->heap.heap_array[0].ordinal,
1536                              timestamp);
1537 
1538                 auxtrace_heap__pop(&pt->heap);
1539 
1540                 if (pt->heap.heap_cnt) {
1541                         ts = pt->heap.heap_array[0].ordinal + 1;
1542                         if (ts > timestamp)
1543                                 ts = timestamp;
1544                 } else {
1545                         ts = timestamp;
1546                 }
1547 
1548                 intel_pt_set_pid_tid_cpu(pt, queue);
1549 
1550                 ret = intel_pt_run_decoder(ptq, &ts);
1551 
1552                 if (ret < 0) {
1553                         auxtrace_heap__add(&pt->heap, queue_nr, ts);
1554                         return ret;
1555                 }
1556 
1557                 if (!ret) {
1558                         ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
1559                         if (ret < 0)
1560                                 return ret;
1561                 } else {
1562                         ptq->on_heap = false;
1563                 }
1564         }
1565 
1566         return 0;
1567 }
1568 
1569 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
1570                                             u64 time_)
1571 {
1572         struct auxtrace_queues *queues = &pt->queues;
1573         unsigned int i;
1574         u64 ts = 0;
1575 
1576         for (i = 0; i < queues->nr_queues; i++) {
1577                 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1578                 struct intel_pt_queue *ptq = queue->priv;
1579 
1580                 if (ptq && (tid == -1 || ptq->tid == tid)) {
1581                         ptq->time = time_;
1582                         intel_pt_set_pid_tid_cpu(pt, queue);
1583                         intel_pt_run_decoder(ptq, &ts);
1584                 }
1585         }
1586         return 0;
1587 }
1588 
1589 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)
1590 {
1591         return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
1592                                     sample->pid, sample->tid, 0);
1593 }
1594 
1595 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu)
1596 {
1597         unsigned i, j;
1598 
1599         if (cpu < 0 || !pt->queues.nr_queues)
1600                 return NULL;
1601 
1602         if ((unsigned)cpu >= pt->queues.nr_queues)
1603                 i = pt->queues.nr_queues - 1;
1604         else
1605                 i = cpu;
1606 
1607         if (pt->queues.queue_array[i].cpu == cpu)
1608                 return pt->queues.queue_array[i].priv;
1609 
1610         for (j = 0; i > 0; j++) {
1611                 if (pt->queues.queue_array[--i].cpu == cpu)
1612                         return pt->queues.queue_array[i].priv;
1613         }
1614 
1615         for (; j < pt->queues.nr_queues; j++) {
1616                 if (pt->queues.queue_array[j].cpu == cpu)
1617                         return pt->queues.queue_array[j].priv;
1618         }
1619 
1620         return NULL;
1621 }
1622 
1623 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid,
1624                                 u64 timestamp)
1625 {
1626         struct intel_pt_queue *ptq;
1627         int err;
1628 
1629         if (!pt->sync_switch)
1630                 return 1;
1631 
1632         ptq = intel_pt_cpu_to_ptq(pt, cpu);
1633         if (!ptq)
1634                 return 1;
1635 
1636         switch (ptq->switch_state) {
1637         case INTEL_PT_SS_NOT_TRACING:
1638                 ptq->next_tid = -1;
1639                 break;
1640         case INTEL_PT_SS_UNKNOWN:
1641         case INTEL_PT_SS_TRACING:
1642                 ptq->next_tid = tid;
1643                 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
1644                 return 0;
1645         case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
1646                 if (!ptq->on_heap) {
1647                         ptq->timestamp = perf_time_to_tsc(timestamp,
1648                                                           &pt->tc);
1649                         err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
1650                                                  ptq->timestamp);
1651                         if (err)
1652                                 return err;
1653                         ptq->on_heap = true;
1654                 }
1655                 ptq->switch_state = INTEL_PT_SS_TRACING;
1656                 break;
1657         case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1658                 ptq->next_tid = tid;
1659                 intel_pt_log("ERROR: cpu %d expecting switch ip\n", cpu);
1660                 break;
1661         default:
1662                 break;
1663         }
1664 
1665         return 1;
1666 }
1667 
1668 static int intel_pt_process_switch(struct intel_pt *pt,
1669                                    struct perf_sample *sample)
1670 {
1671         struct perf_evsel *evsel;
1672         pid_t tid;
1673         int cpu, ret;
1674 
1675         evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);
1676         if (evsel != pt->switch_evsel)
1677                 return 0;
1678 
1679         tid = perf_evsel__intval(evsel, sample, "next_pid");
1680         cpu = sample->cpu;
1681 
1682         intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1683                      cpu, tid, sample->time, perf_time_to_tsc(sample->time,
1684                      &pt->tc));
1685 
1686         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1687         if (ret <= 0)
1688                 return ret;
1689 
1690         return machine__set_current_tid(pt->machine, cpu, -1, tid);
1691 }
1692 
1693 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event,
1694                                    struct perf_sample *sample)
1695 {
1696         bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
1697         pid_t pid, tid;
1698         int cpu, ret;
1699 
1700         cpu = sample->cpu;
1701 
1702         if (pt->have_sched_switch == 3) {
1703                 if (!out)
1704                         return 0;
1705                 if (event->header.type != PERF_RECORD_SWITCH_CPU_WIDE) {
1706                         pr_err("Expecting CPU-wide context switch event\n");
1707                         return -EINVAL;
1708                 }
1709                 pid = event->context_switch.next_prev_pid;
1710                 tid = event->context_switch.next_prev_tid;
1711         } else {
1712                 if (out)
1713                         return 0;
1714                 pid = sample->pid;
1715                 tid = sample->tid;
1716         }
1717 
1718         if (tid == -1) {
1719                 pr_err("context_switch event has no tid\n");
1720                 return -EINVAL;
1721         }
1722 
1723         intel_pt_log("context_switch: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1724                      cpu, pid, tid, sample->time, perf_time_to_tsc(sample->time,
1725                      &pt->tc));
1726 
1727         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1728         if (ret <= 0)
1729                 return ret;
1730 
1731         return machine__set_current_tid(pt->machine, cpu, pid, tid);
1732 }
1733 
1734 static int intel_pt_process_itrace_start(struct intel_pt *pt,
1735                                          union perf_event *event,
1736                                          struct perf_sample *sample)
1737 {
1738         if (!pt->per_cpu_mmaps)
1739                 return 0;
1740 
1741         intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1742                      sample->cpu, event->itrace_start.pid,
1743                      event->itrace_start.tid, sample->time,
1744                      perf_time_to_tsc(sample->time, &pt->tc));
1745 
1746         return machine__set_current_tid(pt->machine, sample->cpu,
1747                                         event->itrace_start.pid,
1748                                         event->itrace_start.tid);
1749 }
1750 
1751 static int intel_pt_process_event(struct perf_session *session,
1752                                   union perf_event *event,
1753                                   struct perf_sample *sample,
1754                                   struct perf_tool *tool)
1755 {
1756         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1757                                            auxtrace);
1758         u64 timestamp;
1759         int err = 0;
1760 
1761         if (dump_trace)
1762                 return 0;
1763 
1764         if (!tool->ordered_events) {
1765                 pr_err("Intel Processor Trace requires ordered events\n");
1766                 return -EINVAL;
1767         }
1768 
1769         if (sample->time && sample->time != (u64)-1)
1770                 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
1771         else
1772                 timestamp = 0;
1773 
1774         if (timestamp || pt->timeless_decoding) {
1775                 err = intel_pt_update_queues(pt);
1776                 if (err)
1777                         return err;
1778         }
1779 
1780         if (pt->timeless_decoding) {
1781                 if (event->header.type == PERF_RECORD_EXIT) {
1782                         err = intel_pt_process_timeless_queues(pt,
1783                                                                event->fork.tid,
1784                                                                sample->time);
1785                 }
1786         } else if (timestamp) {
1787                 err = intel_pt_process_queues(pt, timestamp);
1788         }
1789         if (err)
1790                 return err;
1791 
1792         if (event->header.type == PERF_RECORD_AUX &&
1793             (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&
1794             pt->synth_opts.errors) {
1795                 err = intel_pt_lost(pt, sample);
1796                 if (err)
1797                         return err;
1798         }
1799 
1800         if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
1801                 err = intel_pt_process_switch(pt, sample);
1802         else if (event->header.type == PERF_RECORD_ITRACE_START)
1803                 err = intel_pt_process_itrace_start(pt, event, sample);
1804         else if (event->header.type == PERF_RECORD_SWITCH ||
1805                  event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
1806                 err = intel_pt_context_switch(pt, event, sample);
1807 
1808         intel_pt_log("event %s (%u): cpu %d time %"PRIu64" tsc %#"PRIx64"\n",
1809                      perf_event__name(event->header.type), event->header.type,
1810                      sample->cpu, sample->time, timestamp);
1811 
1812         return err;
1813 }
1814 
1815 static int intel_pt_flush(struct perf_session *session, struct perf_tool *tool)
1816 {
1817         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1818                                            auxtrace);
1819         int ret;
1820 
1821         if (dump_trace)
1822                 return 0;
1823 
1824         if (!tool->ordered_events)
1825                 return -EINVAL;
1826 
1827         ret = intel_pt_update_queues(pt);
1828         if (ret < 0)
1829                 return ret;
1830 
1831         if (pt->timeless_decoding)
1832                 return intel_pt_process_timeless_queues(pt, -1,
1833                                                         MAX_TIMESTAMP - 1);
1834 
1835         return intel_pt_process_queues(pt, MAX_TIMESTAMP);
1836 }
1837 
1838 static void intel_pt_free_events(struct perf_session *session)
1839 {
1840         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1841                                            auxtrace);
1842         struct auxtrace_queues *queues = &pt->queues;
1843         unsigned int i;
1844 
1845         for (i = 0; i < queues->nr_queues; i++) {
1846                 intel_pt_free_queue(queues->queue_array[i].priv);
1847                 queues->queue_array[i].priv = NULL;
1848         }
1849         intel_pt_log_disable();
1850         auxtrace_queues__free(queues);
1851 }
1852 
1853 static void intel_pt_free(struct perf_session *session)
1854 {
1855         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1856                                            auxtrace);
1857 
1858         auxtrace_heap__free(&pt->heap);
1859         intel_pt_free_events(session);
1860         session->auxtrace = NULL;
1861         thread__put(pt->unknown_thread);
1862         addr_filters__exit(&pt->filts);
1863         zfree(&pt->filter);
1864         free(pt);
1865 }
1866 
1867 static int intel_pt_process_auxtrace_event(struct perf_session *session,
1868                                            union perf_event *event,
1869                                            struct perf_tool *tool __maybe_unused)
1870 {
1871         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1872                                            auxtrace);
1873 
1874         if (pt->sampling_mode)
1875                 return 0;
1876 
1877         if (!pt->data_queued) {
1878                 struct auxtrace_buffer *buffer;
1879                 off_t data_offset;
1880                 int fd = perf_data_file__fd(session->file);
1881                 int err;
1882 
1883                 if (perf_data_file__is_pipe(session->file)) {
1884                         data_offset = 0;
1885                 } else {
1886                         data_offset = lseek(fd, 0, SEEK_CUR);
1887                         if (data_offset == -1)
1888                                 return -errno;
1889                 }
1890 
1891                 err = auxtrace_queues__add_event(&pt->queues, session, event,
1892                                                  data_offset, &buffer);
1893                 if (err)
1894                         return err;
1895 
1896                 /* Dump here now we have copied a piped trace out of the pipe */
1897                 if (dump_trace) {
1898                         if (auxtrace_buffer__get_data(buffer, fd)) {
1899                                 intel_pt_dump_event(pt, buffer->data,
1900                                                     buffer->size);
1901                                 auxtrace_buffer__put_data(buffer);
1902                         }
1903                 }
1904         }
1905 
1906         return 0;
1907 }
1908 
1909 struct intel_pt_synth {
1910         struct perf_tool dummy_tool;
1911         struct perf_session *session;
1912 };
1913 
1914 static int intel_pt_event_synth(struct perf_tool *tool,
1915                                 union perf_event *event,
1916                                 struct perf_sample *sample __maybe_unused,
1917                                 struct machine *machine __maybe_unused)
1918 {
1919         struct intel_pt_synth *intel_pt_synth =
1920                         container_of(tool, struct intel_pt_synth, dummy_tool);
1921 
1922         return perf_session__deliver_synth_event(intel_pt_synth->session, event,
1923                                                  NULL);
1924 }
1925 
1926 static int intel_pt_synth_event(struct perf_session *session,
1927                                 struct perf_event_attr *attr, u64 id)
1928 {
1929         struct intel_pt_synth intel_pt_synth;
1930 
1931         memset(&intel_pt_synth, 0, sizeof(struct intel_pt_synth));
1932         intel_pt_synth.session = session;
1933 
1934         return perf_event__synthesize_attr(&intel_pt_synth.dummy_tool, attr, 1,
1935                                            &id, intel_pt_event_synth);
1936 }
1937 
1938 static int intel_pt_synth_events(struct intel_pt *pt,
1939                                  struct perf_session *session)
1940 {
1941         struct perf_evlist *evlist = session->evlist;
1942         struct perf_evsel *evsel;
1943         struct perf_event_attr attr;
1944         bool found = false;
1945         u64 id;
1946         int err;
1947 
1948         evlist__for_each_entry(evlist, evsel) {
1949                 if (evsel->attr.type == pt->pmu_type && evsel->ids) {
1950                         found = true;
1951                         break;
1952                 }
1953         }
1954 
1955         if (!found) {
1956                 pr_debug("There are no selected events with Intel Processor Trace data\n");
1957                 return 0;
1958         }
1959 
1960         memset(&attr, 0, sizeof(struct perf_event_attr));
1961         attr.size = sizeof(struct perf_event_attr);
1962         attr.type = PERF_TYPE_HARDWARE;
1963         attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
1964         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1965                             PERF_SAMPLE_PERIOD;
1966         if (pt->timeless_decoding)
1967                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1968         else
1969                 attr.sample_type |= PERF_SAMPLE_TIME;
1970         if (!pt->per_cpu_mmaps)
1971                 attr.sample_type &= ~(u64)PERF_SAMPLE_CPU;
1972         attr.exclude_user = evsel->attr.exclude_user;
1973         attr.exclude_kernel = evsel->attr.exclude_kernel;
1974         attr.exclude_hv = evsel->attr.exclude_hv;
1975         attr.exclude_host = evsel->attr.exclude_host;
1976         attr.exclude_guest = evsel->attr.exclude_guest;
1977         attr.sample_id_all = evsel->attr.sample_id_all;
1978         attr.read_format = evsel->attr.read_format;
1979 
1980         id = evsel->id[0] + 1000000000;
1981         if (!id)
1982                 id = 1;
1983 
1984         if (pt->synth_opts.instructions) {
1985                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1986                 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
1987                         attr.sample_period =
1988                                 intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
1989                 else
1990                         attr.sample_period = pt->synth_opts.period;
1991                 pt->instructions_sample_period = attr.sample_period;
1992                 if (pt->synth_opts.callchain)
1993                         attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
1994                 if (pt->synth_opts.last_branch)
1995                         attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1996                 pr_debug("Synthesizing 'instructions' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1997                          id, (u64)attr.sample_type);
1998                 err = intel_pt_synth_event(session, &attr, id);
1999                 if (err) {
2000                         pr_err("%s: failed to synthesize 'instructions' event type\n",
2001                                __func__);
2002                         return err;
2003                 }
2004                 pt->sample_instructions = true;
2005                 pt->instructions_sample_type = attr.sample_type;
2006                 pt->instructions_id = id;
2007                 id += 1;
2008         }
2009 
2010         if (pt->synth_opts.transactions) {
2011                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
2012                 attr.sample_period = 1;
2013                 if (pt->synth_opts.callchain)
2014                         attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
2015                 if (pt->synth_opts.last_branch)
2016                         attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
2017                 pr_debug("Synthesizing 'transactions' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
2018                          id, (u64)attr.sample_type);
2019                 err = intel_pt_synth_event(session, &attr, id);
2020                 if (err) {
2021                         pr_err("%s: failed to synthesize 'transactions' event type\n",
2022                                __func__);
2023                         return err;
2024                 }
2025                 pt->sample_transactions = true;
2026                 pt->transactions_id = id;
2027                 id += 1;
2028                 evlist__for_each_entry(evlist, evsel) {
2029                         if (evsel->id && evsel->id[0] == pt->transactions_id) {
2030                                 if (evsel->name)
2031                                         zfree(&evsel->name);
2032                                 evsel->name = strdup("transactions");
2033                                 break;
2034                         }
2035                 }
2036         }
2037 
2038         if (pt->synth_opts.branches) {
2039                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
2040                 attr.sample_period = 1;
2041                 attr.sample_type |= PERF_SAMPLE_ADDR;
2042                 attr.sample_type &= ~(u64)PERF_SAMPLE_CALLCHAIN;
2043                 attr.sample_type &= ~(u64)PERF_SAMPLE_BRANCH_STACK;
2044                 pr_debug("Synthesizing 'branches' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
2045                          id, (u64)attr.sample_type);
2046                 err = intel_pt_synth_event(session, &attr, id);
2047                 if (err) {
2048                         pr_err("%s: failed to synthesize 'branches' event type\n",
2049                                __func__);
2050                         return err;
2051                 }
2052                 pt->sample_branches = true;
2053                 pt->branches_sample_type = attr.sample_type;
2054                 pt->branches_id = id;
2055         }
2056 
2057         pt->synth_needs_swap = evsel->needs_swap;
2058 
2059         return 0;
2060 }
2061 
2062 static struct perf_evsel *intel_pt_find_sched_switch(struct perf_evlist *evlist)
2063 {
2064         struct perf_evsel *evsel;
2065 
2066         evlist__for_each_entry_reverse(evlist, evsel) {
2067                 const char *name = perf_evsel__name(evsel);
2068 
2069                 if (!strcmp(name, "sched:sched_switch"))
2070                         return evsel;
2071         }
2072 
2073         return NULL;
2074 }
2075 
2076 static bool intel_pt_find_switch(struct perf_evlist *evlist)
2077 {
2078         struct perf_evsel *evsel;
2079 
2080         evlist__for_each_entry(evlist, evsel) {
2081                 if (evsel->attr.context_switch)
2082                         return true;
2083         }
2084 
2085         return false;
2086 }
2087 
2088 static int intel_pt_perf_config(const char *var, const char *value, void *data)
2089 {
2090         struct intel_pt *pt = data;
2091 
2092         if (!strcmp(var, "intel-pt.mispred-all"))
2093                 pt->mispred_all = perf_config_bool(var, value);
2094 
2095         return 0;
2096 }
2097 
2098 static const char * const intel_pt_info_fmts[] = {
2099         [INTEL_PT_PMU_TYPE]             = "  PMU Type            %"PRId64"\n",
2100         [INTEL_PT_TIME_SHIFT]           = "  Time Shift          %"PRIu64"\n",
2101         [INTEL_PT_TIME_MULT]            = "  Time Muliplier      %"PRIu64"\n",
2102         [INTEL_PT_TIME_ZERO]            = "  Time Zero           %"PRIu64"\n",
2103         [INTEL_PT_CAP_USER_TIME_ZERO]   = "  Cap Time Zero       %"PRId64"\n",
2104         [INTEL_PT_TSC_BIT]              = "  TSC bit             %#"PRIx64"\n",
2105         [INTEL_PT_NORETCOMP_BIT]        = "  NoRETComp bit       %#"PRIx64"\n",
2106         [INTEL_PT_HAVE_SCHED_SWITCH]    = "  Have sched_switch   %"PRId64"\n",
2107         [INTEL_PT_SNAPSHOT_MODE]        = "  Snapshot mode       %"PRId64"\n",
2108         [INTEL_PT_PER_CPU_MMAPS]        = "  Per-cpu maps        %"PRId64"\n",
2109         [INTEL_PT_MTC_BIT]              = "  MTC bit             %#"PRIx64"\n",
2110         [INTEL_PT_TSC_CTC_N]            = "  TSC:CTC numerator   %"PRIu64"\n",
2111         [INTEL_PT_TSC_CTC_D]            = "  TSC:CTC denominator %"PRIu64"\n",
2112         [INTEL_PT_CYC_BIT]              = "  CYC bit             %#"PRIx64"\n",
2113         [INTEL_PT_MAX_NONTURBO_RATIO]   = "  Max non-turbo ratio %"PRIu64"\n",
2114         [INTEL_PT_FILTER_STR_LEN]       = "  Filter string len.  %"PRIu64"\n",
2115 };
2116 
2117 static void intel_pt_print_info(u64 *arr, int start, int finish)
2118 {
2119         int i;
2120 
2121         if (!dump_trace)
2122                 return;
2123 
2124         for (i = start; i <= finish; i++)
2125                 fprintf(stdout, intel_pt_info_fmts[i], arr[i]);
2126 }
2127 
2128 static void intel_pt_print_info_str(const char *name, const char *str)
2129 {
2130         if (!dump_trace)
2131                 return;
2132 
2133         fprintf(stdout, "  %-20s%s\n", name, str ? str : "");
2134 }
2135 
2136 static bool intel_pt_has(struct auxtrace_info_event *auxtrace_info, int pos)
2137 {
2138         return auxtrace_info->header.size >=
2139                 sizeof(struct auxtrace_info_event) + (sizeof(u64) * (pos + 1));
2140 }
2141 
2142 int intel_pt_process_auxtrace_info(union perf_event *event,
2143                                    struct perf_session *session)
2144 {
2145         struct auxtrace_info_event *auxtrace_info = &event->auxtrace_info;
2146         size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS;
2147         struct intel_pt *pt;
2148         void *info_end;
2149         u64 *info;
2150         int err;
2151 
2152         if (auxtrace_info->header.size < sizeof(struct auxtrace_info_event) +
2153                                         min_sz)
2154                 return -EINVAL;
2155 
2156         pt = zalloc(sizeof(struct intel_pt));
2157         if (!pt)
2158                 return -ENOMEM;
2159 
2160         addr_filters__init(&pt->filts);
2161 
2162         perf_config(intel_pt_perf_config, pt);
2163 
2164         err = auxtrace_queues__init(&pt->queues);
2165         if (err)
2166                 goto err_free;
2167 
2168         intel_pt_log_set_name(INTEL_PT_PMU_NAME);
2169 
2170         pt->session = session;
2171         pt->machine = &session->machines.host; /* No kvm support */
2172         pt->auxtrace_type = auxtrace_info->type;
2173         pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
2174         pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
2175         pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
2176         pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
2177         pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
2178         pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
2179         pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
2180         pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
2181         pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
2182         pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
2183         intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,
2184                             INTEL_PT_PER_CPU_MMAPS);
2185 
2186         if (intel_pt_has(auxtrace_info, INTEL_PT_CYC_BIT)) {
2187                 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
2188                 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
2189                 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
2190                 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
2191                 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
2192                 intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,
2193                                     INTEL_PT_CYC_BIT);
2194         }
2195 
2196         if (intel_pt_has(auxtrace_info, INTEL_PT_MAX_NONTURBO_RATIO)) {
2197                 pt->max_non_turbo_ratio =
2198                         auxtrace_info->priv[INTEL_PT_MAX_NONTURBO_RATIO];
2199                 intel_pt_print_info(&auxtrace_info->priv[0],
2200                                     INTEL_PT_MAX_NONTURBO_RATIO,
2201                                     INTEL_PT_MAX_NONTURBO_RATIO);
2202         }
2203 
2204         info = &auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] + 1;
2205         info_end = (void *)info + auxtrace_info->header.size;
2206 
2207         if (intel_pt_has(auxtrace_info, INTEL_PT_FILTER_STR_LEN)) {
2208                 size_t len;
2209 
2210                 len = auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN];
2211                 intel_pt_print_info(&auxtrace_info->priv[0],
2212                                     INTEL_PT_FILTER_STR_LEN,
2213                                     INTEL_PT_FILTER_STR_LEN);
2214                 if (len) {
2215                         const char *filter = (const char *)info;
2216 
2217                         len = roundup(len + 1, 8);
2218                         info += len >> 3;
2219                         if ((void *)info > info_end) {
2220                                 pr_err("%s: bad filter string length\n", __func__);
2221                                 err = -EINVAL;
2222                                 goto err_free_queues;
2223                         }
2224                         pt->filter = memdup(filter, len);
2225                         if (!pt->filter) {
2226                                 err = -ENOMEM;
2227                                 goto err_free_queues;
2228                         }
2229                         if (session->header.needs_swap)
2230                                 mem_bswap_64(pt->filter, len);
2231                         if (pt->filter[len - 1]) {
2232                                 pr_err("%s: filter string not null terminated\n", __func__);
2233                                 err = -EINVAL;
2234                                 goto err_free_queues;
2235                         }
2236                         err = addr_filters__parse_bare_filter(&pt->filts,
2237                                                               filter);
2238                         if (err)
2239                                 goto err_free_queues;
2240                 }
2241                 intel_pt_print_info_str("Filter string", pt->filter);
2242         }
2243 
2244         pt->timeless_decoding = intel_pt_timeless_decoding(pt);
2245         pt->have_tsc = intel_pt_have_tsc(pt);
2246         pt->sampling_mode = false;
2247         pt->est_tsc = !pt->timeless_decoding;
2248 
2249         pt->unknown_thread = thread__new(999999999, 999999999);
2250         if (!pt->unknown_thread) {
2251                 err = -ENOMEM;
2252                 goto err_free_queues;
2253         }
2254 
2255         /*
2256          * Since this thread will not be kept in any rbtree not in a
2257          * list, initialize its list node so that at thread__put() the
2258          * current thread lifetime assuption is kept and we don't segfault
2259          * at list_del_init().
2260          */
2261         INIT_LIST_HEAD(&pt->unknown_thread->node);
2262 
2263         err = thread__set_comm(pt->unknown_thread, "unknown", 0);
2264         if (err)
2265                 goto err_delete_thread;
2266         if (thread__init_map_groups(pt->unknown_thread, pt->machine)) {
2267                 err = -ENOMEM;
2268                 goto err_delete_thread;
2269         }
2270 
2271         pt->auxtrace.process_event = intel_pt_process_event;
2272         pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
2273         pt->auxtrace.flush_events = intel_pt_flush;
2274         pt->auxtrace.free_events = intel_pt_free_events;
2275         pt->auxtrace.free = intel_pt_free;
2276         session->auxtrace = &pt->auxtrace;
2277 
2278         if (dump_trace)
2279                 return 0;
2280 
2281         if (pt->have_sched_switch == 1) {
2282                 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
2283                 if (!pt->switch_evsel) {
2284                         pr_err("%s: missing sched_switch event\n", __func__);
2285                         err = -EINVAL;
2286                         goto err_delete_thread;
2287                 }
2288         } else if (pt->have_sched_switch == 2 &&
2289                    !intel_pt_find_switch(session->evlist)) {
2290                 pr_err("%s: missing context_switch attribute flag\n", __func__);
2291                 err = -EINVAL;
2292                 goto err_delete_thread;
2293         }
2294 
2295         if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
2296                 pt->synth_opts = *session->itrace_synth_opts;
2297         } else {
2298                 itrace_synth_opts__set_default(&pt->synth_opts);
2299                 if (use_browser != -1) {
2300                         pt->synth_opts.branches = false;
2301                         pt->synth_opts.callchain = true;
2302                 }
2303                 if (session->itrace_synth_opts)
2304                         pt->synth_opts.thread_stack =
2305                                 session->itrace_synth_opts->thread_stack;
2306         }
2307 
2308         if (pt->synth_opts.log)
2309                 intel_pt_log_enable();
2310 
2311         /* Maximum non-turbo ratio is TSC freq / 100 MHz */
2312         if (pt->tc.time_mult) {
2313                 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
2314 
2315                 if (!pt->max_non_turbo_ratio)
2316                         pt->max_non_turbo_ratio =
2317                                         (tsc_freq + 50000000) / 100000000;
2318                 intel_pt_log("TSC frequency %"PRIu64"\n", tsc_freq);
2319                 intel_pt_log("Maximum non-turbo ratio %u\n",
2320                              pt->max_non_turbo_ratio);
2321         }
2322 
2323         if (pt->synth_opts.calls)
2324                 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
2325                                        PERF_IP_FLAG_TRACE_END;
2326         if (pt->synth_opts.returns)
2327                 pt->branches_filter |= PERF_IP_FLAG_RETURN |
2328                                        PERF_IP_FLAG_TRACE_BEGIN;
2329 
2330         if (pt->synth_opts.callchain && !symbol_conf.use_callchain) {
2331                 symbol_conf.use_callchain = true;
2332                 if (callchain_register_param(&callchain_param) < 0) {
2333                         symbol_conf.use_callchain = false;
2334                         pt->synth_opts.callchain = false;
2335                 }
2336         }
2337 
2338         err = intel_pt_synth_events(pt, session);
2339         if (err)
2340                 goto err_delete_thread;
2341 
2342         err = auxtrace_queues__process_index(&pt->queues, session);
2343         if (err)
2344                 goto err_delete_thread;
2345 
2346         if (pt->queues.populated)
2347                 pt->data_queued = true;
2348 
2349         if (pt->timeless_decoding)
2350                 pr_debug2("Intel PT decoding without timestamps\n");
2351 
2352         return 0;
2353 
2354 err_delete_thread:
2355         thread__zput(pt->unknown_thread);
2356 err_free_queues:
2357         intel_pt_log_disable();
2358         auxtrace_queues__free(&pt->queues);
2359         session->auxtrace = NULL;
2360 err_free:
2361         addr_filters__exit(&pt->filts);
2362         zfree(&pt->filter);
2363         free(pt);
2364         return err;
2365 }
2366 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp