~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/events/intel/ds.c

Version: ~ [ linux-5.4-rc7 ] ~ [ linux-5.3.11 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.84 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.154 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.201 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.201 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.77 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 #include <linux/bitops.h>
  2 #include <linux/types.h>
  3 #include <linux/slab.h>
  4 
  5 #include <asm/perf_event.h>
  6 #include <asm/insn.h>
  7 
  8 #include "../perf_event.h"
  9 
 10 /* The size of a BTS record in bytes: */
 11 #define BTS_RECORD_SIZE         24
 12 
 13 #define BTS_BUFFER_SIZE         (PAGE_SIZE << 4)
 14 #define PEBS_BUFFER_SIZE        (PAGE_SIZE << 4)
 15 #define PEBS_FIXUP_SIZE         PAGE_SIZE
 16 
 17 /*
 18  * pebs_record_32 for p4 and core not supported
 19 
 20 struct pebs_record_32 {
 21         u32 flags, ip;
 22         u32 ax, bc, cx, dx;
 23         u32 si, di, bp, sp;
 24 };
 25 
 26  */
 27 
 28 union intel_x86_pebs_dse {
 29         u64 val;
 30         struct {
 31                 unsigned int ld_dse:4;
 32                 unsigned int ld_stlb_miss:1;
 33                 unsigned int ld_locked:1;
 34                 unsigned int ld_reserved:26;
 35         };
 36         struct {
 37                 unsigned int st_l1d_hit:1;
 38                 unsigned int st_reserved1:3;
 39                 unsigned int st_stlb_miss:1;
 40                 unsigned int st_locked:1;
 41                 unsigned int st_reserved2:26;
 42         };
 43 };
 44 
 45 
 46 /*
 47  * Map PEBS Load Latency Data Source encodings to generic
 48  * memory data source information
 49  */
 50 #define P(a, b) PERF_MEM_S(a, b)
 51 #define OP_LH (P(OP, LOAD) | P(LVL, HIT))
 52 #define SNOOP_NONE_MISS (P(SNOOP, NONE) | P(SNOOP, MISS))
 53 
 54 /* Version for Sandy Bridge and later */
 55 static u64 pebs_data_source[] = {
 56         P(OP, LOAD) | P(LVL, MISS) | P(LVL, L3) | P(SNOOP, NA),/* 0x00:ukn L3 */
 57         OP_LH | P(LVL, L1)  | P(SNOOP, NONE),   /* 0x01: L1 local */
 58         OP_LH | P(LVL, LFB) | P(SNOOP, NONE),   /* 0x02: LFB hit */
 59         OP_LH | P(LVL, L2)  | P(SNOOP, NONE),   /* 0x03: L2 hit */
 60         OP_LH | P(LVL, L3)  | P(SNOOP, NONE),   /* 0x04: L3 hit */
 61         OP_LH | P(LVL, L3)  | P(SNOOP, MISS),   /* 0x05: L3 hit, snoop miss */
 62         OP_LH | P(LVL, L3)  | P(SNOOP, HIT),    /* 0x06: L3 hit, snoop hit */
 63         OP_LH | P(LVL, L3)  | P(SNOOP, HITM),   /* 0x07: L3 hit, snoop hitm */
 64         OP_LH | P(LVL, REM_CCE1) | P(SNOOP, HIT),  /* 0x08: L3 miss snoop hit */
 65         OP_LH | P(LVL, REM_CCE1) | P(SNOOP, HITM), /* 0x09: L3 miss snoop hitm*/
 66         OP_LH | P(LVL, LOC_RAM)  | P(SNOOP, HIT),  /* 0x0a: L3 miss, shared */
 67         OP_LH | P(LVL, REM_RAM1) | P(SNOOP, HIT),  /* 0x0b: L3 miss, shared */
 68         OP_LH | P(LVL, LOC_RAM)  | SNOOP_NONE_MISS,/* 0x0c: L3 miss, excl */
 69         OP_LH | P(LVL, REM_RAM1) | SNOOP_NONE_MISS,/* 0x0d: L3 miss, excl */
 70         OP_LH | P(LVL, IO)  | P(SNOOP, NONE), /* 0x0e: I/O */
 71         OP_LH | P(LVL, UNC) | P(SNOOP, NONE), /* 0x0f: uncached */
 72 };
 73 
 74 /* Patch up minor differences in the bits */
 75 void __init intel_pmu_pebs_data_source_nhm(void)
 76 {
 77         pebs_data_source[0x05] = OP_LH | P(LVL, L3)  | P(SNOOP, HIT);
 78         pebs_data_source[0x06] = OP_LH | P(LVL, L3)  | P(SNOOP, HITM);
 79         pebs_data_source[0x07] = OP_LH | P(LVL, L3)  | P(SNOOP, HITM);
 80 }
 81 
 82 static u64 precise_store_data(u64 status)
 83 {
 84         union intel_x86_pebs_dse dse;
 85         u64 val = P(OP, STORE) | P(SNOOP, NA) | P(LVL, L1) | P(TLB, L2);
 86 
 87         dse.val = status;
 88 
 89         /*
 90          * bit 4: TLB access
 91          * 1 = stored missed 2nd level TLB
 92          *
 93          * so it either hit the walker or the OS
 94          * otherwise hit 2nd level TLB
 95          */
 96         if (dse.st_stlb_miss)
 97                 val |= P(TLB, MISS);
 98         else
 99                 val |= P(TLB, HIT);
100 
101         /*
102          * bit 0: hit L1 data cache
103          * if not set, then all we know is that
104          * it missed L1D
105          */
106         if (dse.st_l1d_hit)
107                 val |= P(LVL, HIT);
108         else
109                 val |= P(LVL, MISS);
110 
111         /*
112          * bit 5: Locked prefix
113          */
114         if (dse.st_locked)
115                 val |= P(LOCK, LOCKED);
116 
117         return val;
118 }
119 
120 static u64 precise_datala_hsw(struct perf_event *event, u64 status)
121 {
122         union perf_mem_data_src dse;
123 
124         dse.val = PERF_MEM_NA;
125 
126         if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW)
127                 dse.mem_op = PERF_MEM_OP_STORE;
128         else if (event->hw.flags & PERF_X86_EVENT_PEBS_LD_HSW)
129                 dse.mem_op = PERF_MEM_OP_LOAD;
130 
131         /*
132          * L1 info only valid for following events:
133          *
134          * MEM_UOPS_RETIRED.STLB_MISS_STORES
135          * MEM_UOPS_RETIRED.LOCK_STORES
136          * MEM_UOPS_RETIRED.SPLIT_STORES
137          * MEM_UOPS_RETIRED.ALL_STORES
138          */
139         if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW) {
140                 if (status & 1)
141                         dse.mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT;
142                 else
143                         dse.mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_MISS;
144         }
145         return dse.val;
146 }
147 
148 static u64 load_latency_data(u64 status)
149 {
150         union intel_x86_pebs_dse dse;
151         u64 val;
152         int model = boot_cpu_data.x86_model;
153         int fam = boot_cpu_data.x86;
154 
155         dse.val = status;
156 
157         /*
158          * use the mapping table for bit 0-3
159          */
160         val = pebs_data_source[dse.ld_dse];
161 
162         /*
163          * Nehalem models do not support TLB, Lock infos
164          */
165         if (fam == 0x6 && (model == 26 || model == 30
166             || model == 31 || model == 46)) {
167                 val |= P(TLB, NA) | P(LOCK, NA);
168                 return val;
169         }
170         /*
171          * bit 4: TLB access
172          * 0 = did not miss 2nd level TLB
173          * 1 = missed 2nd level TLB
174          */
175         if (dse.ld_stlb_miss)
176                 val |= P(TLB, MISS) | P(TLB, L2);
177         else
178                 val |= P(TLB, HIT) | P(TLB, L1) | P(TLB, L2);
179 
180         /*
181          * bit 5: locked prefix
182          */
183         if (dse.ld_locked)
184                 val |= P(LOCK, LOCKED);
185 
186         return val;
187 }
188 
189 struct pebs_record_core {
190         u64 flags, ip;
191         u64 ax, bx, cx, dx;
192         u64 si, di, bp, sp;
193         u64 r8,  r9,  r10, r11;
194         u64 r12, r13, r14, r15;
195 };
196 
197 struct pebs_record_nhm {
198         u64 flags, ip;
199         u64 ax, bx, cx, dx;
200         u64 si, di, bp, sp;
201         u64 r8,  r9,  r10, r11;
202         u64 r12, r13, r14, r15;
203         u64 status, dla, dse, lat;
204 };
205 
206 /*
207  * Same as pebs_record_nhm, with two additional fields.
208  */
209 struct pebs_record_hsw {
210         u64 flags, ip;
211         u64 ax, bx, cx, dx;
212         u64 si, di, bp, sp;
213         u64 r8,  r9,  r10, r11;
214         u64 r12, r13, r14, r15;
215         u64 status, dla, dse, lat;
216         u64 real_ip, tsx_tuning;
217 };
218 
219 union hsw_tsx_tuning {
220         struct {
221                 u32 cycles_last_block     : 32,
222                     hle_abort             : 1,
223                     rtm_abort             : 1,
224                     instruction_abort     : 1,
225                     non_instruction_abort : 1,
226                     retry                 : 1,
227                     data_conflict         : 1,
228                     capacity_writes       : 1,
229                     capacity_reads        : 1;
230         };
231         u64         value;
232 };
233 
234 #define PEBS_HSW_TSX_FLAGS      0xff00000000ULL
235 
236 /* Same as HSW, plus TSC */
237 
238 struct pebs_record_skl {
239         u64 flags, ip;
240         u64 ax, bx, cx, dx;
241         u64 si, di, bp, sp;
242         u64 r8,  r9,  r10, r11;
243         u64 r12, r13, r14, r15;
244         u64 status, dla, dse, lat;
245         u64 real_ip, tsx_tuning;
246         u64 tsc;
247 };
248 
249 void init_debug_store_on_cpu(int cpu)
250 {
251         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
252 
253         if (!ds)
254                 return;
255 
256         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
257                      (u32)((u64)(unsigned long)ds),
258                      (u32)((u64)(unsigned long)ds >> 32));
259 }
260 
261 void fini_debug_store_on_cpu(int cpu)
262 {
263         if (!per_cpu(cpu_hw_events, cpu).ds)
264                 return;
265 
266         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
267 }
268 
269 static DEFINE_PER_CPU(void *, insn_buffer);
270 
271 static int alloc_pebs_buffer(int cpu)
272 {
273         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
274         int node = cpu_to_node(cpu);
275         int max;
276         void *buffer, *ibuffer;
277 
278         if (!x86_pmu.pebs)
279                 return 0;
280 
281         buffer = kzalloc_node(x86_pmu.pebs_buffer_size, GFP_KERNEL, node);
282         if (unlikely(!buffer))
283                 return -ENOMEM;
284 
285         /*
286          * HSW+ already provides us the eventing ip; no need to allocate this
287          * buffer then.
288          */
289         if (x86_pmu.intel_cap.pebs_format < 2) {
290                 ibuffer = kzalloc_node(PEBS_FIXUP_SIZE, GFP_KERNEL, node);
291                 if (!ibuffer) {
292                         kfree(buffer);
293                         return -ENOMEM;
294                 }
295                 per_cpu(insn_buffer, cpu) = ibuffer;
296         }
297 
298         max = x86_pmu.pebs_buffer_size / x86_pmu.pebs_record_size;
299 
300         ds->pebs_buffer_base = (u64)(unsigned long)buffer;
301         ds->pebs_index = ds->pebs_buffer_base;
302         ds->pebs_absolute_maximum = ds->pebs_buffer_base +
303                 max * x86_pmu.pebs_record_size;
304 
305         return 0;
306 }
307 
308 static void release_pebs_buffer(int cpu)
309 {
310         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
311 
312         if (!ds || !x86_pmu.pebs)
313                 return;
314 
315         kfree(per_cpu(insn_buffer, cpu));
316         per_cpu(insn_buffer, cpu) = NULL;
317 
318         kfree((void *)(unsigned long)ds->pebs_buffer_base);
319         ds->pebs_buffer_base = 0;
320 }
321 
322 static int alloc_bts_buffer(int cpu)
323 {
324         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
325         int node = cpu_to_node(cpu);
326         int max, thresh;
327         void *buffer;
328 
329         if (!x86_pmu.bts)
330                 return 0;
331 
332         buffer = kzalloc_node(BTS_BUFFER_SIZE, GFP_KERNEL | __GFP_NOWARN, node);
333         if (unlikely(!buffer)) {
334                 WARN_ONCE(1, "%s: BTS buffer allocation failure\n", __func__);
335                 return -ENOMEM;
336         }
337 
338         max = BTS_BUFFER_SIZE / BTS_RECORD_SIZE;
339         thresh = max / 16;
340 
341         ds->bts_buffer_base = (u64)(unsigned long)buffer;
342         ds->bts_index = ds->bts_buffer_base;
343         ds->bts_absolute_maximum = ds->bts_buffer_base +
344                 max * BTS_RECORD_SIZE;
345         ds->bts_interrupt_threshold = ds->bts_absolute_maximum -
346                 thresh * BTS_RECORD_SIZE;
347 
348         return 0;
349 }
350 
351 static void release_bts_buffer(int cpu)
352 {
353         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
354 
355         if (!ds || !x86_pmu.bts)
356                 return;
357 
358         kfree((void *)(unsigned long)ds->bts_buffer_base);
359         ds->bts_buffer_base = 0;
360 }
361 
362 static int alloc_ds_buffer(int cpu)
363 {
364         int node = cpu_to_node(cpu);
365         struct debug_store *ds;
366 
367         ds = kzalloc_node(sizeof(*ds), GFP_KERNEL, node);
368         if (unlikely(!ds))
369                 return -ENOMEM;
370 
371         per_cpu(cpu_hw_events, cpu).ds = ds;
372 
373         return 0;
374 }
375 
376 static void release_ds_buffer(int cpu)
377 {
378         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
379 
380         if (!ds)
381                 return;
382 
383         per_cpu(cpu_hw_events, cpu).ds = NULL;
384         kfree(ds);
385 }
386 
387 void release_ds_buffers(void)
388 {
389         int cpu;
390 
391         if (!x86_pmu.bts && !x86_pmu.pebs)
392                 return;
393 
394         get_online_cpus();
395         for_each_online_cpu(cpu)
396                 fini_debug_store_on_cpu(cpu);
397 
398         for_each_possible_cpu(cpu) {
399                 release_pebs_buffer(cpu);
400                 release_bts_buffer(cpu);
401                 release_ds_buffer(cpu);
402         }
403         put_online_cpus();
404 }
405 
406 void reserve_ds_buffers(void)
407 {
408         int bts_err = 0, pebs_err = 0;
409         int cpu;
410 
411         x86_pmu.bts_active = 0;
412         x86_pmu.pebs_active = 0;
413 
414         if (!x86_pmu.bts && !x86_pmu.pebs)
415                 return;
416 
417         if (!x86_pmu.bts)
418                 bts_err = 1;
419 
420         if (!x86_pmu.pebs)
421                 pebs_err = 1;
422 
423         get_online_cpus();
424 
425         for_each_possible_cpu(cpu) {
426                 if (alloc_ds_buffer(cpu)) {
427                         bts_err = 1;
428                         pebs_err = 1;
429                 }
430 
431                 if (!bts_err && alloc_bts_buffer(cpu))
432                         bts_err = 1;
433 
434                 if (!pebs_err && alloc_pebs_buffer(cpu))
435                         pebs_err = 1;
436 
437                 if (bts_err && pebs_err)
438                         break;
439         }
440 
441         if (bts_err) {
442                 for_each_possible_cpu(cpu)
443                         release_bts_buffer(cpu);
444         }
445 
446         if (pebs_err) {
447                 for_each_possible_cpu(cpu)
448                         release_pebs_buffer(cpu);
449         }
450 
451         if (bts_err && pebs_err) {
452                 for_each_possible_cpu(cpu)
453                         release_ds_buffer(cpu);
454         } else {
455                 if (x86_pmu.bts && !bts_err)
456                         x86_pmu.bts_active = 1;
457 
458                 if (x86_pmu.pebs && !pebs_err)
459                         x86_pmu.pebs_active = 1;
460 
461                 for_each_online_cpu(cpu)
462                         init_debug_store_on_cpu(cpu);
463         }
464 
465         put_online_cpus();
466 }
467 
468 /*
469  * BTS
470  */
471 
472 struct event_constraint bts_constraint =
473         EVENT_CONSTRAINT(0, 1ULL << INTEL_PMC_IDX_FIXED_BTS, 0);
474 
475 void intel_pmu_enable_bts(u64 config)
476 {
477         unsigned long debugctlmsr;
478 
479         debugctlmsr = get_debugctlmsr();
480 
481         debugctlmsr |= DEBUGCTLMSR_TR;
482         debugctlmsr |= DEBUGCTLMSR_BTS;
483         if (config & ARCH_PERFMON_EVENTSEL_INT)
484                 debugctlmsr |= DEBUGCTLMSR_BTINT;
485 
486         if (!(config & ARCH_PERFMON_EVENTSEL_OS))
487                 debugctlmsr |= DEBUGCTLMSR_BTS_OFF_OS;
488 
489         if (!(config & ARCH_PERFMON_EVENTSEL_USR))
490                 debugctlmsr |= DEBUGCTLMSR_BTS_OFF_USR;
491 
492         update_debugctlmsr(debugctlmsr);
493 }
494 
495 void intel_pmu_disable_bts(void)
496 {
497         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
498         unsigned long debugctlmsr;
499 
500         if (!cpuc->ds)
501                 return;
502 
503         debugctlmsr = get_debugctlmsr();
504 
505         debugctlmsr &=
506                 ~(DEBUGCTLMSR_TR | DEBUGCTLMSR_BTS | DEBUGCTLMSR_BTINT |
507                   DEBUGCTLMSR_BTS_OFF_OS | DEBUGCTLMSR_BTS_OFF_USR);
508 
509         update_debugctlmsr(debugctlmsr);
510 }
511 
512 int intel_pmu_drain_bts_buffer(void)
513 {
514         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
515         struct debug_store *ds = cpuc->ds;
516         struct bts_record {
517                 u64     from;
518                 u64     to;
519                 u64     flags;
520         };
521         struct perf_event *event = cpuc->events[INTEL_PMC_IDX_FIXED_BTS];
522         struct bts_record *at, *base, *top;
523         struct perf_output_handle handle;
524         struct perf_event_header header;
525         struct perf_sample_data data;
526         unsigned long skip = 0;
527         struct pt_regs regs;
528 
529         if (!event)
530                 return 0;
531 
532         if (!x86_pmu.bts_active)
533                 return 0;
534 
535         base = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
536         top  = (struct bts_record *)(unsigned long)ds->bts_index;
537 
538         if (top <= base)
539                 return 0;
540 
541         memset(&regs, 0, sizeof(regs));
542 
543         ds->bts_index = ds->bts_buffer_base;
544 
545         perf_sample_data_init(&data, 0, event->hw.last_period);
546 
547         /*
548          * BTS leaks kernel addresses in branches across the cpl boundary,
549          * such as traps or system calls, so unless the user is asking for
550          * kernel tracing (and right now it's not possible), we'd need to
551          * filter them out. But first we need to count how many of those we
552          * have in the current batch. This is an extra O(n) pass, however,
553          * it's much faster than the other one especially considering that
554          * n <= 2560 (BTS_BUFFER_SIZE / BTS_RECORD_SIZE * 15/16; see the
555          * alloc_bts_buffer()).
556          */
557         for (at = base; at < top; at++) {
558                 /*
559                  * Note that right now *this* BTS code only works if
560                  * attr::exclude_kernel is set, but let's keep this extra
561                  * check here in case that changes.
562                  */
563                 if (event->attr.exclude_kernel &&
564                     (kernel_ip(at->from) || kernel_ip(at->to)))
565                         skip++;
566         }
567 
568         /*
569          * Prepare a generic sample, i.e. fill in the invariant fields.
570          * We will overwrite the from and to address before we output
571          * the sample.
572          */
573         rcu_read_lock();
574         perf_prepare_sample(&header, &data, event, &regs);
575 
576         if (perf_output_begin(&handle, event, header.size *
577                               (top - base - skip)))
578                 goto unlock;
579 
580         for (at = base; at < top; at++) {
581                 /* Filter out any records that contain kernel addresses. */
582                 if (event->attr.exclude_kernel &&
583                     (kernel_ip(at->from) || kernel_ip(at->to)))
584                         continue;
585 
586                 data.ip         = at->from;
587                 data.addr       = at->to;
588 
589                 perf_output_sample(&handle, &header, &data, event);
590         }
591 
592         perf_output_end(&handle);
593 
594         /* There's new data available. */
595         event->hw.interrupts++;
596         event->pending_kill = POLL_IN;
597 unlock:
598         rcu_read_unlock();
599         return 1;
600 }
601 
602 static inline void intel_pmu_drain_pebs_buffer(void)
603 {
604         struct pt_regs regs;
605 
606         x86_pmu.drain_pebs(&regs);
607 }
608 
609 void intel_pmu_pebs_sched_task(struct perf_event_context *ctx, bool sched_in)
610 {
611         if (!sched_in)
612                 intel_pmu_drain_pebs_buffer();
613 }
614 
615 /*
616  * PEBS
617  */
618 struct event_constraint intel_core2_pebs_event_constraints[] = {
619         INTEL_FLAGS_UEVENT_CONSTRAINT(0x00c0, 0x1), /* INST_RETIRED.ANY */
620         INTEL_FLAGS_UEVENT_CONSTRAINT(0xfec1, 0x1), /* X87_OPS_RETIRED.ANY */
621         INTEL_FLAGS_UEVENT_CONSTRAINT(0x00c5, 0x1), /* BR_INST_RETIRED.MISPRED */
622         INTEL_FLAGS_UEVENT_CONSTRAINT(0x1fc7, 0x1), /* SIMD_INST_RETURED.ANY */
623         INTEL_FLAGS_EVENT_CONSTRAINT(0xcb, 0x1),    /* MEM_LOAD_RETIRED.* */
624         /* INST_RETIRED.ANY_P, inv=1, cmask=16 (cycles:p). */
625         INTEL_FLAGS_EVENT_CONSTRAINT(0x108000c0, 0x01),
626         EVENT_CONSTRAINT_END
627 };
628 
629 struct event_constraint intel_atom_pebs_event_constraints[] = {
630         INTEL_FLAGS_UEVENT_CONSTRAINT(0x00c0, 0x1), /* INST_RETIRED.ANY */
631         INTEL_FLAGS_UEVENT_CONSTRAINT(0x00c5, 0x1), /* MISPREDICTED_BRANCH_RETIRED */
632         INTEL_FLAGS_EVENT_CONSTRAINT(0xcb, 0x1),    /* MEM_LOAD_RETIRED.* */
633         /* INST_RETIRED.ANY_P, inv=1, cmask=16 (cycles:p). */
634         INTEL_FLAGS_EVENT_CONSTRAINT(0x108000c0, 0x01),
635         /* Allow all events as PEBS with no flags */
636         INTEL_ALL_EVENT_CONSTRAINT(0, 0x1),
637         EVENT_CONSTRAINT_END
638 };
639 
640 struct event_constraint intel_slm_pebs_event_constraints[] = {
641         /* INST_RETIRED.ANY_P, inv=1, cmask=16 (cycles:p). */
642         INTEL_FLAGS_EVENT_CONSTRAINT(0x108000c0, 0x1),
643         /* Allow all events as PEBS with no flags */
644         INTEL_ALL_EVENT_CONSTRAINT(0, 0x1),
645         EVENT_CONSTRAINT_END
646 };
647 
648 struct event_constraint intel_glm_pebs_event_constraints[] = {
649         /* Allow all events as PEBS with no flags */
650         INTEL_ALL_EVENT_CONSTRAINT(0, 0x1),
651         EVENT_CONSTRAINT_END
652 };
653 
654 struct event_constraint intel_nehalem_pebs_event_constraints[] = {
655         INTEL_PLD_CONSTRAINT(0x100b, 0xf),      /* MEM_INST_RETIRED.* */
656         INTEL_FLAGS_EVENT_CONSTRAINT(0x0f, 0xf),    /* MEM_UNCORE_RETIRED.* */
657         INTEL_FLAGS_UEVENT_CONSTRAINT(0x010c, 0xf), /* MEM_STORE_RETIRED.DTLB_MISS */
658         INTEL_FLAGS_EVENT_CONSTRAINT(0xc0, 0xf),    /* INST_RETIRED.ANY */
659         INTEL_EVENT_CONSTRAINT(0xc2, 0xf),    /* UOPS_RETIRED.* */
660         INTEL_FLAGS_EVENT_CONSTRAINT(0xc4, 0xf),    /* BR_INST_RETIRED.* */
661         INTEL_FLAGS_UEVENT_CONSTRAINT(0x02c5, 0xf), /* BR_MISP_RETIRED.NEAR_CALL */
662         INTEL_FLAGS_EVENT_CONSTRAINT(0xc7, 0xf),    /* SSEX_UOPS_RETIRED.* */
663         INTEL_FLAGS_UEVENT_CONSTRAINT(0x20c8, 0xf), /* ITLB_MISS_RETIRED */
664         INTEL_FLAGS_EVENT_CONSTRAINT(0xcb, 0xf),    /* MEM_LOAD_RETIRED.* */
665         INTEL_FLAGS_EVENT_CONSTRAINT(0xf7, 0xf),    /* FP_ASSIST.* */
666         /* INST_RETIRED.ANY_P, inv=1, cmask=16 (cycles:p). */
667         INTEL_FLAGS_EVENT_CONSTRAINT(0x108000c0, 0x0f),
668         EVENT_CONSTRAINT_END
669 };
670 
671 struct event_constraint intel_westmere_pebs_event_constraints[] = {
672         INTEL_PLD_CONSTRAINT(0x100b, 0xf),      /* MEM_INST_RETIRED.* */
673         INTEL_FLAGS_EVENT_CONSTRAINT(0x0f, 0xf),    /* MEM_UNCORE_RETIRED.* */
674         INTEL_FLAGS_UEVENT_CONSTRAINT(0x010c, 0xf), /* MEM_STORE_RETIRED.DTLB_MISS */
675         INTEL_FLAGS_EVENT_CONSTRAINT(0xc0, 0xf),    /* INSTR_RETIRED.* */
676         INTEL_EVENT_CONSTRAINT(0xc2, 0xf),    /* UOPS_RETIRED.* */
677         INTEL_FLAGS_EVENT_CONSTRAINT(0xc4, 0xf),    /* BR_INST_RETIRED.* */
678         INTEL_FLAGS_EVENT_CONSTRAINT(0xc5, 0xf),    /* BR_MISP_RETIRED.* */
679         INTEL_FLAGS_EVENT_CONSTRAINT(0xc7, 0xf),    /* SSEX_UOPS_RETIRED.* */
680         INTEL_FLAGS_UEVENT_CONSTRAINT(0x20c8, 0xf), /* ITLB_MISS_RETIRED */
681         INTEL_FLAGS_EVENT_CONSTRAINT(0xcb, 0xf),    /* MEM_LOAD_RETIRED.* */
682         INTEL_FLAGS_EVENT_CONSTRAINT(0xf7, 0xf),    /* FP_ASSIST.* */
683         /* INST_RETIRED.ANY_P, inv=1, cmask=16 (cycles:p). */
684         INTEL_FLAGS_EVENT_CONSTRAINT(0x108000c0, 0x0f),
685         EVENT_CONSTRAINT_END
686 };
687 
688 struct event_constraint intel_snb_pebs_event_constraints[] = {
689         INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PRECDIST */
690         INTEL_PLD_CONSTRAINT(0x01cd, 0x8),    /* MEM_TRANS_RETIRED.LAT_ABOVE_THR */
691         INTEL_PST_CONSTRAINT(0x02cd, 0x8),    /* MEM_TRANS_RETIRED.PRECISE_STORES */
692         /* UOPS_RETIRED.ALL, inv=1, cmask=16 (cycles:p). */
693         INTEL_FLAGS_EVENT_CONSTRAINT(0x108001c2, 0xf),
694         INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf),    /* MEM_UOP_RETIRED.* */
695         INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf),    /* MEM_LOAD_UOPS_RETIRED.* */
696         INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf),    /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
697         INTEL_EXCLEVT_CONSTRAINT(0xd3, 0xf),    /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */
698         /* Allow all events as PEBS with no flags */
699         INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
700         EVENT_CONSTRAINT_END
701 };
702 
703 struct event_constraint intel_ivb_pebs_event_constraints[] = {
704         INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PRECDIST */
705         INTEL_PLD_CONSTRAINT(0x01cd, 0x8),    /* MEM_TRANS_RETIRED.LAT_ABOVE_THR */
706         INTEL_PST_CONSTRAINT(0x02cd, 0x8),    /* MEM_TRANS_RETIRED.PRECISE_STORES */
707         /* UOPS_RETIRED.ALL, inv=1, cmask=16 (cycles:p). */
708         INTEL_FLAGS_EVENT_CONSTRAINT(0x108001c2, 0xf),
709         /* INST_RETIRED.PREC_DIST, inv=1, cmask=16 (cycles:ppp). */
710         INTEL_FLAGS_EVENT_CONSTRAINT(0x108001c0, 0x2),
711         INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf),    /* MEM_UOP_RETIRED.* */
712         INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf),    /* MEM_LOAD_UOPS_RETIRED.* */
713         INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf),    /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
714         INTEL_EXCLEVT_CONSTRAINT(0xd3, 0xf),    /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */
715         /* Allow all events as PEBS with no flags */
716         INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
717         EVENT_CONSTRAINT_END
718 };
719 
720 struct event_constraint intel_hsw_pebs_event_constraints[] = {
721         INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PRECDIST */
722         INTEL_PLD_CONSTRAINT(0x01cd, 0xf),    /* MEM_TRANS_RETIRED.* */
723         /* UOPS_RETIRED.ALL, inv=1, cmask=16 (cycles:p). */
724         INTEL_FLAGS_EVENT_CONSTRAINT(0x108001c2, 0xf),
725         /* INST_RETIRED.PREC_DIST, inv=1, cmask=16 (cycles:ppp). */
726         INTEL_FLAGS_EVENT_CONSTRAINT(0x108001c0, 0x2),
727         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(0x01c2, 0xf), /* UOPS_RETIRED.ALL */
728         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(0x11d0, 0xf), /* MEM_UOPS_RETIRED.STLB_MISS_LOADS */
729         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(0x21d0, 0xf), /* MEM_UOPS_RETIRED.LOCK_LOADS */
730         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(0x41d0, 0xf), /* MEM_UOPS_RETIRED.SPLIT_LOADS */
731         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(0x81d0, 0xf), /* MEM_UOPS_RETIRED.ALL_LOADS */
732         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(0x12d0, 0xf), /* MEM_UOPS_RETIRED.STLB_MISS_STORES */
733         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(0x42d0, 0xf), /* MEM_UOPS_RETIRED.SPLIT_STORES */
734         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(0x82d0, 0xf), /* MEM_UOPS_RETIRED.ALL_STORES */
735         INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(0xd1, 0xf),    /* MEM_LOAD_UOPS_RETIRED.* */
736         INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(0xd2, 0xf),    /* MEM_LOAD_UOPS_L3_HIT_RETIRED.* */
737         INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(0xd3, 0xf),    /* MEM_LOAD_UOPS_L3_MISS_RETIRED.* */
738         /* Allow all events as PEBS with no flags */
739         INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
740         EVENT_CONSTRAINT_END
741 };
742 
743 struct event_constraint intel_bdw_pebs_event_constraints[] = {
744         INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PRECDIST */
745         INTEL_PLD_CONSTRAINT(0x01cd, 0xf),    /* MEM_TRANS_RETIRED.* */
746         /* UOPS_RETIRED.ALL, inv=1, cmask=16 (cycles:p). */
747         INTEL_FLAGS_EVENT_CONSTRAINT(0x108001c2, 0xf),
748         /* INST_RETIRED.PREC_DIST, inv=1, cmask=16 (cycles:ppp). */
749         INTEL_FLAGS_EVENT_CONSTRAINT(0x108001c0, 0x2),
750         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(0x01c2, 0xf), /* UOPS_RETIRED.ALL */
751         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_UOPS_RETIRED.STLB_MISS_LOADS */
752         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf), /* MEM_UOPS_RETIRED.LOCK_LOADS */
753         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf), /* MEM_UOPS_RETIRED.SPLIT_LOADS */
754         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf), /* MEM_UOPS_RETIRED.ALL_LOADS */
755         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf), /* MEM_UOPS_RETIRED.STLB_MISS_STORES */
756         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf), /* MEM_UOPS_RETIRED.SPLIT_STORES */
757         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf), /* MEM_UOPS_RETIRED.ALL_STORES */
758         INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd1, 0xf),    /* MEM_LOAD_UOPS_RETIRED.* */
759         INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd2, 0xf),    /* MEM_LOAD_UOPS_L3_HIT_RETIRED.* */
760         INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd3, 0xf),    /* MEM_LOAD_UOPS_L3_MISS_RETIRED.* */
761         /* Allow all events as PEBS with no flags */
762         INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
763         EVENT_CONSTRAINT_END
764 };
765 
766 
767 struct event_constraint intel_skl_pebs_event_constraints[] = {
768         INTEL_FLAGS_UEVENT_CONSTRAINT(0x1c0, 0x2),      /* INST_RETIRED.PREC_DIST */
769         /* INST_RETIRED.PREC_DIST, inv=1, cmask=16 (cycles:ppp). */
770         INTEL_FLAGS_EVENT_CONSTRAINT(0x108001c0, 0x2),
771         /* INST_RETIRED.TOTAL_CYCLES_PS (inv=1, cmask=16) (cycles:p). */
772         INTEL_FLAGS_EVENT_CONSTRAINT(0x108000c0, 0x0f),
773         INTEL_PLD_CONSTRAINT(0x1cd, 0xf),                     /* MEM_TRANS_RETIRED.* */
774         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */
775         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_STORES */
776         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf), /* MEM_INST_RETIRED.LOCK_LOADS */
777         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x22d0, 0xf), /* MEM_INST_RETIRED.LOCK_STORES */
778         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf), /* MEM_INST_RETIRED.SPLIT_LOADS */
779         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf), /* MEM_INST_RETIRED.SPLIT_STORES */
780         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf), /* MEM_INST_RETIRED.ALL_LOADS */
781         INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf), /* MEM_INST_RETIRED.ALL_STORES */
782         INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd1, 0xf),    /* MEM_LOAD_RETIRED.* */
783         INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd2, 0xf),    /* MEM_LOAD_L3_HIT_RETIRED.* */
784         INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd3, 0xf),    /* MEM_LOAD_L3_MISS_RETIRED.* */
785         /* Allow all events as PEBS with no flags */
786         INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
787         EVENT_CONSTRAINT_END
788 };
789 
790 struct event_constraint *intel_pebs_constraints(struct perf_event *event)
791 {
792         struct event_constraint *c;
793 
794         if (!event->attr.precise_ip)
795                 return NULL;
796 
797         if (x86_pmu.pebs_constraints) {
798                 for_each_event_constraint(c, x86_pmu.pebs_constraints) {
799                         if ((event->hw.config & c->cmask) == c->code) {
800                                 event->hw.flags |= c->flags;
801                                 return c;
802                         }
803                 }
804         }
805 
806         return &emptyconstraint;
807 }
808 
809 static inline bool pebs_is_enabled(struct cpu_hw_events *cpuc)
810 {
811         return (cpuc->pebs_enabled & ((1ULL << MAX_PEBS_EVENTS) - 1));
812 }
813 
814 void intel_pmu_pebs_enable(struct perf_event *event)
815 {
816         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
817         struct hw_perf_event *hwc = &event->hw;
818         struct debug_store *ds = cpuc->ds;
819         bool first_pebs;
820         u64 threshold;
821 
822         hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
823 
824         first_pebs = !pebs_is_enabled(cpuc);
825         cpuc->pebs_enabled |= 1ULL << hwc->idx;
826 
827         if (event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT)
828                 cpuc->pebs_enabled |= 1ULL << (hwc->idx + 32);
829         else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST)
830                 cpuc->pebs_enabled |= 1ULL << 63;
831 
832         /*
833          * When the event is constrained enough we can use a larger
834          * threshold and run the event with less frequent PMI.
835          */
836         if (hwc->flags & PERF_X86_EVENT_FREERUNNING) {
837                 threshold = ds->pebs_absolute_maximum -
838                         x86_pmu.max_pebs_events * x86_pmu.pebs_record_size;
839 
840                 if (first_pebs)
841                         perf_sched_cb_inc(event->ctx->pmu);
842         } else {
843                 threshold = ds->pebs_buffer_base + x86_pmu.pebs_record_size;
844 
845                 /*
846                  * If not all events can use larger buffer,
847                  * roll back to threshold = 1
848                  */
849                 if (!first_pebs &&
850                     (ds->pebs_interrupt_threshold > threshold))
851                         perf_sched_cb_dec(event->ctx->pmu);
852         }
853 
854         /* Use auto-reload if possible to save a MSR write in the PMI */
855         if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {
856                 ds->pebs_event_reset[hwc->idx] =
857                         (u64)(-hwc->sample_period) & x86_pmu.cntval_mask;
858         }
859 
860         if (first_pebs || ds->pebs_interrupt_threshold > threshold)
861                 ds->pebs_interrupt_threshold = threshold;
862 }
863 
864 void intel_pmu_pebs_disable(struct perf_event *event)
865 {
866         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
867         struct hw_perf_event *hwc = &event->hw;
868         struct debug_store *ds = cpuc->ds;
869         bool large_pebs = ds->pebs_interrupt_threshold >
870                 ds->pebs_buffer_base + x86_pmu.pebs_record_size;
871 
872         if (large_pebs)
873                 intel_pmu_drain_pebs_buffer();
874 
875         cpuc->pebs_enabled &= ~(1ULL << hwc->idx);
876 
877         if (event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT)
878                 cpuc->pebs_enabled &= ~(1ULL << (hwc->idx + 32));
879         else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST)
880                 cpuc->pebs_enabled &= ~(1ULL << 63);
881 
882         if (large_pebs && !pebs_is_enabled(cpuc))
883                 perf_sched_cb_dec(event->ctx->pmu);
884 
885         if (cpuc->enabled)
886                 wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
887 
888         hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
889 }
890 
891 void intel_pmu_pebs_enable_all(void)
892 {
893         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
894 
895         if (cpuc->pebs_enabled)
896                 wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
897 }
898 
899 void intel_pmu_pebs_disable_all(void)
900 {
901         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
902 
903         if (cpuc->pebs_enabled)
904                 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
905 }
906 
907 static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
908 {
909         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
910         unsigned long from = cpuc->lbr_entries[0].from;
911         unsigned long old_to, to = cpuc->lbr_entries[0].to;
912         unsigned long ip = regs->ip;
913         int is_64bit = 0;
914         void *kaddr;
915         int size;
916 
917         /*
918          * We don't need to fixup if the PEBS assist is fault like
919          */
920         if (!x86_pmu.intel_cap.pebs_trap)
921                 return 1;
922 
923         /*
924          * No LBR entry, no basic block, no rewinding
925          */
926         if (!cpuc->lbr_stack.nr || !from || !to)
927                 return 0;
928 
929         /*
930          * Basic blocks should never cross user/kernel boundaries
931          */
932         if (kernel_ip(ip) != kernel_ip(to))
933                 return 0;
934 
935         /*
936          * unsigned math, either ip is before the start (impossible) or
937          * the basic block is larger than 1 page (sanity)
938          */
939         if ((ip - to) > PEBS_FIXUP_SIZE)
940                 return 0;
941 
942         /*
943          * We sampled a branch insn, rewind using the LBR stack
944          */
945         if (ip == to) {
946                 set_linear_ip(regs, from);
947                 return 1;
948         }
949 
950         size = ip - to;
951         if (!kernel_ip(ip)) {
952                 int bytes;
953                 u8 *buf = this_cpu_read(insn_buffer);
954 
955                 /* 'size' must fit our buffer, see above */
956                 bytes = copy_from_user_nmi(buf, (void __user *)to, size);
957                 if (bytes != 0)
958                         return 0;
959 
960                 kaddr = buf;
961         } else {
962                 kaddr = (void *)to;
963         }
964 
965         do {
966                 struct insn insn;
967 
968                 old_to = to;
969 
970 #ifdef CONFIG_X86_64
971                 is_64bit = kernel_ip(to) || !test_thread_flag(TIF_IA32);
972 #endif
973                 insn_init(&insn, kaddr, size, is_64bit);
974                 insn_get_length(&insn);
975                 /*
976                  * Make sure there was not a problem decoding the
977                  * instruction and getting the length.  This is
978                  * doubly important because we have an infinite
979                  * loop if insn.length=0.
980                  */
981                 if (!insn.length)
982                         break;
983 
984                 to += insn.length;
985                 kaddr += insn.length;
986                 size -= insn.length;
987         } while (to < ip);
988 
989         if (to == ip) {
990                 set_linear_ip(regs, old_to);
991                 return 1;
992         }
993 
994         /*
995          * Even though we decoded the basic block, the instruction stream
996          * never matched the given IP, either the TO or the IP got corrupted.
997          */
998         return 0;
999 }
1000 
1001 static inline u64 intel_hsw_weight(struct pebs_record_skl *pebs)
1002 {
1003         if (pebs->tsx_tuning) {
1004                 union hsw_tsx_tuning tsx = { .value = pebs->tsx_tuning };
1005                 return tsx.cycles_last_block;
1006         }
1007         return 0;
1008 }
1009 
1010 static inline u64 intel_hsw_transaction(struct pebs_record_skl *pebs)
1011 {
1012         u64 txn = (pebs->tsx_tuning & PEBS_HSW_TSX_FLAGS) >> 32;
1013 
1014         /* For RTM XABORTs also log the abort code from AX */
1015         if ((txn & PERF_TXN_TRANSACTION) && (pebs->ax & 1))
1016                 txn |= ((pebs->ax >> 24) & 0xff) << PERF_TXN_ABORT_SHIFT;
1017         return txn;
1018 }
1019 
1020 static void setup_pebs_sample_data(struct perf_event *event,
1021                                    struct pt_regs *iregs, void *__pebs,
1022                                    struct perf_sample_data *data,
1023                                    struct pt_regs *regs)
1024 {
1025 #define PERF_X86_EVENT_PEBS_HSW_PREC \
1026                 (PERF_X86_EVENT_PEBS_ST_HSW | \
1027                  PERF_X86_EVENT_PEBS_LD_HSW | \
1028                  PERF_X86_EVENT_PEBS_NA_HSW)
1029         /*
1030          * We cast to the biggest pebs_record but are careful not to
1031          * unconditionally access the 'extra' entries.
1032          */
1033         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1034         struct pebs_record_skl *pebs = __pebs;
1035         u64 sample_type;
1036         int fll, fst, dsrc;
1037         int fl = event->hw.flags;
1038 
1039         if (pebs == NULL)
1040                 return;
1041 
1042         sample_type = event->attr.sample_type;
1043         dsrc = sample_type & PERF_SAMPLE_DATA_SRC;
1044 
1045         fll = fl & PERF_X86_EVENT_PEBS_LDLAT;
1046         fst = fl & (PERF_X86_EVENT_PEBS_ST | PERF_X86_EVENT_PEBS_HSW_PREC);
1047 
1048         perf_sample_data_init(data, 0, event->hw.last_period);
1049 
1050         data->period = event->hw.last_period;
1051 
1052         /*
1053          * Use latency for weight (only avail with PEBS-LL)
1054          */
1055         if (fll && (sample_type & PERF_SAMPLE_WEIGHT))
1056                 data->weight = pebs->lat;
1057 
1058         /*
1059          * data.data_src encodes the data source
1060          */
1061         if (dsrc) {
1062                 u64 val = PERF_MEM_NA;
1063                 if (fll)
1064                         val = load_latency_data(pebs->dse);
1065                 else if (fst && (fl & PERF_X86_EVENT_PEBS_HSW_PREC))
1066                         val = precise_datala_hsw(event, pebs->dse);
1067                 else if (fst)
1068                         val = precise_store_data(pebs->dse);
1069                 data->data_src.val = val;
1070         }
1071 
1072         /*
1073          * We use the interrupt regs as a base because the PEBS record does not
1074          * contain a full regs set, specifically it seems to lack segment
1075          * descriptors, which get used by things like user_mode().
1076          *
1077          * In the simple case fix up only the IP for PERF_SAMPLE_IP.
1078          *
1079          * We must however always use BP,SP from iregs for the unwinder to stay
1080          * sane; the record BP,SP can point into thin air when the record is
1081          * from a previous PMI context or an (I)RET happend between the record
1082          * and PMI.
1083          */
1084         *regs = *iregs;
1085         regs->flags = pebs->flags;
1086         set_linear_ip(regs, pebs->ip);
1087 
1088         if (sample_type & PERF_SAMPLE_REGS_INTR) {
1089                 regs->ax = pebs->ax;
1090                 regs->bx = pebs->bx;
1091                 regs->cx = pebs->cx;
1092                 regs->dx = pebs->dx;
1093                 regs->si = pebs->si;
1094                 regs->di = pebs->di;
1095 
1096                 /*
1097                  * Per the above; only set BP,SP if we don't need callchains.
1098                  *
1099                  * XXX: does this make sense?
1100                  */
1101                 if (!(sample_type & PERF_SAMPLE_CALLCHAIN)) {
1102                         regs->bp = pebs->bp;
1103                         regs->sp = pebs->sp;
1104                 }
1105 
1106                 /*
1107                  * Preserve PERF_EFLAGS_VM from set_linear_ip().
1108                  */
1109                 regs->flags = pebs->flags | (regs->flags & PERF_EFLAGS_VM);
1110 #ifndef CONFIG_X86_32
1111                 regs->r8 = pebs->r8;
1112                 regs->r9 = pebs->r9;
1113                 regs->r10 = pebs->r10;
1114                 regs->r11 = pebs->r11;
1115                 regs->r12 = pebs->r12;
1116                 regs->r13 = pebs->r13;
1117                 regs->r14 = pebs->r14;
1118                 regs->r15 = pebs->r15;
1119 #endif
1120         }
1121 
1122         if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format >= 2) {
1123                 regs->ip = pebs->real_ip;
1124                 regs->flags |= PERF_EFLAGS_EXACT;
1125         } else if (event->attr.precise_ip > 1 && intel_pmu_pebs_fixup_ip(regs))
1126                 regs->flags |= PERF_EFLAGS_EXACT;
1127         else
1128                 regs->flags &= ~PERF_EFLAGS_EXACT;
1129 
1130         if ((sample_type & PERF_SAMPLE_ADDR) &&
1131             x86_pmu.intel_cap.pebs_format >= 1)
1132                 data->addr = pebs->dla;
1133 
1134         if (x86_pmu.intel_cap.pebs_format >= 2) {
1135                 /* Only set the TSX weight when no memory weight. */
1136                 if ((sample_type & PERF_SAMPLE_WEIGHT) && !fll)
1137                         data->weight = intel_hsw_weight(pebs);
1138 
1139                 if (sample_type & PERF_SAMPLE_TRANSACTION)
1140                         data->txn = intel_hsw_transaction(pebs);
1141         }
1142 
1143         /*
1144          * v3 supplies an accurate time stamp, so we use that
1145          * for the time stamp.
1146          *
1147          * We can only do this for the default trace clock.
1148          */
1149         if (x86_pmu.intel_cap.pebs_format >= 3 &&
1150                 event->attr.use_clockid == 0)
1151                 data->time = native_sched_clock_from_tsc(pebs->tsc);
1152 
1153         if (has_branch_stack(event))
1154                 data->br_stack = &cpuc->lbr_stack;
1155 }
1156 
1157 static inline void *
1158 get_next_pebs_record_by_bit(void *base, void *top, int bit)
1159 {
1160         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1161         void *at;
1162         u64 pebs_status;
1163 
1164         /*
1165          * fmt0 does not have a status bitfield (does not use
1166          * perf_record_nhm format)
1167          */
1168         if (x86_pmu.intel_cap.pebs_format < 1)
1169                 return base;
1170 
1171         if (base == NULL)
1172                 return NULL;
1173 
1174         for (at = base; at < top; at += x86_pmu.pebs_record_size) {
1175                 struct pebs_record_nhm *p = at;
1176 
1177                 if (test_bit(bit, (unsigned long *)&p->status)) {
1178                         /* PEBS v3 has accurate status bits */
1179                         if (x86_pmu.intel_cap.pebs_format >= 3)
1180                                 return at;
1181 
1182                         if (p->status == (1 << bit))
1183                                 return at;
1184 
1185                         /* clear non-PEBS bit and re-check */
1186                         pebs_status = p->status & cpuc->pebs_enabled;
1187                         pebs_status &= (1ULL << MAX_PEBS_EVENTS) - 1;
1188                         if (pebs_status == (1 << bit))
1189                                 return at;
1190                 }
1191         }
1192         return NULL;
1193 }
1194 
1195 static void __intel_pmu_pebs_event(struct perf_event *event,
1196                                    struct pt_regs *iregs,
1197                                    void *base, void *top,
1198                                    int bit, int count)
1199 {
1200         struct perf_sample_data data;
1201         struct pt_regs regs;
1202         void *at = get_next_pebs_record_by_bit(base, top, bit);
1203 
1204         if (!intel_pmu_save_and_restart(event) &&
1205             !(event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD))
1206                 return;
1207 
1208         while (count > 1) {
1209                 setup_pebs_sample_data(event, iregs, at, &data, &regs);
1210                 perf_event_output(event, &data, &regs);
1211                 at += x86_pmu.pebs_record_size;
1212                 at = get_next_pebs_record_by_bit(at, top, bit);
1213                 count--;
1214         }
1215 
1216         setup_pebs_sample_data(event, iregs, at, &data, &regs);
1217 
1218         /*
1219          * All but the last records are processed.
1220          * The last one is left to be able to call the overflow handler.
1221          */
1222         if (perf_event_overflow(event, &data, &regs)) {
1223                 x86_pmu_stop(event, 0);
1224                 return;
1225         }
1226 
1227 }
1228 
1229 static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)
1230 {
1231         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1232         struct debug_store *ds = cpuc->ds;
1233         struct perf_event *event = cpuc->events[0]; /* PMC0 only */
1234         struct pebs_record_core *at, *top;
1235         int n;
1236 
1237         if (!x86_pmu.pebs_active)
1238                 return;
1239 
1240         at  = (struct pebs_record_core *)(unsigned long)ds->pebs_buffer_base;
1241         top = (struct pebs_record_core *)(unsigned long)ds->pebs_index;
1242 
1243         /*
1244          * Whatever else happens, drain the thing
1245          */
1246         ds->pebs_index = ds->pebs_buffer_base;
1247 
1248         if (!test_bit(0, cpuc->active_mask))
1249                 return;
1250 
1251         WARN_ON_ONCE(!event);
1252 
1253         if (!event->attr.precise_ip)
1254                 return;
1255 
1256         n = top - at;
1257         if (n <= 0)
1258                 return;
1259 
1260         __intel_pmu_pebs_event(event, iregs, at, top, 0, n);
1261 }
1262 
1263 static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs)
1264 {
1265         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1266         struct debug_store *ds = cpuc->ds;
1267         struct perf_event *event;
1268         void *base, *at, *top;
1269         short counts[MAX_PEBS_EVENTS] = {};
1270         short error[MAX_PEBS_EVENTS] = {};
1271         int bit, i;
1272 
1273         if (!x86_pmu.pebs_active)
1274                 return;
1275 
1276         base = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base;
1277         top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index;
1278 
1279         ds->pebs_index = ds->pebs_buffer_base;
1280 
1281         if (unlikely(base >= top))
1282                 return;
1283 
1284         for (at = base; at < top; at += x86_pmu.pebs_record_size) {
1285                 struct pebs_record_nhm *p = at;
1286                 u64 pebs_status;
1287 
1288                 pebs_status = p->status & cpuc->pebs_enabled;
1289                 pebs_status &= (1ULL << x86_pmu.max_pebs_events) - 1;
1290 
1291                 /* PEBS v3 has more accurate status bits */
1292                 if (x86_pmu.intel_cap.pebs_format >= 3) {
1293                         for_each_set_bit(bit, (unsigned long *)&pebs_status,
1294                                          x86_pmu.max_pebs_events)
1295                                 counts[bit]++;
1296 
1297                         continue;
1298                 }
1299 
1300                 /*
1301                  * On some CPUs the PEBS status can be zero when PEBS is
1302                  * racing with clearing of GLOBAL_STATUS.
1303                  *
1304                  * Normally we would drop that record, but in the
1305                  * case when there is only a single active PEBS event
1306                  * we can assume it's for that event.
1307                  */
1308                 if (!pebs_status && cpuc->pebs_enabled &&
1309                         !(cpuc->pebs_enabled & (cpuc->pebs_enabled-1)))
1310                         pebs_status = cpuc->pebs_enabled;
1311 
1312                 bit = find_first_bit((unsigned long *)&pebs_status,
1313                                         x86_pmu.max_pebs_events);
1314                 if (bit >= x86_pmu.max_pebs_events)
1315                         continue;
1316 
1317                 /*
1318                  * The PEBS hardware does not deal well with the situation
1319                  * when events happen near to each other and multiple bits
1320                  * are set. But it should happen rarely.
1321                  *
1322                  * If these events include one PEBS and multiple non-PEBS
1323                  * events, it doesn't impact PEBS record. The record will
1324                  * be handled normally. (slow path)
1325                  *
1326                  * If these events include two or more PEBS events, the
1327                  * records for the events can be collapsed into a single
1328                  * one, and it's not possible to reconstruct all events
1329                  * that caused the PEBS record. It's called collision.
1330                  * If collision happened, the record will be dropped.
1331                  */
1332                 if (p->status != (1ULL << bit)) {
1333                         for_each_set_bit(i, (unsigned long *)&pebs_status,
1334                                          x86_pmu.max_pebs_events)
1335                                 error[i]++;
1336                         continue;
1337                 }
1338 
1339                 counts[bit]++;
1340         }
1341 
1342         for (bit = 0; bit < x86_pmu.max_pebs_events; bit++) {
1343                 if ((counts[bit] == 0) && (error[bit] == 0))
1344                         continue;
1345 
1346                 event = cpuc->events[bit];
1347                 if (WARN_ON_ONCE(!event))
1348                         continue;
1349 
1350                 if (WARN_ON_ONCE(!event->attr.precise_ip))
1351                         continue;
1352 
1353                 /* log dropped samples number */
1354                 if (error[bit])
1355                         perf_log_lost_samples(event, error[bit]);
1356 
1357                 if (counts[bit]) {
1358                         __intel_pmu_pebs_event(event, iregs, base,
1359                                                top, bit, counts[bit]);
1360                 }
1361         }
1362 }
1363 
1364 /*
1365  * BTS, PEBS probe and setup
1366  */
1367 
1368 void __init intel_ds_init(void)
1369 {
1370         /*
1371          * No support for 32bit formats
1372          */
1373         if (!boot_cpu_has(X86_FEATURE_DTES64))
1374                 return;
1375 
1376         x86_pmu.bts  = boot_cpu_has(X86_FEATURE_BTS);
1377         x86_pmu.pebs = boot_cpu_has(X86_FEATURE_PEBS);
1378         x86_pmu.pebs_buffer_size = PEBS_BUFFER_SIZE;
1379         if (x86_pmu.pebs) {
1380                 char pebs_type = x86_pmu.intel_cap.pebs_trap ?  '+' : '-';
1381                 int format = x86_pmu.intel_cap.pebs_format;
1382 
1383                 switch (format) {
1384                 case 0:
1385                         pr_cont("PEBS fmt0%c, ", pebs_type);
1386                         x86_pmu.pebs_record_size = sizeof(struct pebs_record_core);
1387                         /*
1388                          * Using >PAGE_SIZE buffers makes the WRMSR to
1389                          * PERF_GLOBAL_CTRL in intel_pmu_enable_all()
1390                          * mysteriously hang on Core2.
1391                          *
1392                          * As a workaround, we don't do this.
1393                          */
1394                         x86_pmu.pebs_buffer_size = PAGE_SIZE;
1395                         x86_pmu.drain_pebs = intel_pmu_drain_pebs_core;
1396                         break;
1397 
1398                 case 1:
1399                         pr_cont("PEBS fmt1%c, ", pebs_type);
1400                         x86_pmu.pebs_record_size = sizeof(struct pebs_record_nhm);
1401                         x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
1402                         break;
1403 
1404                 case 2:
1405                         pr_cont("PEBS fmt2%c, ", pebs_type);
1406                         x86_pmu.pebs_record_size = sizeof(struct pebs_record_hsw);
1407                         x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
1408                         break;
1409 
1410                 case 3:
1411                         pr_cont("PEBS fmt3%c, ", pebs_type);
1412                         x86_pmu.pebs_record_size =
1413                                                 sizeof(struct pebs_record_skl);
1414                         x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
1415                         x86_pmu.free_running_flags |= PERF_SAMPLE_TIME;
1416                         break;
1417 
1418                 default:
1419                         pr_cont("no PEBS fmt%d%c, ", format, pebs_type);
1420                         x86_pmu.pebs = 0;
1421                 }
1422         }
1423 }
1424 
1425 void perf_restore_debug_store(void)
1426 {
1427         struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);
1428 
1429         if (!x86_pmu.bts && !x86_pmu.pebs)
1430                 return;
1431 
1432         wrmsrl(MSR_IA32_DS_AREA, (unsigned long)ds);
1433 }
1434 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp