~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/events/intel/lbr.c

Version: ~ [ linux-5.1-rc1 ] ~ [ linux-5.0.2 ] ~ [ linux-4.20.16 ] ~ [ linux-4.19.29 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.106 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.163 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.176 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.136 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.63 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 #include <linux/perf_event.h>
  3 #include <linux/types.h>
  4 
  5 #include <asm/perf_event.h>
  6 #include <asm/msr.h>
  7 #include <asm/insn.h>
  8 
  9 #include "../perf_event.h"
 10 
 11 enum {
 12         LBR_FORMAT_32           = 0x00,
 13         LBR_FORMAT_LIP          = 0x01,
 14         LBR_FORMAT_EIP          = 0x02,
 15         LBR_FORMAT_EIP_FLAGS    = 0x03,
 16         LBR_FORMAT_EIP_FLAGS2   = 0x04,
 17         LBR_FORMAT_INFO         = 0x05,
 18         LBR_FORMAT_TIME         = 0x06,
 19         LBR_FORMAT_MAX_KNOWN    = LBR_FORMAT_TIME,
 20 };
 21 
 22 static const enum {
 23         LBR_EIP_FLAGS           = 1,
 24         LBR_TSX                 = 2,
 25 } lbr_desc[LBR_FORMAT_MAX_KNOWN + 1] = {
 26         [LBR_FORMAT_EIP_FLAGS]  = LBR_EIP_FLAGS,
 27         [LBR_FORMAT_EIP_FLAGS2] = LBR_EIP_FLAGS | LBR_TSX,
 28 };
 29 
 30 /*
 31  * Intel LBR_SELECT bits
 32  * Intel Vol3a, April 2011, Section 16.7 Table 16-10
 33  *
 34  * Hardware branch filter (not available on all CPUs)
 35  */
 36 #define LBR_KERNEL_BIT          0 /* do not capture at ring0 */
 37 #define LBR_USER_BIT            1 /* do not capture at ring > 0 */
 38 #define LBR_JCC_BIT             2 /* do not capture conditional branches */
 39 #define LBR_REL_CALL_BIT        3 /* do not capture relative calls */
 40 #define LBR_IND_CALL_BIT        4 /* do not capture indirect calls */
 41 #define LBR_RETURN_BIT          5 /* do not capture near returns */
 42 #define LBR_IND_JMP_BIT         6 /* do not capture indirect jumps */
 43 #define LBR_REL_JMP_BIT         7 /* do not capture relative jumps */
 44 #define LBR_FAR_BIT             8 /* do not capture far branches */
 45 #define LBR_CALL_STACK_BIT      9 /* enable call stack */
 46 
 47 /*
 48  * Following bit only exists in Linux; we mask it out before writing it to
 49  * the actual MSR. But it helps the constraint perf code to understand
 50  * that this is a separate configuration.
 51  */
 52 #define LBR_NO_INFO_BIT        63 /* don't read LBR_INFO. */
 53 
 54 #define LBR_KERNEL      (1 << LBR_KERNEL_BIT)
 55 #define LBR_USER        (1 << LBR_USER_BIT)
 56 #define LBR_JCC         (1 << LBR_JCC_BIT)
 57 #define LBR_REL_CALL    (1 << LBR_REL_CALL_BIT)
 58 #define LBR_IND_CALL    (1 << LBR_IND_CALL_BIT)
 59 #define LBR_RETURN      (1 << LBR_RETURN_BIT)
 60 #define LBR_REL_JMP     (1 << LBR_REL_JMP_BIT)
 61 #define LBR_IND_JMP     (1 << LBR_IND_JMP_BIT)
 62 #define LBR_FAR         (1 << LBR_FAR_BIT)
 63 #define LBR_CALL_STACK  (1 << LBR_CALL_STACK_BIT)
 64 #define LBR_NO_INFO     (1ULL << LBR_NO_INFO_BIT)
 65 
 66 #define LBR_PLM (LBR_KERNEL | LBR_USER)
 67 
 68 #define LBR_SEL_MASK    0x3ff   /* valid bits in LBR_SELECT */
 69 #define LBR_NOT_SUPP    -1      /* LBR filter not supported */
 70 #define LBR_IGN         0       /* ignored */
 71 
 72 #define LBR_ANY          \
 73         (LBR_JCC        |\
 74          LBR_REL_CALL   |\
 75          LBR_IND_CALL   |\
 76          LBR_RETURN     |\
 77          LBR_REL_JMP    |\
 78          LBR_IND_JMP    |\
 79          LBR_FAR)
 80 
 81 #define LBR_FROM_FLAG_MISPRED   BIT_ULL(63)
 82 #define LBR_FROM_FLAG_IN_TX     BIT_ULL(62)
 83 #define LBR_FROM_FLAG_ABORT     BIT_ULL(61)
 84 
 85 #define LBR_FROM_SIGNEXT_2MSB   (BIT_ULL(60) | BIT_ULL(59))
 86 
 87 /*
 88  * x86control flow change classification
 89  * x86control flow changes include branches, interrupts, traps, faults
 90  */
 91 enum {
 92         X86_BR_NONE             = 0,      /* unknown */
 93 
 94         X86_BR_USER             = 1 << 0, /* branch target is user */
 95         X86_BR_KERNEL           = 1 << 1, /* branch target is kernel */
 96 
 97         X86_BR_CALL             = 1 << 2, /* call */
 98         X86_BR_RET              = 1 << 3, /* return */
 99         X86_BR_SYSCALL          = 1 << 4, /* syscall */
100         X86_BR_SYSRET           = 1 << 5, /* syscall return */
101         X86_BR_INT              = 1 << 6, /* sw interrupt */
102         X86_BR_IRET             = 1 << 7, /* return from interrupt */
103         X86_BR_JCC              = 1 << 8, /* conditional */
104         X86_BR_JMP              = 1 << 9, /* jump */
105         X86_BR_IRQ              = 1 << 10,/* hw interrupt or trap or fault */
106         X86_BR_IND_CALL         = 1 << 11,/* indirect calls */
107         X86_BR_ABORT            = 1 << 12,/* transaction abort */
108         X86_BR_IN_TX            = 1 << 13,/* in transaction */
109         X86_BR_NO_TX            = 1 << 14,/* not in transaction */
110         X86_BR_ZERO_CALL        = 1 << 15,/* zero length call */
111         X86_BR_CALL_STACK       = 1 << 16,/* call stack */
112         X86_BR_IND_JMP          = 1 << 17,/* indirect jump */
113 
114         X86_BR_TYPE_SAVE        = 1 << 18,/* indicate to save branch type */
115 
116 };
117 
118 #define X86_BR_PLM (X86_BR_USER | X86_BR_KERNEL)
119 #define X86_BR_ANYTX (X86_BR_NO_TX | X86_BR_IN_TX)
120 
121 #define X86_BR_ANY       \
122         (X86_BR_CALL    |\
123          X86_BR_RET     |\
124          X86_BR_SYSCALL |\
125          X86_BR_SYSRET  |\
126          X86_BR_INT     |\
127          X86_BR_IRET    |\
128          X86_BR_JCC     |\
129          X86_BR_JMP      |\
130          X86_BR_IRQ      |\
131          X86_BR_ABORT    |\
132          X86_BR_IND_CALL |\
133          X86_BR_IND_JMP  |\
134          X86_BR_ZERO_CALL)
135 
136 #define X86_BR_ALL (X86_BR_PLM | X86_BR_ANY)
137 
138 #define X86_BR_ANY_CALL          \
139         (X86_BR_CALL            |\
140          X86_BR_IND_CALL        |\
141          X86_BR_ZERO_CALL       |\
142          X86_BR_SYSCALL         |\
143          X86_BR_IRQ             |\
144          X86_BR_INT)
145 
146 static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc);
147 
148 /*
149  * We only support LBR implementations that have FREEZE_LBRS_ON_PMI
150  * otherwise it becomes near impossible to get a reliable stack.
151  */
152 
153 static void __intel_pmu_lbr_enable(bool pmi)
154 {
155         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
156         u64 debugctl, lbr_select = 0, orig_debugctl;
157 
158         /*
159          * No need to unfreeze manually, as v4 can do that as part
160          * of the GLOBAL_STATUS ack.
161          */
162         if (pmi && x86_pmu.version >= 4)
163                 return;
164 
165         /*
166          * No need to reprogram LBR_SELECT in a PMI, as it
167          * did not change.
168          */
169         if (cpuc->lbr_sel)
170                 lbr_select = cpuc->lbr_sel->config & x86_pmu.lbr_sel_mask;
171         if (!pmi && cpuc->lbr_sel)
172                 wrmsrl(MSR_LBR_SELECT, lbr_select);
173 
174         rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
175         orig_debugctl = debugctl;
176         debugctl |= DEBUGCTLMSR_LBR;
177         /*
178          * LBR callstack does not work well with FREEZE_LBRS_ON_PMI.
179          * If FREEZE_LBRS_ON_PMI is set, PMI near call/return instructions
180          * may cause superfluous increase/decrease of LBR_TOS.
181          */
182         if (!(lbr_select & LBR_CALL_STACK))
183                 debugctl |= DEBUGCTLMSR_FREEZE_LBRS_ON_PMI;
184         if (orig_debugctl != debugctl)
185                 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
186 }
187 
188 static void __intel_pmu_lbr_disable(void)
189 {
190         u64 debugctl;
191 
192         rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
193         debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
194         wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
195 }
196 
197 static void intel_pmu_lbr_reset_32(void)
198 {
199         int i;
200 
201         for (i = 0; i < x86_pmu.lbr_nr; i++)
202                 wrmsrl(x86_pmu.lbr_from + i, 0);
203 }
204 
205 static void intel_pmu_lbr_reset_64(void)
206 {
207         int i;
208 
209         for (i = 0; i < x86_pmu.lbr_nr; i++) {
210                 wrmsrl(x86_pmu.lbr_from + i, 0);
211                 wrmsrl(x86_pmu.lbr_to   + i, 0);
212                 if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO)
213                         wrmsrl(MSR_LBR_INFO_0 + i, 0);
214         }
215 }
216 
217 void intel_pmu_lbr_reset(void)
218 {
219         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
220 
221         if (!x86_pmu.lbr_nr)
222                 return;
223 
224         if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
225                 intel_pmu_lbr_reset_32();
226         else
227                 intel_pmu_lbr_reset_64();
228 
229         cpuc->last_task_ctx = NULL;
230         cpuc->last_log_id = 0;
231 }
232 
233 /*
234  * TOS = most recently recorded branch
235  */
236 static inline u64 intel_pmu_lbr_tos(void)
237 {
238         u64 tos;
239 
240         rdmsrl(x86_pmu.lbr_tos, tos);
241         return tos;
242 }
243 
244 enum {
245         LBR_NONE,
246         LBR_VALID,
247 };
248 
249 /*
250  * For formats with LBR_TSX flags (e.g. LBR_FORMAT_EIP_FLAGS2), bits 61:62 in
251  * MSR_LAST_BRANCH_FROM_x are the TSX flags when TSX is supported, but when
252  * TSX is not supported they have no consistent behavior:
253  *
254  *   - For wrmsr(), bits 61:62 are considered part of the sign extension.
255  *   - For HW updates (branch captures) bits 61:62 are always OFF and are not
256  *     part of the sign extension.
257  *
258  * Therefore, if:
259  *
260  *   1) LBR has TSX format
261  *   2) CPU has no TSX support enabled
262  *
263  * ... then any value passed to wrmsr() must be sign extended to 63 bits and any
264  * value from rdmsr() must be converted to have a 61 bits sign extension,
265  * ignoring the TSX flags.
266  */
267 static inline bool lbr_from_signext_quirk_needed(void)
268 {
269         int lbr_format = x86_pmu.intel_cap.lbr_format;
270         bool tsx_support = boot_cpu_has(X86_FEATURE_HLE) ||
271                            boot_cpu_has(X86_FEATURE_RTM);
272 
273         return !tsx_support && (lbr_desc[lbr_format] & LBR_TSX);
274 }
275 
276 DEFINE_STATIC_KEY_FALSE(lbr_from_quirk_key);
277 
278 /* If quirk is enabled, ensure sign extension is 63 bits: */
279 inline u64 lbr_from_signext_quirk_wr(u64 val)
280 {
281         if (static_branch_unlikely(&lbr_from_quirk_key)) {
282                 /*
283                  * Sign extend into bits 61:62 while preserving bit 63.
284                  *
285                  * Quirk is enabled when TSX is disabled. Therefore TSX bits
286                  * in val are always OFF and must be changed to be sign
287                  * extension bits. Since bits 59:60 are guaranteed to be
288                  * part of the sign extension bits, we can just copy them
289                  * to 61:62.
290                  */
291                 val |= (LBR_FROM_SIGNEXT_2MSB & val) << 2;
292         }
293         return val;
294 }
295 
296 /*
297  * If quirk is needed, ensure sign extension is 61 bits:
298  */
299 static u64 lbr_from_signext_quirk_rd(u64 val)
300 {
301         if (static_branch_unlikely(&lbr_from_quirk_key)) {
302                 /*
303                  * Quirk is on when TSX is not enabled. Therefore TSX
304                  * flags must be read as OFF.
305                  */
306                 val &= ~(LBR_FROM_FLAG_IN_TX | LBR_FROM_FLAG_ABORT);
307         }
308         return val;
309 }
310 
311 static inline void wrlbr_from(unsigned int idx, u64 val)
312 {
313         val = lbr_from_signext_quirk_wr(val);
314         wrmsrl(x86_pmu.lbr_from + idx, val);
315 }
316 
317 static inline void wrlbr_to(unsigned int idx, u64 val)
318 {
319         wrmsrl(x86_pmu.lbr_to + idx, val);
320 }
321 
322 static inline u64 rdlbr_from(unsigned int idx)
323 {
324         u64 val;
325 
326         rdmsrl(x86_pmu.lbr_from + idx, val);
327 
328         return lbr_from_signext_quirk_rd(val);
329 }
330 
331 static inline u64 rdlbr_to(unsigned int idx)
332 {
333         u64 val;
334 
335         rdmsrl(x86_pmu.lbr_to + idx, val);
336 
337         return val;
338 }
339 
340 static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx)
341 {
342         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
343         int i;
344         unsigned lbr_idx, mask;
345         u64 tos;
346 
347         if (task_ctx->lbr_callstack_users == 0 ||
348             task_ctx->lbr_stack_state == LBR_NONE) {
349                 intel_pmu_lbr_reset();
350                 return;
351         }
352 
353         tos = task_ctx->tos;
354         /*
355          * Does not restore the LBR registers, if
356          * - No one else touched them, and
357          * - Did not enter C6
358          */
359         if ((task_ctx == cpuc->last_task_ctx) &&
360             (task_ctx->log_id == cpuc->last_log_id) &&
361             rdlbr_from(tos)) {
362                 task_ctx->lbr_stack_state = LBR_NONE;
363                 return;
364         }
365 
366         mask = x86_pmu.lbr_nr - 1;
367         for (i = 0; i < task_ctx->valid_lbrs; i++) {
368                 lbr_idx = (tos - i) & mask;
369                 wrlbr_from(lbr_idx, task_ctx->lbr_from[i]);
370                 wrlbr_to  (lbr_idx, task_ctx->lbr_to[i]);
371 
372                 if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO)
373                         wrmsrl(MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]);
374         }
375 
376         for (; i < x86_pmu.lbr_nr; i++) {
377                 lbr_idx = (tos - i) & mask;
378                 wrlbr_from(lbr_idx, 0);
379                 wrlbr_to(lbr_idx, 0);
380                 if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO)
381                         wrmsrl(MSR_LBR_INFO_0 + lbr_idx, 0);
382         }
383 
384         wrmsrl(x86_pmu.lbr_tos, tos);
385         task_ctx->lbr_stack_state = LBR_NONE;
386 }
387 
388 static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx)
389 {
390         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
391         unsigned lbr_idx, mask;
392         u64 tos, from;
393         int i;
394 
395         if (task_ctx->lbr_callstack_users == 0) {
396                 task_ctx->lbr_stack_state = LBR_NONE;
397                 return;
398         }
399 
400         mask = x86_pmu.lbr_nr - 1;
401         tos = intel_pmu_lbr_tos();
402         for (i = 0; i < x86_pmu.lbr_nr; i++) {
403                 lbr_idx = (tos - i) & mask;
404                 from = rdlbr_from(lbr_idx);
405                 if (!from)
406                         break;
407                 task_ctx->lbr_from[i] = from;
408                 task_ctx->lbr_to[i]   = rdlbr_to(lbr_idx);
409                 if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO)
410                         rdmsrl(MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]);
411         }
412         task_ctx->valid_lbrs = i;
413         task_ctx->tos = tos;
414         task_ctx->lbr_stack_state = LBR_VALID;
415 
416         cpuc->last_task_ctx = task_ctx;
417         cpuc->last_log_id = ++task_ctx->log_id;
418 }
419 
420 void intel_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in)
421 {
422         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
423         struct x86_perf_task_context *task_ctx;
424 
425         if (!cpuc->lbr_users)
426                 return;
427 
428         /*
429          * If LBR callstack feature is enabled and the stack was saved when
430          * the task was scheduled out, restore the stack. Otherwise flush
431          * the LBR stack.
432          */
433         task_ctx = ctx ? ctx->task_ctx_data : NULL;
434         if (task_ctx) {
435                 if (sched_in)
436                         __intel_pmu_lbr_restore(task_ctx);
437                 else
438                         __intel_pmu_lbr_save(task_ctx);
439                 return;
440         }
441 
442         /*
443          * Since a context switch can flip the address space and LBR entries
444          * are not tagged with an identifier, we need to wipe the LBR, even for
445          * per-cpu events. You simply cannot resolve the branches from the old
446          * address space.
447          */
448         if (sched_in)
449                 intel_pmu_lbr_reset();
450 }
451 
452 static inline bool branch_user_callstack(unsigned br_sel)
453 {
454         return (br_sel & X86_BR_USER) && (br_sel & X86_BR_CALL_STACK);
455 }
456 
457 void intel_pmu_lbr_add(struct perf_event *event)
458 {
459         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
460         struct x86_perf_task_context *task_ctx;
461 
462         if (!x86_pmu.lbr_nr)
463                 return;
464 
465         cpuc->br_sel = event->hw.branch_reg.reg;
466 
467         if (branch_user_callstack(cpuc->br_sel) && event->ctx->task_ctx_data) {
468                 task_ctx = event->ctx->task_ctx_data;
469                 task_ctx->lbr_callstack_users++;
470         }
471 
472         /*
473          * Request pmu::sched_task() callback, which will fire inside the
474          * regular perf event scheduling, so that call will:
475          *
476          *  - restore or wipe; when LBR-callstack,
477          *  - wipe; otherwise,
478          *
479          * when this is from __perf_event_task_sched_in().
480          *
481          * However, if this is from perf_install_in_context(), no such callback
482          * will follow and we'll need to reset the LBR here if this is the
483          * first LBR event.
484          *
485          * The problem is, we cannot tell these cases apart... but we can
486          * exclude the biggest chunk of cases by looking at
487          * event->total_time_running. An event that has accrued runtime cannot
488          * be 'new'. Conversely, a new event can get installed through the
489          * context switch path for the first time.
490          */
491         perf_sched_cb_inc(event->ctx->pmu);
492         if (!cpuc->lbr_users++ && !event->total_time_running)
493                 intel_pmu_lbr_reset();
494 }
495 
496 void intel_pmu_lbr_del(struct perf_event *event)
497 {
498         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
499         struct x86_perf_task_context *task_ctx;
500 
501         if (!x86_pmu.lbr_nr)
502                 return;
503 
504         if (branch_user_callstack(cpuc->br_sel) &&
505             event->ctx->task_ctx_data) {
506                 task_ctx = event->ctx->task_ctx_data;
507                 task_ctx->lbr_callstack_users--;
508         }
509 
510         cpuc->lbr_users--;
511         WARN_ON_ONCE(cpuc->lbr_users < 0);
512         perf_sched_cb_dec(event->ctx->pmu);
513 }
514 
515 void intel_pmu_lbr_enable_all(bool pmi)
516 {
517         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
518 
519         if (cpuc->lbr_users)
520                 __intel_pmu_lbr_enable(pmi);
521 }
522 
523 void intel_pmu_lbr_disable_all(void)
524 {
525         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
526 
527         if (cpuc->lbr_users)
528                 __intel_pmu_lbr_disable();
529 }
530 
531 static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
532 {
533         unsigned long mask = x86_pmu.lbr_nr - 1;
534         u64 tos = intel_pmu_lbr_tos();
535         int i;
536 
537         for (i = 0; i < x86_pmu.lbr_nr; i++) {
538                 unsigned long lbr_idx = (tos - i) & mask;
539                 union {
540                         struct {
541                                 u32 from;
542                                 u32 to;
543                         };
544                         u64     lbr;
545                 } msr_lastbranch;
546 
547                 rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
548 
549                 cpuc->lbr_entries[i].from       = msr_lastbranch.from;
550                 cpuc->lbr_entries[i].to         = msr_lastbranch.to;
551                 cpuc->lbr_entries[i].mispred    = 0;
552                 cpuc->lbr_entries[i].predicted  = 0;
553                 cpuc->lbr_entries[i].in_tx      = 0;
554                 cpuc->lbr_entries[i].abort      = 0;
555                 cpuc->lbr_entries[i].cycles     = 0;
556                 cpuc->lbr_entries[i].type       = 0;
557                 cpuc->lbr_entries[i].reserved   = 0;
558         }
559         cpuc->lbr_stack.nr = i;
560 }
561 
562 /*
563  * Due to lack of segmentation in Linux the effective address (offset)
564  * is the same as the linear address, allowing us to merge the LIP and EIP
565  * LBR formats.
566  */
567 static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
568 {
569         bool need_info = false, call_stack = false;
570         unsigned long mask = x86_pmu.lbr_nr - 1;
571         int lbr_format = x86_pmu.intel_cap.lbr_format;
572         u64 tos = intel_pmu_lbr_tos();
573         int i;
574         int out = 0;
575         int num = x86_pmu.lbr_nr;
576 
577         if (cpuc->lbr_sel) {
578                 need_info = !(cpuc->lbr_sel->config & LBR_NO_INFO);
579                 if (cpuc->lbr_sel->config & LBR_CALL_STACK)
580                         call_stack = true;
581         }
582 
583         for (i = 0; i < num; i++) {
584                 unsigned long lbr_idx = (tos - i) & mask;
585                 u64 from, to, mis = 0, pred = 0, in_tx = 0, abort = 0;
586                 int skip = 0;
587                 u16 cycles = 0;
588                 int lbr_flags = lbr_desc[lbr_format];
589 
590                 from = rdlbr_from(lbr_idx);
591                 to   = rdlbr_to(lbr_idx);
592 
593                 /*
594                  * Read LBR call stack entries
595                  * until invalid entry (0s) is detected.
596                  */
597                 if (call_stack && !from)
598                         break;
599 
600                 if (lbr_format == LBR_FORMAT_INFO && need_info) {
601                         u64 info;
602 
603                         rdmsrl(MSR_LBR_INFO_0 + lbr_idx, info);
604                         mis = !!(info & LBR_INFO_MISPRED);
605                         pred = !mis;
606                         in_tx = !!(info & LBR_INFO_IN_TX);
607                         abort = !!(info & LBR_INFO_ABORT);
608                         cycles = (info & LBR_INFO_CYCLES);
609                 }
610 
611                 if (lbr_format == LBR_FORMAT_TIME) {
612                         mis = !!(from & LBR_FROM_FLAG_MISPRED);
613                         pred = !mis;
614                         skip = 1;
615                         cycles = ((to >> 48) & LBR_INFO_CYCLES);
616 
617                         to = (u64)((((s64)to) << 16) >> 16);
618                 }
619 
620                 if (lbr_flags & LBR_EIP_FLAGS) {
621                         mis = !!(from & LBR_FROM_FLAG_MISPRED);
622                         pred = !mis;
623                         skip = 1;
624                 }
625                 if (lbr_flags & LBR_TSX) {
626                         in_tx = !!(from & LBR_FROM_FLAG_IN_TX);
627                         abort = !!(from & LBR_FROM_FLAG_ABORT);
628                         skip = 3;
629                 }
630                 from = (u64)((((s64)from) << skip) >> skip);
631 
632                 /*
633                  * Some CPUs report duplicated abort records,
634                  * with the second entry not having an abort bit set.
635                  * Skip them here. This loop runs backwards,
636                  * so we need to undo the previous record.
637                  * If the abort just happened outside the window
638                  * the extra entry cannot be removed.
639                  */
640                 if (abort && x86_pmu.lbr_double_abort && out > 0)
641                         out--;
642 
643                 cpuc->lbr_entries[out].from      = from;
644                 cpuc->lbr_entries[out].to        = to;
645                 cpuc->lbr_entries[out].mispred   = mis;
646                 cpuc->lbr_entries[out].predicted = pred;
647                 cpuc->lbr_entries[out].in_tx     = in_tx;
648                 cpuc->lbr_entries[out].abort     = abort;
649                 cpuc->lbr_entries[out].cycles    = cycles;
650                 cpuc->lbr_entries[out].type      = 0;
651                 cpuc->lbr_entries[out].reserved  = 0;
652                 out++;
653         }
654         cpuc->lbr_stack.nr = out;
655 }
656 
657 void intel_pmu_lbr_read(void)
658 {
659         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
660 
661         if (!cpuc->lbr_users)
662                 return;
663 
664         if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
665                 intel_pmu_lbr_read_32(cpuc);
666         else
667                 intel_pmu_lbr_read_64(cpuc);
668 
669         intel_pmu_lbr_filter(cpuc);
670 }
671 
672 /*
673  * SW filter is used:
674  * - in case there is no HW filter
675  * - in case the HW filter has errata or limitations
676  */
677 static int intel_pmu_setup_sw_lbr_filter(struct perf_event *event)
678 {
679         u64 br_type = event->attr.branch_sample_type;
680         int mask = 0;
681 
682         if (br_type & PERF_SAMPLE_BRANCH_USER)
683                 mask |= X86_BR_USER;
684 
685         if (br_type & PERF_SAMPLE_BRANCH_KERNEL)
686                 mask |= X86_BR_KERNEL;
687 
688         /* we ignore BRANCH_HV here */
689 
690         if (br_type & PERF_SAMPLE_BRANCH_ANY)
691                 mask |= X86_BR_ANY;
692 
693         if (br_type & PERF_SAMPLE_BRANCH_ANY_CALL)
694                 mask |= X86_BR_ANY_CALL;
695 
696         if (br_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
697                 mask |= X86_BR_RET | X86_BR_IRET | X86_BR_SYSRET;
698 
699         if (br_type & PERF_SAMPLE_BRANCH_IND_CALL)
700                 mask |= X86_BR_IND_CALL;
701 
702         if (br_type & PERF_SAMPLE_BRANCH_ABORT_TX)
703                 mask |= X86_BR_ABORT;
704 
705         if (br_type & PERF_SAMPLE_BRANCH_IN_TX)
706                 mask |= X86_BR_IN_TX;
707 
708         if (br_type & PERF_SAMPLE_BRANCH_NO_TX)
709                 mask |= X86_BR_NO_TX;
710 
711         if (br_type & PERF_SAMPLE_BRANCH_COND)
712                 mask |= X86_BR_JCC;
713 
714         if (br_type & PERF_SAMPLE_BRANCH_CALL_STACK) {
715                 if (!x86_pmu_has_lbr_callstack())
716                         return -EOPNOTSUPP;
717                 if (mask & ~(X86_BR_USER | X86_BR_KERNEL))
718                         return -EINVAL;
719                 mask |= X86_BR_CALL | X86_BR_IND_CALL | X86_BR_RET |
720                         X86_BR_CALL_STACK;
721         }
722 
723         if (br_type & PERF_SAMPLE_BRANCH_IND_JUMP)
724                 mask |= X86_BR_IND_JMP;
725 
726         if (br_type & PERF_SAMPLE_BRANCH_CALL)
727                 mask |= X86_BR_CALL | X86_BR_ZERO_CALL;
728 
729         if (br_type & PERF_SAMPLE_BRANCH_TYPE_SAVE)
730                 mask |= X86_BR_TYPE_SAVE;
731 
732         /*
733          * stash actual user request into reg, it may
734          * be used by fixup code for some CPU
735          */
736         event->hw.branch_reg.reg = mask;
737         return 0;
738 }
739 
740 /*
741  * setup the HW LBR filter
742  * Used only when available, may not be enough to disambiguate
743  * all branches, may need the help of the SW filter
744  */
745 static int intel_pmu_setup_hw_lbr_filter(struct perf_event *event)
746 {
747         struct hw_perf_event_extra *reg;
748         u64 br_type = event->attr.branch_sample_type;
749         u64 mask = 0, v;
750         int i;
751 
752         for (i = 0; i < PERF_SAMPLE_BRANCH_MAX_SHIFT; i++) {
753                 if (!(br_type & (1ULL << i)))
754                         continue;
755 
756                 v = x86_pmu.lbr_sel_map[i];
757                 if (v == LBR_NOT_SUPP)
758                         return -EOPNOTSUPP;
759 
760                 if (v != LBR_IGN)
761                         mask |= v;
762         }
763 
764         reg = &event->hw.branch_reg;
765         reg->idx = EXTRA_REG_LBR;
766 
767         /*
768          * The first 9 bits (LBR_SEL_MASK) in LBR_SELECT operate
769          * in suppress mode. So LBR_SELECT should be set to
770          * (~mask & LBR_SEL_MASK) | (mask & ~LBR_SEL_MASK)
771          * But the 10th bit LBR_CALL_STACK does not operate
772          * in suppress mode.
773          */
774         reg->config = mask ^ (x86_pmu.lbr_sel_mask & ~LBR_CALL_STACK);
775 
776         if ((br_type & PERF_SAMPLE_BRANCH_NO_CYCLES) &&
777             (br_type & PERF_SAMPLE_BRANCH_NO_FLAGS) &&
778             (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO))
779                 reg->config |= LBR_NO_INFO;
780 
781         return 0;
782 }
783 
784 int intel_pmu_setup_lbr_filter(struct perf_event *event)
785 {
786         int ret = 0;
787 
788         /*
789          * no LBR on this PMU
790          */
791         if (!x86_pmu.lbr_nr)
792                 return -EOPNOTSUPP;
793 
794         /*
795          * setup SW LBR filter
796          */
797         ret = intel_pmu_setup_sw_lbr_filter(event);
798         if (ret)
799                 return ret;
800 
801         /*
802          * setup HW LBR filter, if any
803          */
804         if (x86_pmu.lbr_sel_map)
805                 ret = intel_pmu_setup_hw_lbr_filter(event);
806 
807         return ret;
808 }
809 
810 /*
811  * return the type of control flow change at address "from"
812  * instruction is not necessarily a branch (in case of interrupt).
813  *
814  * The branch type returned also includes the priv level of the
815  * target of the control flow change (X86_BR_USER, X86_BR_KERNEL).
816  *
817  * If a branch type is unknown OR the instruction cannot be
818  * decoded (e.g., text page not present), then X86_BR_NONE is
819  * returned.
820  */
821 static int branch_type(unsigned long from, unsigned long to, int abort)
822 {
823         struct insn insn;
824         void *addr;
825         int bytes_read, bytes_left;
826         int ret = X86_BR_NONE;
827         int ext, to_plm, from_plm;
828         u8 buf[MAX_INSN_SIZE];
829         int is64 = 0;
830 
831         to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER;
832         from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER;
833 
834         /*
835          * maybe zero if lbr did not fill up after a reset by the time
836          * we get a PMU interrupt
837          */
838         if (from == 0 || to == 0)
839                 return X86_BR_NONE;
840 
841         if (abort)
842                 return X86_BR_ABORT | to_plm;
843 
844         if (from_plm == X86_BR_USER) {
845                 /*
846                  * can happen if measuring at the user level only
847                  * and we interrupt in a kernel thread, e.g., idle.
848                  */
849                 if (!current->mm)
850                         return X86_BR_NONE;
851 
852                 /* may fail if text not present */
853                 bytes_left = copy_from_user_nmi(buf, (void __user *)from,
854                                                 MAX_INSN_SIZE);
855                 bytes_read = MAX_INSN_SIZE - bytes_left;
856                 if (!bytes_read)
857                         return X86_BR_NONE;
858 
859                 addr = buf;
860         } else {
861                 /*
862                  * The LBR logs any address in the IP, even if the IP just
863                  * faulted. This means userspace can control the from address.
864                  * Ensure we don't blindy read any address by validating it is
865                  * a known text address.
866                  */
867                 if (kernel_text_address(from)) {
868                         addr = (void *)from;
869                         /*
870                          * Assume we can get the maximum possible size
871                          * when grabbing kernel data.  This is not
872                          * _strictly_ true since we could possibly be
873                          * executing up next to a memory hole, but
874                          * it is very unlikely to be a problem.
875                          */
876                         bytes_read = MAX_INSN_SIZE;
877                 } else {
878                         return X86_BR_NONE;
879                 }
880         }
881 
882         /*
883          * decoder needs to know the ABI especially
884          * on 64-bit systems running 32-bit apps
885          */
886 #ifdef CONFIG_X86_64
887         is64 = kernel_ip((unsigned long)addr) || !test_thread_flag(TIF_IA32);
888 #endif
889         insn_init(&insn, addr, bytes_read, is64);
890         insn_get_opcode(&insn);
891         if (!insn.opcode.got)
892                 return X86_BR_ABORT;
893 
894         switch (insn.opcode.bytes[0]) {
895         case 0xf:
896                 switch (insn.opcode.bytes[1]) {
897                 case 0x05: /* syscall */
898                 case 0x34: /* sysenter */
899                         ret = X86_BR_SYSCALL;
900                         break;
901                 case 0x07: /* sysret */
902                 case 0x35: /* sysexit */
903                         ret = X86_BR_SYSRET;
904                         break;
905                 case 0x80 ... 0x8f: /* conditional */
906                         ret = X86_BR_JCC;
907                         break;
908                 default:
909                         ret = X86_BR_NONE;
910                 }
911                 break;
912         case 0x70 ... 0x7f: /* conditional */
913                 ret = X86_BR_JCC;
914                 break;
915         case 0xc2: /* near ret */
916         case 0xc3: /* near ret */
917         case 0xca: /* far ret */
918         case 0xcb: /* far ret */
919                 ret = X86_BR_RET;
920                 break;
921         case 0xcf: /* iret */
922                 ret = X86_BR_IRET;
923                 break;
924         case 0xcc ... 0xce: /* int */
925                 ret = X86_BR_INT;
926                 break;
927         case 0xe8: /* call near rel */
928                 insn_get_immediate(&insn);
929                 if (insn.immediate1.value == 0) {
930                         /* zero length call */
931                         ret = X86_BR_ZERO_CALL;
932                         break;
933                 }
934                 /* fall through */
935         case 0x9a: /* call far absolute */
936                 ret = X86_BR_CALL;
937                 break;
938         case 0xe0 ... 0xe3: /* loop jmp */
939                 ret = X86_BR_JCC;
940                 break;
941         case 0xe9 ... 0xeb: /* jmp */
942                 ret = X86_BR_JMP;
943                 break;
944         case 0xff: /* call near absolute, call far absolute ind */
945                 insn_get_modrm(&insn);
946                 ext = (insn.modrm.bytes[0] >> 3) & 0x7;
947                 switch (ext) {
948                 case 2: /* near ind call */
949                 case 3: /* far ind call */
950                         ret = X86_BR_IND_CALL;
951                         break;
952                 case 4:
953                 case 5:
954                         ret = X86_BR_IND_JMP;
955                         break;
956                 }
957                 break;
958         default:
959                 ret = X86_BR_NONE;
960         }
961         /*
962          * interrupts, traps, faults (and thus ring transition) may
963          * occur on any instructions. Thus, to classify them correctly,
964          * we need to first look at the from and to priv levels. If they
965          * are different and to is in the kernel, then it indicates
966          * a ring transition. If the from instruction is not a ring
967          * transition instr (syscall, systenter, int), then it means
968          * it was a irq, trap or fault.
969          *
970          * we have no way of detecting kernel to kernel faults.
971          */
972         if (from_plm == X86_BR_USER && to_plm == X86_BR_KERNEL
973             && ret != X86_BR_SYSCALL && ret != X86_BR_INT)
974                 ret = X86_BR_IRQ;
975 
976         /*
977          * branch priv level determined by target as
978          * is done by HW when LBR_SELECT is implemented
979          */
980         if (ret != X86_BR_NONE)
981                 ret |= to_plm;
982 
983         return ret;
984 }
985 
986 #define X86_BR_TYPE_MAP_MAX     16
987 
988 static int branch_map[X86_BR_TYPE_MAP_MAX] = {
989         PERF_BR_CALL,           /* X86_BR_CALL */
990         PERF_BR_RET,            /* X86_BR_RET */
991         PERF_BR_SYSCALL,        /* X86_BR_SYSCALL */
992         PERF_BR_SYSRET,         /* X86_BR_SYSRET */
993         PERF_BR_UNKNOWN,        /* X86_BR_INT */
994         PERF_BR_UNKNOWN,        /* X86_BR_IRET */
995         PERF_BR_COND,           /* X86_BR_JCC */
996         PERF_BR_UNCOND,         /* X86_BR_JMP */
997         PERF_BR_UNKNOWN,        /* X86_BR_IRQ */
998         PERF_BR_IND_CALL,       /* X86_BR_IND_CALL */
999         PERF_BR_UNKNOWN,        /* X86_BR_ABORT */
1000         PERF_BR_UNKNOWN,        /* X86_BR_IN_TX */
1001         PERF_BR_UNKNOWN,        /* X86_BR_NO_TX */
1002         PERF_BR_CALL,           /* X86_BR_ZERO_CALL */
1003         PERF_BR_UNKNOWN,        /* X86_BR_CALL_STACK */
1004         PERF_BR_IND,            /* X86_BR_IND_JMP */
1005 };
1006 
1007 static int
1008 common_branch_type(int type)
1009 {
1010         int i;
1011 
1012         type >>= 2; /* skip X86_BR_USER and X86_BR_KERNEL */
1013 
1014         if (type) {
1015                 i = __ffs(type);
1016                 if (i < X86_BR_TYPE_MAP_MAX)
1017                         return branch_map[i];
1018         }
1019 
1020         return PERF_BR_UNKNOWN;
1021 }
1022 
1023 /*
1024  * implement actual branch filter based on user demand.
1025  * Hardware may not exactly satisfy that request, thus
1026  * we need to inspect opcodes. Mismatched branches are
1027  * discarded. Therefore, the number of branches returned
1028  * in PERF_SAMPLE_BRANCH_STACK sample may vary.
1029  */
1030 static void
1031 intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
1032 {
1033         u64 from, to;
1034         int br_sel = cpuc->br_sel;
1035         int i, j, type;
1036         bool compress = false;
1037 
1038         /* if sampling all branches, then nothing to filter */
1039         if (((br_sel & X86_BR_ALL) == X86_BR_ALL) &&
1040             ((br_sel & X86_BR_TYPE_SAVE) != X86_BR_TYPE_SAVE))
1041                 return;
1042 
1043         for (i = 0; i < cpuc->lbr_stack.nr; i++) {
1044 
1045                 from = cpuc->lbr_entries[i].from;
1046                 to = cpuc->lbr_entries[i].to;
1047 
1048                 type = branch_type(from, to, cpuc->lbr_entries[i].abort);
1049                 if (type != X86_BR_NONE && (br_sel & X86_BR_ANYTX)) {
1050                         if (cpuc->lbr_entries[i].in_tx)
1051                                 type |= X86_BR_IN_TX;
1052                         else
1053                                 type |= X86_BR_NO_TX;
1054                 }
1055 
1056                 /* if type does not correspond, then discard */
1057                 if (type == X86_BR_NONE || (br_sel & type) != type) {
1058                         cpuc->lbr_entries[i].from = 0;
1059                         compress = true;
1060                 }
1061 
1062                 if ((br_sel & X86_BR_TYPE_SAVE) == X86_BR_TYPE_SAVE)
1063                         cpuc->lbr_entries[i].type = common_branch_type(type);
1064         }
1065 
1066         if (!compress)
1067                 return;
1068 
1069         /* remove all entries with from=0 */
1070         for (i = 0; i < cpuc->lbr_stack.nr; ) {
1071                 if (!cpuc->lbr_entries[i].from) {
1072                         j = i;
1073                         while (++j < cpuc->lbr_stack.nr)
1074                                 cpuc->lbr_entries[j-1] = cpuc->lbr_entries[j];
1075                         cpuc->lbr_stack.nr--;
1076                         if (!cpuc->lbr_entries[i].from)
1077                                 continue;
1078                 }
1079                 i++;
1080         }
1081 }
1082 
1083 /*
1084  * Map interface branch filters onto LBR filters
1085  */
1086 static const int nhm_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = {
1087         [PERF_SAMPLE_BRANCH_ANY_SHIFT]          = LBR_ANY,
1088         [PERF_SAMPLE_BRANCH_USER_SHIFT]         = LBR_USER,
1089         [PERF_SAMPLE_BRANCH_KERNEL_SHIFT]       = LBR_KERNEL,
1090         [PERF_SAMPLE_BRANCH_HV_SHIFT]           = LBR_IGN,
1091         [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT]   = LBR_RETURN | LBR_REL_JMP
1092                                                 | LBR_IND_JMP | LBR_FAR,
1093         /*
1094          * NHM/WSM erratum: must include REL_JMP+IND_JMP to get CALL branches
1095          */
1096         [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] =
1097          LBR_REL_CALL | LBR_IND_CALL | LBR_REL_JMP | LBR_IND_JMP | LBR_FAR,
1098         /*
1099          * NHM/WSM erratum: must include IND_JMP to capture IND_CALL
1100          */
1101         [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL | LBR_IND_JMP,
1102         [PERF_SAMPLE_BRANCH_COND_SHIFT]     = LBR_JCC,
1103         [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP,
1104 };
1105 
1106 static const int snb_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = {
1107         [PERF_SAMPLE_BRANCH_ANY_SHIFT]          = LBR_ANY,
1108         [PERF_SAMPLE_BRANCH_USER_SHIFT]         = LBR_USER,
1109         [PERF_SAMPLE_BRANCH_KERNEL_SHIFT]       = LBR_KERNEL,
1110         [PERF_SAMPLE_BRANCH_HV_SHIFT]           = LBR_IGN,
1111         [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT]   = LBR_RETURN | LBR_FAR,
1112         [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT]     = LBR_REL_CALL | LBR_IND_CALL
1113                                                 | LBR_FAR,
1114         [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT]     = LBR_IND_CALL,
1115         [PERF_SAMPLE_BRANCH_COND_SHIFT]         = LBR_JCC,
1116         [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT]     = LBR_IND_JMP,
1117         [PERF_SAMPLE_BRANCH_CALL_SHIFT]         = LBR_REL_CALL,
1118 };
1119 
1120 static const int hsw_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = {
1121         [PERF_SAMPLE_BRANCH_ANY_SHIFT]          = LBR_ANY,
1122         [PERF_SAMPLE_BRANCH_USER_SHIFT]         = LBR_USER,
1123         [PERF_SAMPLE_BRANCH_KERNEL_SHIFT]       = LBR_KERNEL,
1124         [PERF_SAMPLE_BRANCH_HV_SHIFT]           = LBR_IGN,
1125         [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT]   = LBR_RETURN | LBR_FAR,
1126         [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT]     = LBR_REL_CALL | LBR_IND_CALL
1127                                                 | LBR_FAR,
1128         [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT]     = LBR_IND_CALL,
1129         [PERF_SAMPLE_BRANCH_COND_SHIFT]         = LBR_JCC,
1130         [PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT]   = LBR_REL_CALL | LBR_IND_CALL
1131                                                 | LBR_RETURN | LBR_CALL_STACK,
1132         [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT]     = LBR_IND_JMP,
1133         [PERF_SAMPLE_BRANCH_CALL_SHIFT]         = LBR_REL_CALL,
1134 };
1135 
1136 /* core */
1137 void __init intel_pmu_lbr_init_core(void)
1138 {
1139         x86_pmu.lbr_nr     = 4;
1140         x86_pmu.lbr_tos    = MSR_LBR_TOS;
1141         x86_pmu.lbr_from   = MSR_LBR_CORE_FROM;
1142         x86_pmu.lbr_to     = MSR_LBR_CORE_TO;
1143 
1144         /*
1145          * SW branch filter usage:
1146          * - compensate for lack of HW filter
1147          */
1148 }
1149 
1150 /* nehalem/westmere */
1151 void __init intel_pmu_lbr_init_nhm(void)
1152 {
1153         x86_pmu.lbr_nr     = 16;
1154         x86_pmu.lbr_tos    = MSR_LBR_TOS;
1155         x86_pmu.lbr_from   = MSR_LBR_NHM_FROM;
1156         x86_pmu.lbr_to     = MSR_LBR_NHM_TO;
1157 
1158         x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
1159         x86_pmu.lbr_sel_map  = nhm_lbr_sel_map;
1160 
1161         /*
1162          * SW branch filter usage:
1163          * - workaround LBR_SEL errata (see above)
1164          * - support syscall, sysret capture.
1165          *   That requires LBR_FAR but that means far
1166          *   jmp need to be filtered out
1167          */
1168 }
1169 
1170 /* sandy bridge */
1171 void __init intel_pmu_lbr_init_snb(void)
1172 {
1173         x86_pmu.lbr_nr   = 16;
1174         x86_pmu.lbr_tos  = MSR_LBR_TOS;
1175         x86_pmu.lbr_from = MSR_LBR_NHM_FROM;
1176         x86_pmu.lbr_to   = MSR_LBR_NHM_TO;
1177 
1178         x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
1179         x86_pmu.lbr_sel_map  = snb_lbr_sel_map;
1180 
1181         /*
1182          * SW branch filter usage:
1183          * - support syscall, sysret capture.
1184          *   That requires LBR_FAR but that means far
1185          *   jmp need to be filtered out
1186          */
1187 }
1188 
1189 /* haswell */
1190 void intel_pmu_lbr_init_hsw(void)
1191 {
1192         x86_pmu.lbr_nr   = 16;
1193         x86_pmu.lbr_tos  = MSR_LBR_TOS;
1194         x86_pmu.lbr_from = MSR_LBR_NHM_FROM;
1195         x86_pmu.lbr_to   = MSR_LBR_NHM_TO;
1196 
1197         x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
1198         x86_pmu.lbr_sel_map  = hsw_lbr_sel_map;
1199 
1200         if (lbr_from_signext_quirk_needed())
1201                 static_branch_enable(&lbr_from_quirk_key);
1202 }
1203 
1204 /* skylake */
1205 __init void intel_pmu_lbr_init_skl(void)
1206 {
1207         x86_pmu.lbr_nr   = 32;
1208         x86_pmu.lbr_tos  = MSR_LBR_TOS;
1209         x86_pmu.lbr_from = MSR_LBR_NHM_FROM;
1210         x86_pmu.lbr_to   = MSR_LBR_NHM_TO;
1211 
1212         x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
1213         x86_pmu.lbr_sel_map  = hsw_lbr_sel_map;
1214 
1215         /*
1216          * SW branch filter usage:
1217          * - support syscall, sysret capture.
1218          *   That requires LBR_FAR but that means far
1219          *   jmp need to be filtered out
1220          */
1221 }
1222 
1223 /* atom */
1224 void __init intel_pmu_lbr_init_atom(void)
1225 {
1226         /*
1227          * only models starting at stepping 10 seems
1228          * to have an operational LBR which can freeze
1229          * on PMU interrupt
1230          */
1231         if (boot_cpu_data.x86_model == 28
1232             && boot_cpu_data.x86_stepping < 10) {
1233                 pr_cont("LBR disabled due to erratum");
1234                 return;
1235         }
1236 
1237         x86_pmu.lbr_nr     = 8;
1238         x86_pmu.lbr_tos    = MSR_LBR_TOS;
1239         x86_pmu.lbr_from   = MSR_LBR_CORE_FROM;
1240         x86_pmu.lbr_to     = MSR_LBR_CORE_TO;
1241 
1242         /*
1243          * SW branch filter usage:
1244          * - compensate for lack of HW filter
1245          */
1246 }
1247 
1248 /* slm */
1249 void __init intel_pmu_lbr_init_slm(void)
1250 {
1251         x86_pmu.lbr_nr     = 8;
1252         x86_pmu.lbr_tos    = MSR_LBR_TOS;
1253         x86_pmu.lbr_from   = MSR_LBR_CORE_FROM;
1254         x86_pmu.lbr_to     = MSR_LBR_CORE_TO;
1255 
1256         x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
1257         x86_pmu.lbr_sel_map  = nhm_lbr_sel_map;
1258 
1259         /*
1260          * SW branch filter usage:
1261          * - compensate for lack of HW filter
1262          */
1263         pr_cont("8-deep LBR, ");
1264 }
1265 
1266 /* Knights Landing */
1267 void intel_pmu_lbr_init_knl(void)
1268 {
1269         x86_pmu.lbr_nr     = 8;
1270         x86_pmu.lbr_tos    = MSR_LBR_TOS;
1271         x86_pmu.lbr_from   = MSR_LBR_NHM_FROM;
1272         x86_pmu.lbr_to     = MSR_LBR_NHM_TO;
1273 
1274         x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
1275         x86_pmu.lbr_sel_map  = snb_lbr_sel_map;
1276 
1277         /* Knights Landing does have MISPREDICT bit */
1278         if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_LIP)
1279                 x86_pmu.intel_cap.lbr_format = LBR_FORMAT_EIP_FLAGS;
1280 }
1281 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp