~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/kernel/cpu/perf_event.h

Version: ~ [ linux-5.8-rc3 ] ~ [ linux-5.7.5 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.48 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.129 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.185 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.228 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.228 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Performance events x86 architecture header
  3  *
  4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
  6  *  Copyright (C) 2009 Jaswinder Singh Rajput
  7  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
  8  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
  9  *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
 10  *  Copyright (C) 2009 Google, Inc., Stephane Eranian
 11  *
 12  *  For licencing details see kernel-base/COPYING
 13  */
 14 
 15 #include <linux/perf_event.h>
 16 
 17 #if 0
 18 #undef wrmsrl
 19 #define wrmsrl(msr, val)                                                \
 20 do {                                                                    \
 21         unsigned int _msr = (msr);                                      \
 22         u64 _val = (val);                                               \
 23         trace_printk("wrmsrl(%x, %Lx)\n", (unsigned int)(_msr),         \
 24                         (unsigned long long)(_val));                    \
 25         native_write_msr((_msr), (u32)(_val), (u32)(_val >> 32));       \
 26 } while (0)
 27 #endif
 28 
 29 /*
 30  *          |   NHM/WSM    |      SNB     |
 31  * register -------------------------------
 32  *          |  HT  | no HT |  HT  | no HT |
 33  *-----------------------------------------
 34  * offcore  | core | core  | cpu  | core  |
 35  * lbr_sel  | core | core  | cpu  | core  |
 36  * ld_lat   | cpu  | core  | cpu  | core  |
 37  *-----------------------------------------
 38  *
 39  * Given that there is a small number of shared regs,
 40  * we can pre-allocate their slot in the per-cpu
 41  * per-core reg tables.
 42  */
 43 enum extra_reg_type {
 44         EXTRA_REG_NONE  = -1,   /* not used */
 45 
 46         EXTRA_REG_RSP_0 = 0,    /* offcore_response_0 */
 47         EXTRA_REG_RSP_1 = 1,    /* offcore_response_1 */
 48         EXTRA_REG_LBR   = 2,    /* lbr_select */
 49         EXTRA_REG_LDLAT = 3,    /* ld_lat_threshold */
 50 
 51         EXTRA_REG_MAX           /* number of entries needed */
 52 };
 53 
 54 struct event_constraint {
 55         union {
 56                 unsigned long   idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 57                 u64             idxmsk64;
 58         };
 59         u64     code;
 60         u64     cmask;
 61         int     weight;
 62         int     overlap;
 63         int     flags;
 64 };
 65 /*
 66  * struct hw_perf_event.flags flags
 67  */
 68 #define PERF_X86_EVENT_PEBS_LDLAT       0x1 /* ld+ldlat data address sampling */
 69 #define PERF_X86_EVENT_PEBS_ST          0x2 /* st data address sampling */
 70 #define PERF_X86_EVENT_PEBS_ST_HSW      0x4 /* haswell style datala, store */
 71 #define PERF_X86_EVENT_COMMITTED        0x8 /* event passed commit_txn */
 72 #define PERF_X86_EVENT_PEBS_LD_HSW      0x10 /* haswell style datala, load */
 73 #define PERF_X86_EVENT_PEBS_NA_HSW      0x20 /* haswell style datala, unknown */
 74 
 75 struct amd_nb {
 76         int nb_id;  /* NorthBridge id */
 77         int refcnt; /* reference count */
 78         struct perf_event *owners[X86_PMC_IDX_MAX];
 79         struct event_constraint event_constraints[X86_PMC_IDX_MAX];
 80 };
 81 
 82 /* The maximal number of PEBS events: */
 83 #define MAX_PEBS_EVENTS         8
 84 
 85 /*
 86  * A debug store configuration.
 87  *
 88  * We only support architectures that use 64bit fields.
 89  */
 90 struct debug_store {
 91         u64     bts_buffer_base;
 92         u64     bts_index;
 93         u64     bts_absolute_maximum;
 94         u64     bts_interrupt_threshold;
 95         u64     pebs_buffer_base;
 96         u64     pebs_index;
 97         u64     pebs_absolute_maximum;
 98         u64     pebs_interrupt_threshold;
 99         u64     pebs_event_reset[MAX_PEBS_EVENTS];
100 };
101 
102 /*
103  * Per register state.
104  */
105 struct er_account {
106         raw_spinlock_t          lock;   /* per-core: protect structure */
107         u64                 config;     /* extra MSR config */
108         u64                 reg;        /* extra MSR number */
109         atomic_t            ref;        /* reference count */
110 };
111 
112 /*
113  * Per core/cpu state
114  *
115  * Used to coordinate shared registers between HT threads or
116  * among events on a single PMU.
117  */
118 struct intel_shared_regs {
119         struct er_account       regs[EXTRA_REG_MAX];
120         int                     refcnt;         /* per-core: #HT threads */
121         unsigned                core_id;        /* per-core: core id */
122 };
123 
124 #define MAX_LBR_ENTRIES         16
125 
126 struct cpu_hw_events {
127         /*
128          * Generic x86 PMC bits
129          */
130         struct perf_event       *events[X86_PMC_IDX_MAX]; /* in counter order */
131         unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
132         unsigned long           running[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
133         int                     enabled;
134 
135         int                     n_events; /* the # of events in the below arrays */
136         int                     n_added;  /* the # last events in the below arrays;
137                                              they've never been enabled yet */
138         int                     n_txn;    /* the # last events in the below arrays;
139                                              added in the current transaction */
140         int                     assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
141         u64                     tags[X86_PMC_IDX_MAX];
142         struct perf_event       *event_list[X86_PMC_IDX_MAX]; /* in enabled order */
143 
144         unsigned int            group_flag;
145         int                     is_fake;
146 
147         /*
148          * Intel DebugStore bits
149          */
150         struct debug_store      *ds;
151         u64                     pebs_enabled;
152 
153         /*
154          * Intel LBR bits
155          */
156         int                             lbr_users;
157         void                            *lbr_context;
158         struct perf_branch_stack        lbr_stack;
159         struct perf_branch_entry        lbr_entries[MAX_LBR_ENTRIES];
160         struct er_account               *lbr_sel;
161         u64                             br_sel;
162 
163         /*
164          * Intel host/guest exclude bits
165          */
166         u64                             intel_ctrl_guest_mask;
167         u64                             intel_ctrl_host_mask;
168         struct perf_guest_switch_msr    guest_switch_msrs[X86_PMC_IDX_MAX];
169 
170         /*
171          * Intel checkpoint mask
172          */
173         u64                             intel_cp_status;
174 
175         /*
176          * manage shared (per-core, per-cpu) registers
177          * used on Intel NHM/WSM/SNB
178          */
179         struct intel_shared_regs        *shared_regs;
180 
181         /*
182          * AMD specific bits
183          */
184         struct amd_nb                   *amd_nb;
185         /* Inverted mask of bits to clear in the perf_ctr ctrl registers */
186         u64                             perf_ctr_virt_mask;
187 
188         void                            *kfree_on_online;
189 };
190 
191 #define __EVENT_CONSTRAINT(c, n, m, w, o, f) {\
192         { .idxmsk64 = (n) },            \
193         .code = (c),                    \
194         .cmask = (m),                   \
195         .weight = (w),                  \
196         .overlap = (o),                 \
197         .flags = f,                     \
198 }
199 
200 #define EVENT_CONSTRAINT(c, n, m)       \
201         __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 0, 0)
202 
203 /*
204  * The overlap flag marks event constraints with overlapping counter
205  * masks. This is the case if the counter mask of such an event is not
206  * a subset of any other counter mask of a constraint with an equal or
207  * higher weight, e.g.:
208  *
209  *  c_overlaps = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0);
210  *  c_another1 = EVENT_CONSTRAINT(0, 0x07, 0);
211  *  c_another2 = EVENT_CONSTRAINT(0, 0x38, 0);
212  *
213  * The event scheduler may not select the correct counter in the first
214  * cycle because it needs to know which subsequent events will be
215  * scheduled. It may fail to schedule the events then. So we set the
216  * overlap flag for such constraints to give the scheduler a hint which
217  * events to select for counter rescheduling.
218  *
219  * Care must be taken as the rescheduling algorithm is O(n!) which
220  * will increase scheduling cycles for an over-commited system
221  * dramatically.  The number of such EVENT_CONSTRAINT_OVERLAP() macros
222  * and its counter masks must be kept at a minimum.
223  */
224 #define EVENT_CONSTRAINT_OVERLAP(c, n, m)       \
225         __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 1, 0)
226 
227 /*
228  * Constraint on the Event code.
229  */
230 #define INTEL_EVENT_CONSTRAINT(c, n)    \
231         EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT)
232 
233 /*
234  * Constraint on the Event code + UMask + fixed-mask
235  *
236  * filter mask to validate fixed counter events.
237  * the following filters disqualify for fixed counters:
238  *  - inv
239  *  - edge
240  *  - cnt-mask
241  *  - in_tx
242  *  - in_tx_checkpointed
243  *  The other filters are supported by fixed counters.
244  *  The any-thread option is supported starting with v3.
245  */
246 #define FIXED_EVENT_FLAGS (X86_RAW_EVENT_MASK|HSW_IN_TX|HSW_IN_TX_CHECKPOINTED)
247 #define FIXED_EVENT_CONSTRAINT(c, n)    \
248         EVENT_CONSTRAINT(c, (1ULL << (32+n)), FIXED_EVENT_FLAGS)
249 
250 /*
251  * Constraint on the Event code + UMask
252  */
253 #define INTEL_UEVENT_CONSTRAINT(c, n)   \
254         EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)
255 
256 /* Like UEVENT_CONSTRAINT, but match flags too */
257 #define INTEL_FLAGS_UEVENT_CONSTRAINT(c, n)     \
258         EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS)
259 
260 #define INTEL_PLD_CONSTRAINT(c, n)      \
261         __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
262                            HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LDLAT)
263 
264 #define INTEL_PST_CONSTRAINT(c, n)      \
265         __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
266                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST)
267 
268 /* Event constraint, but match on all event flags too. */
269 #define INTEL_FLAGS_EVENT_CONSTRAINT(c, n) \
270         EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS)
271 
272 /* Check only flags, but allow all event/umask */
273 #define INTEL_ALL_EVENT_CONSTRAINT(code, n)     \
274         EVENT_CONSTRAINT(code, n, X86_ALL_EVENT_FLAGS)
275 
276 /* Check flags and event code, and set the HSW store flag */
277 #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_ST(code, n) \
278         __EVENT_CONSTRAINT(code, n,                     \
279                           ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
280                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
281 
282 /* Check flags and event code, and set the HSW load flag */
283 #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(code, n) \
284         __EVENT_CONSTRAINT(code, n,                     \
285                           ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
286                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
287 
288 /* Check flags and event code/umask, and set the HSW store flag */
289 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(code, n) \
290         __EVENT_CONSTRAINT(code, n,                     \
291                           INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
292                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
293 
294 /* Check flags and event code/umask, and set the HSW load flag */
295 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(code, n) \
296         __EVENT_CONSTRAINT(code, n,                     \
297                           INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
298                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
299 
300 /* Check flags and event code/umask, and set the HSW N/A flag */
301 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(code, n) \
302         __EVENT_CONSTRAINT(code, n,                     \
303                           INTEL_ARCH_EVENT_MASK|INTEL_ARCH_EVENT_MASK, \
304                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_NA_HSW)
305 
306 
307 /*
308  * We define the end marker as having a weight of -1
309  * to enable blacklisting of events using a counter bitmask
310  * of zero and thus a weight of zero.
311  * The end marker has a weight that cannot possibly be
312  * obtained from counting the bits in the bitmask.
313  */
314 #define EVENT_CONSTRAINT_END { .weight = -1 }
315 
316 /*
317  * Check for end marker with weight == -1
318  */
319 #define for_each_event_constraint(e, c) \
320         for ((e) = (c); (e)->weight != -1; (e)++)
321 
322 /*
323  * Extra registers for specific events.
324  *
325  * Some events need large masks and require external MSRs.
326  * Those extra MSRs end up being shared for all events on
327  * a PMU and sometimes between PMU of sibling HT threads.
328  * In either case, the kernel needs to handle conflicting
329  * accesses to those extra, shared, regs. The data structure
330  * to manage those registers is stored in cpu_hw_event.
331  */
332 struct extra_reg {
333         unsigned int            event;
334         unsigned int            msr;
335         u64                     config_mask;
336         u64                     valid_mask;
337         int                     idx;  /* per_xxx->regs[] reg index */
338         bool                    extra_msr_access;
339 };
340 
341 #define EVENT_EXTRA_REG(e, ms, m, vm, i) {      \
342         .event = (e),                   \
343         .msr = (ms),                    \
344         .config_mask = (m),             \
345         .valid_mask = (vm),             \
346         .idx = EXTRA_REG_##i,           \
347         .extra_msr_access = true,       \
348         }
349 
350 #define INTEL_EVENT_EXTRA_REG(event, msr, vm, idx)      \
351         EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm, idx)
352 
353 #define INTEL_UEVENT_EXTRA_REG(event, msr, vm, idx) \
354         EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT | \
355                         ARCH_PERFMON_EVENTSEL_UMASK, vm, idx)
356 
357 #define INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(c) \
358         INTEL_UEVENT_EXTRA_REG(c, \
359                                MSR_PEBS_LD_LAT_THRESHOLD, \
360                                0xffff, \
361                                LDLAT)
362 
363 #define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0, RSP_0)
364 
365 union perf_capabilities {
366         struct {
367                 u64     lbr_format:6;
368                 u64     pebs_trap:1;
369                 u64     pebs_arch_reg:1;
370                 u64     pebs_format:4;
371                 u64     smm_freeze:1;
372                 /*
373                  * PMU supports separate counter range for writing
374                  * values > 32bit.
375                  */
376                 u64     full_width_write:1;
377         };
378         u64     capabilities;
379 };
380 
381 struct x86_pmu_quirk {
382         struct x86_pmu_quirk *next;
383         void (*func)(void);
384 };
385 
386 union x86_pmu_config {
387         struct {
388                 u64 event:8,
389                     umask:8,
390                     usr:1,
391                     os:1,
392                     edge:1,
393                     pc:1,
394                     interrupt:1,
395                     __reserved1:1,
396                     en:1,
397                     inv:1,
398                     cmask:8,
399                     event2:4,
400                     __reserved2:4,
401                     go:1,
402                     ho:1;
403         } bits;
404         u64 value;
405 };
406 
407 #define X86_CONFIG(args...) ((union x86_pmu_config){.bits = {args}}).value
408 
409 /*
410  * struct x86_pmu - generic x86 pmu
411  */
412 struct x86_pmu {
413         /*
414          * Generic x86 PMC bits
415          */
416         const char      *name;
417         int             version;
418         int             (*handle_irq)(struct pt_regs *);
419         void            (*disable_all)(void);
420         void            (*enable_all)(int added);
421         void            (*enable)(struct perf_event *);
422         void            (*disable)(struct perf_event *);
423         int             (*hw_config)(struct perf_event *event);
424         int             (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
425         unsigned        eventsel;
426         unsigned        perfctr;
427         int             (*addr_offset)(int index, bool eventsel);
428         int             (*rdpmc_index)(int index);
429         u64             (*event_map)(int);
430         int             max_events;
431         int             num_counters;
432         int             num_counters_fixed;
433         int             cntval_bits;
434         u64             cntval_mask;
435         union {
436                         unsigned long events_maskl;
437                         unsigned long events_mask[BITS_TO_LONGS(ARCH_PERFMON_EVENTS_COUNT)];
438         };
439         int             events_mask_len;
440         int             apic;
441         u64             max_period;
442         struct event_constraint *
443                         (*get_event_constraints)(struct cpu_hw_events *cpuc,
444                                                  struct perf_event *event);
445 
446         void            (*put_event_constraints)(struct cpu_hw_events *cpuc,
447                                                  struct perf_event *event);
448         struct event_constraint *event_constraints;
449         struct x86_pmu_quirk *quirks;
450         int             perfctr_second_write;
451         bool            late_ack;
452 
453         /*
454          * sysfs attrs
455          */
456         int             attr_rdpmc_broken;
457         int             attr_rdpmc;
458         struct attribute **format_attrs;
459         struct attribute **event_attrs;
460 
461         ssize_t         (*events_sysfs_show)(char *page, u64 config);
462         struct attribute **cpu_events;
463 
464         /*
465          * CPU Hotplug hooks
466          */
467         int             (*cpu_prepare)(int cpu);
468         void            (*cpu_starting)(int cpu);
469         void            (*cpu_dying)(int cpu);
470         void            (*cpu_dead)(int cpu);
471 
472         void            (*check_microcode)(void);
473         void            (*flush_branch_stack)(void);
474 
475         /*
476          * Intel Arch Perfmon v2+
477          */
478         u64                     intel_ctrl;
479         union perf_capabilities intel_cap;
480 
481         /*
482          * Intel DebugStore bits
483          */
484         unsigned int    bts             :1,
485                         bts_active      :1,
486                         pebs            :1,
487                         pebs_active     :1,
488                         pebs_broken     :1;
489         int             pebs_record_size;
490         void            (*drain_pebs)(struct pt_regs *regs);
491         struct event_constraint *pebs_constraints;
492         void            (*pebs_aliases)(struct perf_event *event);
493         int             max_pebs_events;
494 
495         /*
496          * Intel LBR
497          */
498         unsigned long   lbr_tos, lbr_from, lbr_to; /* MSR base regs       */
499         int             lbr_nr;                    /* hardware stack size */
500         u64             lbr_sel_mask;              /* LBR_SELECT valid bits */
501         const int       *lbr_sel_map;              /* lbr_select mappings */
502         bool            lbr_double_abort;          /* duplicated lbr aborts */
503 
504         /*
505          * Extra registers for events
506          */
507         struct extra_reg *extra_regs;
508         unsigned int er_flags;
509 
510         /*
511          * Intel host/guest support (KVM)
512          */
513         struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr);
514 };
515 
516 #define x86_add_quirk(func_)                                            \
517 do {                                                                    \
518         static struct x86_pmu_quirk __quirk __initdata = {              \
519                 .func = func_,                                          \
520         };                                                              \
521         __quirk.next = x86_pmu.quirks;                                  \
522         x86_pmu.quirks = &__quirk;                                      \
523 } while (0)
524 
525 #define ERF_NO_HT_SHARING       1
526 #define ERF_HAS_RSP_1           2
527 
528 #define EVENT_VAR(_id)  event_attr_##_id
529 #define EVENT_PTR(_id) &event_attr_##_id.attr.attr
530 
531 #define EVENT_ATTR(_name, _id)                                          \
532 static struct perf_pmu_events_attr EVENT_VAR(_id) = {                   \
533         .attr           = __ATTR(_name, 0444, events_sysfs_show, NULL), \
534         .id             = PERF_COUNT_HW_##_id,                          \
535         .event_str      = NULL,                                         \
536 };
537 
538 #define EVENT_ATTR_STR(_name, v, str)                                   \
539 static struct perf_pmu_events_attr event_attr_##v = {                   \
540         .attr           = __ATTR(_name, 0444, events_sysfs_show, NULL), \
541         .id             = 0,                                            \
542         .event_str      = str,                                          \
543 };
544 
545 extern struct x86_pmu x86_pmu __read_mostly;
546 
547 DECLARE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
548 
549 int x86_perf_event_set_period(struct perf_event *event);
550 
551 /*
552  * Generalized hw caching related hw_event table, filled
553  * in on a per model basis. A value of 0 means
554  * 'not supported', -1 means 'hw_event makes no sense on
555  * this CPU', any other value means the raw hw_event
556  * ID.
557  */
558 
559 #define C(x) PERF_COUNT_HW_CACHE_##x
560 
561 extern u64 __read_mostly hw_cache_event_ids
562                                 [PERF_COUNT_HW_CACHE_MAX]
563                                 [PERF_COUNT_HW_CACHE_OP_MAX]
564                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
565 extern u64 __read_mostly hw_cache_extra_regs
566                                 [PERF_COUNT_HW_CACHE_MAX]
567                                 [PERF_COUNT_HW_CACHE_OP_MAX]
568                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
569 
570 u64 x86_perf_event_update(struct perf_event *event);
571 
572 static inline unsigned int x86_pmu_config_addr(int index)
573 {
574         return x86_pmu.eventsel + (x86_pmu.addr_offset ?
575                                    x86_pmu.addr_offset(index, true) : index);
576 }
577 
578 static inline unsigned int x86_pmu_event_addr(int index)
579 {
580         return x86_pmu.perfctr + (x86_pmu.addr_offset ?
581                                   x86_pmu.addr_offset(index, false) : index);
582 }
583 
584 static inline int x86_pmu_rdpmc_index(int index)
585 {
586         return x86_pmu.rdpmc_index ? x86_pmu.rdpmc_index(index) : index;
587 }
588 
589 int x86_setup_perfctr(struct perf_event *event);
590 
591 int x86_pmu_hw_config(struct perf_event *event);
592 
593 void x86_pmu_disable_all(void);
594 
595 static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
596                                           u64 enable_mask)
597 {
598         u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);
599 
600         if (hwc->extra_reg.reg)
601                 wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config);
602         wrmsrl(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask);
603 }
604 
605 void x86_pmu_enable_all(int added);
606 
607 int perf_assign_events(struct perf_event **events, int n,
608                         int wmin, int wmax, int *assign);
609 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign);
610 
611 void x86_pmu_stop(struct perf_event *event, int flags);
612 
613 static inline void x86_pmu_disable_event(struct perf_event *event)
614 {
615         struct hw_perf_event *hwc = &event->hw;
616 
617         wrmsrl(hwc->config_base, hwc->config);
618 }
619 
620 void x86_pmu_enable_event(struct perf_event *event);
621 
622 int x86_pmu_handle_irq(struct pt_regs *regs);
623 
624 extern struct event_constraint emptyconstraint;
625 
626 extern struct event_constraint unconstrained;
627 
628 static inline bool kernel_ip(unsigned long ip)
629 {
630 #ifdef CONFIG_X86_32
631         return ip > PAGE_OFFSET;
632 #else
633         return (long)ip < 0;
634 #endif
635 }
636 
637 /*
638  * Not all PMUs provide the right context information to place the reported IP
639  * into full context. Specifically segment registers are typically not
640  * supplied.
641  *
642  * Assuming the address is a linear address (it is for IBS), we fake the CS and
643  * vm86 mode using the known zero-based code segment and 'fix up' the registers
644  * to reflect this.
645  *
646  * Intel PEBS/LBR appear to typically provide the effective address, nothing
647  * much we can do about that but pray and treat it like a linear address.
648  */
649 static inline void set_linear_ip(struct pt_regs *regs, unsigned long ip)
650 {
651         regs->cs = kernel_ip(ip) ? __KERNEL_CS : __USER_CS;
652         if (regs->flags & X86_VM_MASK)
653                 regs->flags ^= (PERF_EFLAGS_VM | X86_VM_MASK);
654         regs->ip = ip;
655 }
656 
657 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event);
658 ssize_t intel_event_sysfs_show(char *page, u64 config);
659 
660 #ifdef CONFIG_CPU_SUP_AMD
661 
662 int amd_pmu_init(void);
663 
664 #else /* CONFIG_CPU_SUP_AMD */
665 
666 static inline int amd_pmu_init(void)
667 {
668         return 0;
669 }
670 
671 #endif /* CONFIG_CPU_SUP_AMD */
672 
673 #ifdef CONFIG_CPU_SUP_INTEL
674 
675 int intel_pmu_save_and_restart(struct perf_event *event);
676 
677 struct event_constraint *
678 x86_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event);
679 
680 struct intel_shared_regs *allocate_shared_regs(int cpu);
681 
682 int intel_pmu_init(void);
683 
684 void init_debug_store_on_cpu(int cpu);
685 
686 void fini_debug_store_on_cpu(int cpu);
687 
688 void release_ds_buffers(void);
689 
690 void reserve_ds_buffers(void);
691 
692 extern struct event_constraint bts_constraint;
693 
694 void intel_pmu_enable_bts(u64 config);
695 
696 void intel_pmu_disable_bts(void);
697 
698 int intel_pmu_drain_bts_buffer(void);
699 
700 extern struct event_constraint intel_core2_pebs_event_constraints[];
701 
702 extern struct event_constraint intel_atom_pebs_event_constraints[];
703 
704 extern struct event_constraint intel_slm_pebs_event_constraints[];
705 
706 extern struct event_constraint intel_nehalem_pebs_event_constraints[];
707 
708 extern struct event_constraint intel_westmere_pebs_event_constraints[];
709 
710 extern struct event_constraint intel_snb_pebs_event_constraints[];
711 
712 extern struct event_constraint intel_ivb_pebs_event_constraints[];
713 
714 extern struct event_constraint intel_hsw_pebs_event_constraints[];
715 
716 struct event_constraint *intel_pebs_constraints(struct perf_event *event);
717 
718 void intel_pmu_pebs_enable(struct perf_event *event);
719 
720 void intel_pmu_pebs_disable(struct perf_event *event);
721 
722 void intel_pmu_pebs_enable_all(void);
723 
724 void intel_pmu_pebs_disable_all(void);
725 
726 void intel_ds_init(void);
727 
728 void intel_pmu_lbr_reset(void);
729 
730 void intel_pmu_lbr_enable(struct perf_event *event);
731 
732 void intel_pmu_lbr_disable(struct perf_event *event);
733 
734 void intel_pmu_lbr_enable_all(void);
735 
736 void intel_pmu_lbr_disable_all(void);
737 
738 void intel_pmu_lbr_read(void);
739 
740 void intel_pmu_lbr_init_core(void);
741 
742 void intel_pmu_lbr_init_nhm(void);
743 
744 void intel_pmu_lbr_init_atom(void);
745 
746 void intel_pmu_lbr_init_snb(void);
747 
748 int intel_pmu_setup_lbr_filter(struct perf_event *event);
749 
750 int p4_pmu_init(void);
751 
752 int p6_pmu_init(void);
753 
754 int knc_pmu_init(void);
755 
756 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr,
757                           char *page);
758 
759 #else /* CONFIG_CPU_SUP_INTEL */
760 
761 static inline void reserve_ds_buffers(void)
762 {
763 }
764 
765 static inline void release_ds_buffers(void)
766 {
767 }
768 
769 static inline int intel_pmu_init(void)
770 {
771         return 0;
772 }
773 
774 static inline struct intel_shared_regs *allocate_shared_regs(int cpu)
775 {
776         return NULL;
777 }
778 
779 #endif /* CONFIG_CPU_SUP_INTEL */
780 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp