~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/oprofile/op_model_amd.c

Version: ~ [ linux-5.8 ] ~ [ linux-5.7.12 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.55 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.136 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.191 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.232 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.232 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * @file op_model_amd.c
  3  * athlon / K7 / K8 / Family 10h model-specific MSR operations
  4  *
  5  * @remark Copyright 2002-2009 OProfile authors
  6  * @remark Read the file COPYING
  7  *
  8  * @author John Levon
  9  * @author Philippe Elie
 10  * @author Graydon Hoare
 11  * @author Robert Richter <robert.richter@amd.com>
 12  * @author Barry Kasindorf <barry.kasindorf@amd.com>
 13  * @author Jason Yeh <jason.yeh@amd.com>
 14  * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
 15  */
 16 
 17 #include <linux/oprofile.h>
 18 #include <linux/device.h>
 19 #include <linux/pci.h>
 20 #include <linux/percpu.h>
 21 
 22 #include <asm/ptrace.h>
 23 #include <asm/msr.h>
 24 #include <asm/nmi.h>
 25 #include <asm/apic.h>
 26 #include <asm/processor.h>
 27 
 28 #include "op_x86_model.h"
 29 #include "op_counter.h"
 30 
 31 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 32 #define NUM_VIRT_COUNTERS       32
 33 #else
 34 #define NUM_VIRT_COUNTERS       0
 35 #endif
 36 
 37 #define OP_EVENT_MASK                   0x0FFF
 38 #define OP_CTR_OVERFLOW                 (1ULL<<31)
 39 
 40 #define MSR_AMD_EVENTSEL_RESERVED       ((0xFFFFFCF0ULL<<32)|(1ULL<<21))
 41 
 42 static int num_counters;
 43 static unsigned long reset_value[OP_MAX_COUNTER];
 44 
 45 #define IBS_FETCH_SIZE                  6
 46 #define IBS_OP_SIZE                     12
 47 
 48 static u32 ibs_caps;
 49 
 50 struct ibs_config {
 51         unsigned long op_enabled;
 52         unsigned long fetch_enabled;
 53         unsigned long max_cnt_fetch;
 54         unsigned long max_cnt_op;
 55         unsigned long rand_en;
 56         unsigned long dispatched_ops;
 57         unsigned long branch_target;
 58 };
 59 
 60 struct ibs_state {
 61         u64             ibs_op_ctl;
 62         int             branch_target;
 63         unsigned long   sample_size;
 64 };
 65 
 66 static struct ibs_config ibs_config;
 67 static struct ibs_state ibs_state;
 68 
 69 /*
 70  * IBS randomization macros
 71  */
 72 #define IBS_RANDOM_BITS                 12
 73 #define IBS_RANDOM_MASK                 ((1ULL << IBS_RANDOM_BITS) - 1)
 74 #define IBS_RANDOM_MAXCNT_OFFSET        (1ULL << (IBS_RANDOM_BITS - 5))
 75 
 76 /*
 77  * 16-bit Linear Feedback Shift Register (LFSR)
 78  *
 79  *                       16   14   13    11
 80  * Feedback polynomial = X  + X  + X  +  X  + 1
 81  */
 82 static unsigned int lfsr_random(void)
 83 {
 84         static unsigned int lfsr_value = 0xF00D;
 85         unsigned int bit;
 86 
 87         /* Compute next bit to shift in */
 88         bit = ((lfsr_value >> 0) ^
 89                (lfsr_value >> 2) ^
 90                (lfsr_value >> 3) ^
 91                (lfsr_value >> 5)) & 0x0001;
 92 
 93         /* Advance to next register value */
 94         lfsr_value = (lfsr_value >> 1) | (bit << 15);
 95 
 96         return lfsr_value;
 97 }
 98 
 99 /*
100  * IBS software randomization
101  *
102  * The IBS periodic op counter is randomized in software. The lower 12
103  * bits of the 20 bit counter are randomized. IbsOpCurCnt is
104  * initialized with a 12 bit random value.
105  */
106 static inline u64 op_amd_randomize_ibs_op(u64 val)
107 {
108         unsigned int random = lfsr_random();
109 
110         if (!(ibs_caps & IBS_CAPS_RDWROPCNT))
111                 /*
112                  * Work around if the hw can not write to IbsOpCurCnt
113                  *
114                  * Randomize the lower 8 bits of the 16 bit
115                  * IbsOpMaxCnt [15:0] value in the range of -128 to
116                  * +127 by adding/subtracting an offset to the
117                  * maximum count (IbsOpMaxCnt).
118                  *
119                  * To avoid over or underflows and protect upper bits
120                  * starting at bit 16, the initial value for
121                  * IbsOpMaxCnt must fit in the range from 0x0081 to
122                  * 0xff80.
123                  */
124                 val += (s8)(random >> 4);
125         else
126                 val |= (u64)(random & IBS_RANDOM_MASK) << 32;
127 
128         return val;
129 }
130 
131 static inline void
132 op_amd_handle_ibs(struct pt_regs * const regs,
133                   struct op_msrs const * const msrs)
134 {
135         u64 val, ctl;
136         struct op_entry entry;
137 
138         if (!ibs_caps)
139                 return;
140 
141         if (ibs_config.fetch_enabled) {
142                 rdmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
143                 if (ctl & IBS_FETCH_VAL) {
144                         rdmsrl(MSR_AMD64_IBSFETCHLINAD, val);
145                         oprofile_write_reserve(&entry, regs, val,
146                                                IBS_FETCH_CODE, IBS_FETCH_SIZE);
147                         oprofile_add_data64(&entry, val);
148                         oprofile_add_data64(&entry, ctl);
149                         rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, val);
150                         oprofile_add_data64(&entry, val);
151                         oprofile_write_commit(&entry);
152 
153                         /* reenable the IRQ */
154                         ctl &= ~(IBS_FETCH_VAL | IBS_FETCH_CNT);
155                         ctl |= IBS_FETCH_ENABLE;
156                         wrmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
157                 }
158         }
159 
160         if (ibs_config.op_enabled) {
161                 rdmsrl(MSR_AMD64_IBSOPCTL, ctl);
162                 if (ctl & IBS_OP_VAL) {
163                         rdmsrl(MSR_AMD64_IBSOPRIP, val);
164                         oprofile_write_reserve(&entry, regs, val, IBS_OP_CODE,
165                                                ibs_state.sample_size);
166                         oprofile_add_data64(&entry, val);
167                         rdmsrl(MSR_AMD64_IBSOPDATA, val);
168                         oprofile_add_data64(&entry, val);
169                         rdmsrl(MSR_AMD64_IBSOPDATA2, val);
170                         oprofile_add_data64(&entry, val);
171                         rdmsrl(MSR_AMD64_IBSOPDATA3, val);
172                         oprofile_add_data64(&entry, val);
173                         rdmsrl(MSR_AMD64_IBSDCLINAD, val);
174                         oprofile_add_data64(&entry, val);
175                         rdmsrl(MSR_AMD64_IBSDCPHYSAD, val);
176                         oprofile_add_data64(&entry, val);
177                         if (ibs_state.branch_target) {
178                                 rdmsrl(MSR_AMD64_IBSBRTARGET, val);
179                                 oprofile_add_data(&entry, (unsigned long)val);
180                         }
181                         oprofile_write_commit(&entry);
182 
183                         /* reenable the IRQ */
184                         ctl = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl);
185                         wrmsrl(MSR_AMD64_IBSOPCTL, ctl);
186                 }
187         }
188 }
189 
190 static inline void op_amd_start_ibs(void)
191 {
192         u64 val;
193 
194         if (!ibs_caps)
195                 return;
196 
197         memset(&ibs_state, 0, sizeof(ibs_state));
198 
199         /*
200          * Note: Since the max count settings may out of range we
201          * write back the actual used values so that userland can read
202          * it.
203          */
204 
205         if (ibs_config.fetch_enabled) {
206                 val = ibs_config.max_cnt_fetch >> 4;
207                 val = min(val, IBS_FETCH_MAX_CNT);
208                 ibs_config.max_cnt_fetch = val << 4;
209                 val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0;
210                 val |= IBS_FETCH_ENABLE;
211                 wrmsrl(MSR_AMD64_IBSFETCHCTL, val);
212         }
213 
214         if (ibs_config.op_enabled) {
215                 val = ibs_config.max_cnt_op >> 4;
216                 if (!(ibs_caps & IBS_CAPS_RDWROPCNT)) {
217                         /*
218                          * IbsOpCurCnt not supported.  See
219                          * op_amd_randomize_ibs_op() for details.
220                          */
221                         val = clamp(val, 0x0081ULL, 0xFF80ULL);
222                         ibs_config.max_cnt_op = val << 4;
223                 } else {
224                         /*
225                          * The start value is randomized with a
226                          * positive offset, we need to compensate it
227                          * with the half of the randomized range. Also
228                          * avoid underflows.
229                          */
230                         val += IBS_RANDOM_MAXCNT_OFFSET;
231                         if (ibs_caps & IBS_CAPS_OPCNTEXT)
232                                 val = min(val, IBS_OP_MAX_CNT_EXT);
233                         else
234                                 val = min(val, IBS_OP_MAX_CNT);
235                         ibs_config.max_cnt_op =
236                                 (val - IBS_RANDOM_MAXCNT_OFFSET) << 4;
237                 }
238                 val = ((val & ~IBS_OP_MAX_CNT) << 4) | (val & IBS_OP_MAX_CNT);
239                 val |= ibs_config.dispatched_ops ? IBS_OP_CNT_CTL : 0;
240                 val |= IBS_OP_ENABLE;
241                 ibs_state.ibs_op_ctl = val;
242                 ibs_state.sample_size = IBS_OP_SIZE;
243                 if (ibs_config.branch_target) {
244                         ibs_state.branch_target = 1;
245                         ibs_state.sample_size++;
246                 }
247                 val = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl);
248                 wrmsrl(MSR_AMD64_IBSOPCTL, val);
249         }
250 }
251 
252 static void op_amd_stop_ibs(void)
253 {
254         if (!ibs_caps)
255                 return;
256 
257         if (ibs_config.fetch_enabled)
258                 /* clear max count and enable */
259                 wrmsrl(MSR_AMD64_IBSFETCHCTL, 0);
260 
261         if (ibs_config.op_enabled)
262                 /* clear max count and enable */
263                 wrmsrl(MSR_AMD64_IBSOPCTL, 0);
264 }
265 
266 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
267 
268 static void op_mux_switch_ctrl(struct op_x86_model_spec const *model,
269                                struct op_msrs const * const msrs)
270 {
271         u64 val;
272         int i;
273 
274         /* enable active counters */
275         for (i = 0; i < num_counters; ++i) {
276                 int virt = op_x86_phys_to_virt(i);
277                 if (!reset_value[virt])
278                         continue;
279                 rdmsrl(msrs->controls[i].addr, val);
280                 val &= model->reserved;
281                 val |= op_x86_get_ctrl(model, &counter_config[virt]);
282                 wrmsrl(msrs->controls[i].addr, val);
283         }
284 }
285 
286 #endif
287 
288 /* functions for op_amd_spec */
289 
290 static void op_amd_shutdown(struct op_msrs const * const msrs)
291 {
292         int i;
293 
294         for (i = 0; i < num_counters; ++i) {
295                 if (!msrs->counters[i].addr)
296                         continue;
297                 release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
298                 release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
299         }
300 }
301 
302 static int op_amd_fill_in_addresses(struct op_msrs * const msrs)
303 {
304         int i;
305 
306         for (i = 0; i < num_counters; i++) {
307                 if (!reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
308                         goto fail;
309                 if (!reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i)) {
310                         release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
311                         goto fail;
312                 }
313                 /* both registers must be reserved */
314                 if (num_counters == AMD64_NUM_COUNTERS_CORE) {
315                         msrs->counters[i].addr = MSR_F15H_PERF_CTR + (i << 1);
316                         msrs->controls[i].addr = MSR_F15H_PERF_CTL + (i << 1);
317                 } else {
318                         msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
319                         msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
320                 }
321                 continue;
322         fail:
323                 if (!counter_config[i].enabled)
324                         continue;
325                 op_x86_warn_reserved(i);
326                 op_amd_shutdown(msrs);
327                 return -EBUSY;
328         }
329 
330         return 0;
331 }
332 
333 static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
334                               struct op_msrs const * const msrs)
335 {
336         u64 val;
337         int i;
338 
339         /* setup reset_value */
340         for (i = 0; i < OP_MAX_COUNTER; ++i) {
341                 if (counter_config[i].enabled
342                     && msrs->counters[op_x86_virt_to_phys(i)].addr)
343                         reset_value[i] = counter_config[i].count;
344                 else
345                         reset_value[i] = 0;
346         }
347 
348         /* clear all counters */
349         for (i = 0; i < num_counters; ++i) {
350                 if (!msrs->controls[i].addr)
351                         continue;
352                 rdmsrl(msrs->controls[i].addr, val);
353                 if (val & ARCH_PERFMON_EVENTSEL_ENABLE)
354                         op_x86_warn_in_use(i);
355                 val &= model->reserved;
356                 wrmsrl(msrs->controls[i].addr, val);
357                 /*
358                  * avoid a false detection of ctr overflows in NMI
359                  * handler
360                  */
361                 wrmsrl(msrs->counters[i].addr, -1LL);
362         }
363 
364         /* enable active counters */
365         for (i = 0; i < num_counters; ++i) {
366                 int virt = op_x86_phys_to_virt(i);
367                 if (!reset_value[virt])
368                         continue;
369 
370                 /* setup counter registers */
371                 wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
372 
373                 /* setup control registers */
374                 rdmsrl(msrs->controls[i].addr, val);
375                 val &= model->reserved;
376                 val |= op_x86_get_ctrl(model, &counter_config[virt]);
377                 wrmsrl(msrs->controls[i].addr, val);
378         }
379 }
380 
381 static int op_amd_check_ctrs(struct pt_regs * const regs,
382                              struct op_msrs const * const msrs)
383 {
384         u64 val;
385         int i;
386 
387         for (i = 0; i < num_counters; ++i) {
388                 int virt = op_x86_phys_to_virt(i);
389                 if (!reset_value[virt])
390                         continue;
391                 rdmsrl(msrs->counters[i].addr, val);
392                 /* bit is clear if overflowed: */
393                 if (val & OP_CTR_OVERFLOW)
394                         continue;
395                 oprofile_add_sample(regs, virt);
396                 wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
397         }
398 
399         op_amd_handle_ibs(regs, msrs);
400 
401         /* See op_model_ppro.c */
402         return 1;
403 }
404 
405 static void op_amd_start(struct op_msrs const * const msrs)
406 {
407         u64 val;
408         int i;
409 
410         for (i = 0; i < num_counters; ++i) {
411                 if (!reset_value[op_x86_phys_to_virt(i)])
412                         continue;
413                 rdmsrl(msrs->controls[i].addr, val);
414                 val |= ARCH_PERFMON_EVENTSEL_ENABLE;
415                 wrmsrl(msrs->controls[i].addr, val);
416         }
417 
418         op_amd_start_ibs();
419 }
420 
421 static void op_amd_stop(struct op_msrs const * const msrs)
422 {
423         u64 val;
424         int i;
425 
426         /*
427          * Subtle: stop on all counters to avoid race with setting our
428          * pm callback
429          */
430         for (i = 0; i < num_counters; ++i) {
431                 if (!reset_value[op_x86_phys_to_virt(i)])
432                         continue;
433                 rdmsrl(msrs->controls[i].addr, val);
434                 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
435                 wrmsrl(msrs->controls[i].addr, val);
436         }
437 
438         op_amd_stop_ibs();
439 }
440 
441 /*
442  * check and reserve APIC extended interrupt LVT offset for IBS if
443  * available
444  */
445 
446 static void init_ibs(void)
447 {
448         ibs_caps = get_ibs_caps();
449 
450         if (!ibs_caps)
451                 return;
452 
453         printk(KERN_INFO "oprofile: AMD IBS detected (0x%08x)\n", ibs_caps);
454 }
455 
456 static int (*create_arch_files)(struct dentry *root);
457 
458 static int setup_ibs_files(struct dentry *root)
459 {
460         struct dentry *dir;
461         int ret = 0;
462 
463         /* architecture specific files */
464         if (create_arch_files)
465                 ret = create_arch_files(root);
466 
467         if (ret)
468                 return ret;
469 
470         if (!ibs_caps)
471                 return ret;
472 
473         /* model specific files */
474 
475         /* setup some reasonable defaults */
476         memset(&ibs_config, 0, sizeof(ibs_config));
477         ibs_config.max_cnt_fetch = 250000;
478         ibs_config.max_cnt_op = 250000;
479 
480         if (ibs_caps & IBS_CAPS_FETCHSAM) {
481                 dir = oprofilefs_mkdir(root, "ibs_fetch");
482                 oprofilefs_create_ulong(dir, "enable",
483                                         &ibs_config.fetch_enabled);
484                 oprofilefs_create_ulong(dir, "max_count",
485                                         &ibs_config.max_cnt_fetch);
486                 oprofilefs_create_ulong(dir, "rand_enable",
487                                         &ibs_config.rand_en);
488         }
489 
490         if (ibs_caps & IBS_CAPS_OPSAM) {
491                 dir = oprofilefs_mkdir(root, "ibs_op");
492                 oprofilefs_create_ulong(dir, "enable",
493                                         &ibs_config.op_enabled);
494                 oprofilefs_create_ulong(dir, "max_count",
495                                         &ibs_config.max_cnt_op);
496                 if (ibs_caps & IBS_CAPS_OPCNT)
497                         oprofilefs_create_ulong(dir, "dispatched_ops",
498                                                 &ibs_config.dispatched_ops);
499                 if (ibs_caps & IBS_CAPS_BRNTRGT)
500                         oprofilefs_create_ulong(dir, "branch_target",
501                                                 &ibs_config.branch_target);
502         }
503 
504         return 0;
505 }
506 
507 struct op_x86_model_spec op_amd_spec;
508 
509 static int op_amd_init(struct oprofile_operations *ops)
510 {
511         init_ibs();
512         create_arch_files = ops->create_files;
513         ops->create_files = setup_ibs_files;
514 
515         if (boot_cpu_data.x86 == 0x15) {
516                 num_counters = AMD64_NUM_COUNTERS_CORE;
517         } else {
518                 num_counters = AMD64_NUM_COUNTERS;
519         }
520 
521         op_amd_spec.num_counters = num_counters;
522         op_amd_spec.num_controls = num_counters;
523         op_amd_spec.num_virt_counters = max(num_counters, NUM_VIRT_COUNTERS);
524 
525         return 0;
526 }
527 
528 struct op_x86_model_spec op_amd_spec = {
529         /* num_counters/num_controls filled in at runtime */
530         .reserved               = MSR_AMD_EVENTSEL_RESERVED,
531         .event_mask             = OP_EVENT_MASK,
532         .init                   = op_amd_init,
533         .fill_in_addresses      = &op_amd_fill_in_addresses,
534         .setup_ctrs             = &op_amd_setup_ctrs,
535         .check_ctrs             = &op_amd_check_ctrs,
536         .start                  = &op_amd_start,
537         .stop                   = &op_amd_stop,
538         .shutdown               = &op_amd_shutdown,
539 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
540         .switch_ctrl            = &op_mux_switch_ctrl,
541 #endif
542 };
543 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp