~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/events/intel/rapl.c

Version: ~ [ linux-5.12-rc1 ] ~ [ linux-5.11.2 ] ~ [ linux-5.10.19 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.101 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.177 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.222 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.258 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.258 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * perf_event_intel_rapl.c: support Intel RAPL energy consumption counters
  3  * Copyright (C) 2013 Google, Inc., Stephane Eranian
  4  *
  5  * Intel RAPL interface is specified in the IA-32 Manual Vol3b
  6  * section 14.7.1 (September 2013)
  7  *
  8  * RAPL provides more controls than just reporting energy consumption
  9  * however here we only expose the 3 energy consumption free running
 10  * counters (pp0, pkg, dram).
 11  *
 12  * Each of those counters increments in a power unit defined by the
 13  * RAPL_POWER_UNIT MSR. On SandyBridge, this unit is 1/(2^16) Joules
 14  * but it can vary.
 15  *
 16  * Counter to rapl events mappings:
 17  *
 18  *  pp0 counter: consumption of all physical cores (power plane 0)
 19  *        event: rapl_energy_cores
 20  *    perf code: 0x1
 21  *
 22  *  pkg counter: consumption of the whole processor package
 23  *        event: rapl_energy_pkg
 24  *    perf code: 0x2
 25  *
 26  * dram counter: consumption of the dram domain (servers only)
 27  *        event: rapl_energy_dram
 28  *    perf code: 0x3
 29  *
 30  * gpu counter: consumption of the builtin-gpu domain (client only)
 31  *        event: rapl_energy_gpu
 32  *    perf code: 0x4
 33  *
 34  *  psys counter: consumption of the builtin-psys domain (client only)
 35  *        event: rapl_energy_psys
 36  *    perf code: 0x5
 37  *
 38  * We manage those counters as free running (read-only). They may be
 39  * use simultaneously by other tools, such as turbostat.
 40  *
 41  * The events only support system-wide mode counting. There is no
 42  * sampling support because it does not make sense and is not
 43  * supported by the RAPL hardware.
 44  *
 45  * Because we want to avoid floating-point operations in the kernel,
 46  * the events are all reported in fixed point arithmetic (32.32).
 47  * Tools must adjust the counts to convert them to Watts using
 48  * the duration of the measurement. Tools may use a function such as
 49  * ldexp(raw_count, -32);
 50  */
 51 
 52 #define pr_fmt(fmt) "RAPL PMU: " fmt
 53 
 54 #include <linux/module.h>
 55 #include <linux/slab.h>
 56 #include <linux/perf_event.h>
 57 #include <asm/cpu_device_id.h>
 58 #include <asm/intel-family.h>
 59 #include "../perf_event.h"
 60 
 61 MODULE_LICENSE("GPL");
 62 
 63 /*
 64  * RAPL energy status counters
 65  */
 66 #define RAPL_IDX_PP0_NRG_STAT   0       /* all cores */
 67 #define INTEL_RAPL_PP0          0x1     /* pseudo-encoding */
 68 #define RAPL_IDX_PKG_NRG_STAT   1       /* entire package */
 69 #define INTEL_RAPL_PKG          0x2     /* pseudo-encoding */
 70 #define RAPL_IDX_RAM_NRG_STAT   2       /* DRAM */
 71 #define INTEL_RAPL_RAM          0x3     /* pseudo-encoding */
 72 #define RAPL_IDX_PP1_NRG_STAT   3       /* gpu */
 73 #define INTEL_RAPL_PP1          0x4     /* pseudo-encoding */
 74 #define RAPL_IDX_PSYS_NRG_STAT  4       /* psys */
 75 #define INTEL_RAPL_PSYS         0x5     /* pseudo-encoding */
 76 
 77 #define NR_RAPL_DOMAINS         0x5
 78 static const char *const rapl_domain_names[NR_RAPL_DOMAINS] __initconst = {
 79         "pp0-core",
 80         "package",
 81         "dram",
 82         "pp1-gpu",
 83         "psys",
 84 };
 85 
 86 /* Clients have PP0, PKG */
 87 #define RAPL_IDX_CLN    (1<<RAPL_IDX_PP0_NRG_STAT|\
 88                          1<<RAPL_IDX_PKG_NRG_STAT|\
 89                          1<<RAPL_IDX_PP1_NRG_STAT)
 90 
 91 /* Servers have PP0, PKG, RAM */
 92 #define RAPL_IDX_SRV    (1<<RAPL_IDX_PP0_NRG_STAT|\
 93                          1<<RAPL_IDX_PKG_NRG_STAT|\
 94                          1<<RAPL_IDX_RAM_NRG_STAT)
 95 
 96 /* Servers have PP0, PKG, RAM, PP1 */
 97 #define RAPL_IDX_HSW    (1<<RAPL_IDX_PP0_NRG_STAT|\
 98                          1<<RAPL_IDX_PKG_NRG_STAT|\
 99                          1<<RAPL_IDX_RAM_NRG_STAT|\
100                          1<<RAPL_IDX_PP1_NRG_STAT)
101 
102 /* SKL clients have PP0, PKG, RAM, PP1, PSYS */
103 #define RAPL_IDX_SKL_CLN (1<<RAPL_IDX_PP0_NRG_STAT|\
104                           1<<RAPL_IDX_PKG_NRG_STAT|\
105                           1<<RAPL_IDX_RAM_NRG_STAT|\
106                           1<<RAPL_IDX_PP1_NRG_STAT|\
107                           1<<RAPL_IDX_PSYS_NRG_STAT)
108 
109 /* Knights Landing has PKG, RAM */
110 #define RAPL_IDX_KNL    (1<<RAPL_IDX_PKG_NRG_STAT|\
111                          1<<RAPL_IDX_RAM_NRG_STAT)
112 
113 /*
114  * event code: LSB 8 bits, passed in attr->config
115  * any other bit is reserved
116  */
117 #define RAPL_EVENT_MASK 0xFFULL
118 
119 #define DEFINE_RAPL_FORMAT_ATTR(_var, _name, _format)           \
120 static ssize_t __rapl_##_var##_show(struct kobject *kobj,       \
121                                 struct kobj_attribute *attr,    \
122                                 char *page)                     \
123 {                                                               \
124         BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE);             \
125         return sprintf(page, _format "\n");                     \
126 }                                                               \
127 static struct kobj_attribute format_attr_##_var =               \
128         __ATTR(_name, 0444, __rapl_##_var##_show, NULL)
129 
130 #define RAPL_CNTR_WIDTH 32
131 
132 #define RAPL_EVENT_ATTR_STR(_name, v, str)                                      \
133 static struct perf_pmu_events_attr event_attr_##v = {                           \
134         .attr           = __ATTR(_name, 0444, perf_event_sysfs_show, NULL),     \
135         .id             = 0,                                                    \
136         .event_str      = str,                                                  \
137 };
138 
139 struct rapl_pmu {
140         raw_spinlock_t          lock;
141         int                     n_active;
142         int                     cpu;
143         struct list_head        active_list;
144         struct pmu              *pmu;
145         ktime_t                 timer_interval;
146         struct hrtimer          hrtimer;
147 };
148 
149 struct rapl_pmus {
150         struct pmu              pmu;
151         unsigned int            maxpkg;
152         struct rapl_pmu         *pmus[];
153 };
154 
155  /* 1/2^hw_unit Joule */
156 static int rapl_hw_unit[NR_RAPL_DOMAINS] __read_mostly;
157 static struct rapl_pmus *rapl_pmus;
158 static cpumask_t rapl_cpu_mask;
159 static unsigned int rapl_cntr_mask;
160 static u64 rapl_timer_ms;
161 
162 static inline struct rapl_pmu *cpu_to_rapl_pmu(unsigned int cpu)
163 {
164         return rapl_pmus->pmus[topology_logical_package_id(cpu)];
165 }
166 
167 static inline u64 rapl_read_counter(struct perf_event *event)
168 {
169         u64 raw;
170         rdmsrl(event->hw.event_base, raw);
171         return raw;
172 }
173 
174 static inline u64 rapl_scale(u64 v, int cfg)
175 {
176         if (cfg > NR_RAPL_DOMAINS) {
177                 pr_warn("Invalid domain %d, failed to scale data\n", cfg);
178                 return v;
179         }
180         /*
181          * scale delta to smallest unit (1/2^32)
182          * users must then scale back: count * 1/(1e9*2^32) to get Joules
183          * or use ldexp(count, -32).
184          * Watts = Joules/Time delta
185          */
186         return v << (32 - rapl_hw_unit[cfg - 1]);
187 }
188 
189 static u64 rapl_event_update(struct perf_event *event)
190 {
191         struct hw_perf_event *hwc = &event->hw;
192         u64 prev_raw_count, new_raw_count;
193         s64 delta, sdelta;
194         int shift = RAPL_CNTR_WIDTH;
195 
196 again:
197         prev_raw_count = local64_read(&hwc->prev_count);
198         rdmsrl(event->hw.event_base, new_raw_count);
199 
200         if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
201                             new_raw_count) != prev_raw_count) {
202                 cpu_relax();
203                 goto again;
204         }
205 
206         /*
207          * Now we have the new raw value and have updated the prev
208          * timestamp already. We can now calculate the elapsed delta
209          * (event-)time and add that to the generic event.
210          *
211          * Careful, not all hw sign-extends above the physical width
212          * of the count.
213          */
214         delta = (new_raw_count << shift) - (prev_raw_count << shift);
215         delta >>= shift;
216 
217         sdelta = rapl_scale(delta, event->hw.config);
218 
219         local64_add(sdelta, &event->count);
220 
221         return new_raw_count;
222 }
223 
224 static void rapl_start_hrtimer(struct rapl_pmu *pmu)
225 {
226        hrtimer_start(&pmu->hrtimer, pmu->timer_interval,
227                      HRTIMER_MODE_REL_PINNED);
228 }
229 
230 static enum hrtimer_restart rapl_hrtimer_handle(struct hrtimer *hrtimer)
231 {
232         struct rapl_pmu *pmu = container_of(hrtimer, struct rapl_pmu, hrtimer);
233         struct perf_event *event;
234         unsigned long flags;
235 
236         if (!pmu->n_active)
237                 return HRTIMER_NORESTART;
238 
239         raw_spin_lock_irqsave(&pmu->lock, flags);
240 
241         list_for_each_entry(event, &pmu->active_list, active_entry)
242                 rapl_event_update(event);
243 
244         raw_spin_unlock_irqrestore(&pmu->lock, flags);
245 
246         hrtimer_forward_now(hrtimer, pmu->timer_interval);
247 
248         return HRTIMER_RESTART;
249 }
250 
251 static void rapl_hrtimer_init(struct rapl_pmu *pmu)
252 {
253         struct hrtimer *hr = &pmu->hrtimer;
254 
255         hrtimer_init(hr, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
256         hr->function = rapl_hrtimer_handle;
257 }
258 
259 static void __rapl_pmu_event_start(struct rapl_pmu *pmu,
260                                    struct perf_event *event)
261 {
262         if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
263                 return;
264 
265         event->hw.state = 0;
266 
267         list_add_tail(&event->active_entry, &pmu->active_list);
268 
269         local64_set(&event->hw.prev_count, rapl_read_counter(event));
270 
271         pmu->n_active++;
272         if (pmu->n_active == 1)
273                 rapl_start_hrtimer(pmu);
274 }
275 
276 static void rapl_pmu_event_start(struct perf_event *event, int mode)
277 {
278         struct rapl_pmu *pmu = event->pmu_private;
279         unsigned long flags;
280 
281         raw_spin_lock_irqsave(&pmu->lock, flags);
282         __rapl_pmu_event_start(pmu, event);
283         raw_spin_unlock_irqrestore(&pmu->lock, flags);
284 }
285 
286 static void rapl_pmu_event_stop(struct perf_event *event, int mode)
287 {
288         struct rapl_pmu *pmu = event->pmu_private;
289         struct hw_perf_event *hwc = &event->hw;
290         unsigned long flags;
291 
292         raw_spin_lock_irqsave(&pmu->lock, flags);
293 
294         /* mark event as deactivated and stopped */
295         if (!(hwc->state & PERF_HES_STOPPED)) {
296                 WARN_ON_ONCE(pmu->n_active <= 0);
297                 pmu->n_active--;
298                 if (pmu->n_active == 0)
299                         hrtimer_cancel(&pmu->hrtimer);
300 
301                 list_del(&event->active_entry);
302 
303                 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
304                 hwc->state |= PERF_HES_STOPPED;
305         }
306 
307         /* check if update of sw counter is necessary */
308         if ((mode & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
309                 /*
310                  * Drain the remaining delta count out of a event
311                  * that we are disabling:
312                  */
313                 rapl_event_update(event);
314                 hwc->state |= PERF_HES_UPTODATE;
315         }
316 
317         raw_spin_unlock_irqrestore(&pmu->lock, flags);
318 }
319 
320 static int rapl_pmu_event_add(struct perf_event *event, int mode)
321 {
322         struct rapl_pmu *pmu = event->pmu_private;
323         struct hw_perf_event *hwc = &event->hw;
324         unsigned long flags;
325 
326         raw_spin_lock_irqsave(&pmu->lock, flags);
327 
328         hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
329 
330         if (mode & PERF_EF_START)
331                 __rapl_pmu_event_start(pmu, event);
332 
333         raw_spin_unlock_irqrestore(&pmu->lock, flags);
334 
335         return 0;
336 }
337 
338 static void rapl_pmu_event_del(struct perf_event *event, int flags)
339 {
340         rapl_pmu_event_stop(event, PERF_EF_UPDATE);
341 }
342 
343 static int rapl_pmu_event_init(struct perf_event *event)
344 {
345         u64 cfg = event->attr.config & RAPL_EVENT_MASK;
346         int bit, msr, ret = 0;
347         struct rapl_pmu *pmu;
348 
349         /* only look at RAPL events */
350         if (event->attr.type != rapl_pmus->pmu.type)
351                 return -ENOENT;
352 
353         /* check only supported bits are set */
354         if (event->attr.config & ~RAPL_EVENT_MASK)
355                 return -EINVAL;
356 
357         if (event->cpu < 0)
358                 return -EINVAL;
359 
360         /*
361          * check event is known (determines counter)
362          */
363         switch (cfg) {
364         case INTEL_RAPL_PP0:
365                 bit = RAPL_IDX_PP0_NRG_STAT;
366                 msr = MSR_PP0_ENERGY_STATUS;
367                 break;
368         case INTEL_RAPL_PKG:
369                 bit = RAPL_IDX_PKG_NRG_STAT;
370                 msr = MSR_PKG_ENERGY_STATUS;
371                 break;
372         case INTEL_RAPL_RAM:
373                 bit = RAPL_IDX_RAM_NRG_STAT;
374                 msr = MSR_DRAM_ENERGY_STATUS;
375                 break;
376         case INTEL_RAPL_PP1:
377                 bit = RAPL_IDX_PP1_NRG_STAT;
378                 msr = MSR_PP1_ENERGY_STATUS;
379                 break;
380         case INTEL_RAPL_PSYS:
381                 bit = RAPL_IDX_PSYS_NRG_STAT;
382                 msr = MSR_PLATFORM_ENERGY_STATUS;
383                 break;
384         default:
385                 return -EINVAL;
386         }
387         /* check event supported */
388         if (!(rapl_cntr_mask & (1 << bit)))
389                 return -EINVAL;
390 
391         /* unsupported modes and filters */
392         if (event->attr.exclude_user   ||
393             event->attr.exclude_kernel ||
394             event->attr.exclude_hv     ||
395             event->attr.exclude_idle   ||
396             event->attr.exclude_host   ||
397             event->attr.exclude_guest  ||
398             event->attr.sample_period) /* no sampling */
399                 return -EINVAL;
400 
401         /* must be done before validate_group */
402         pmu = cpu_to_rapl_pmu(event->cpu);
403         event->cpu = pmu->cpu;
404         event->pmu_private = pmu;
405         event->hw.event_base = msr;
406         event->hw.config = cfg;
407         event->hw.idx = bit;
408 
409         return ret;
410 }
411 
412 static void rapl_pmu_event_read(struct perf_event *event)
413 {
414         rapl_event_update(event);
415 }
416 
417 static ssize_t rapl_get_attr_cpumask(struct device *dev,
418                                 struct device_attribute *attr, char *buf)
419 {
420         return cpumap_print_to_pagebuf(true, buf, &rapl_cpu_mask);
421 }
422 
423 static DEVICE_ATTR(cpumask, S_IRUGO, rapl_get_attr_cpumask, NULL);
424 
425 static struct attribute *rapl_pmu_attrs[] = {
426         &dev_attr_cpumask.attr,
427         NULL,
428 };
429 
430 static struct attribute_group rapl_pmu_attr_group = {
431         .attrs = rapl_pmu_attrs,
432 };
433 
434 RAPL_EVENT_ATTR_STR(energy-cores, rapl_cores, "event=0x01");
435 RAPL_EVENT_ATTR_STR(energy-pkg  ,   rapl_pkg, "event=0x02");
436 RAPL_EVENT_ATTR_STR(energy-ram  ,   rapl_ram, "event=0x03");
437 RAPL_EVENT_ATTR_STR(energy-gpu  ,   rapl_gpu, "event=0x04");
438 RAPL_EVENT_ATTR_STR(energy-psys,   rapl_psys, "event=0x05");
439 
440 RAPL_EVENT_ATTR_STR(energy-cores.unit, rapl_cores_unit, "Joules");
441 RAPL_EVENT_ATTR_STR(energy-pkg.unit  ,   rapl_pkg_unit, "Joules");
442 RAPL_EVENT_ATTR_STR(energy-ram.unit  ,   rapl_ram_unit, "Joules");
443 RAPL_EVENT_ATTR_STR(energy-gpu.unit  ,   rapl_gpu_unit, "Joules");
444 RAPL_EVENT_ATTR_STR(energy-psys.unit,   rapl_psys_unit, "Joules");
445 
446 /*
447  * we compute in 0.23 nJ increments regardless of MSR
448  */
449 RAPL_EVENT_ATTR_STR(energy-cores.scale, rapl_cores_scale, "2.3283064365386962890625e-10");
450 RAPL_EVENT_ATTR_STR(energy-pkg.scale,     rapl_pkg_scale, "2.3283064365386962890625e-10");
451 RAPL_EVENT_ATTR_STR(energy-ram.scale,     rapl_ram_scale, "2.3283064365386962890625e-10");
452 RAPL_EVENT_ATTR_STR(energy-gpu.scale,     rapl_gpu_scale, "2.3283064365386962890625e-10");
453 RAPL_EVENT_ATTR_STR(energy-psys.scale,   rapl_psys_scale, "2.3283064365386962890625e-10");
454 
455 static struct attribute *rapl_events_srv_attr[] = {
456         EVENT_PTR(rapl_cores),
457         EVENT_PTR(rapl_pkg),
458         EVENT_PTR(rapl_ram),
459 
460         EVENT_PTR(rapl_cores_unit),
461         EVENT_PTR(rapl_pkg_unit),
462         EVENT_PTR(rapl_ram_unit),
463 
464         EVENT_PTR(rapl_cores_scale),
465         EVENT_PTR(rapl_pkg_scale),
466         EVENT_PTR(rapl_ram_scale),
467         NULL,
468 };
469 
470 static struct attribute *rapl_events_cln_attr[] = {
471         EVENT_PTR(rapl_cores),
472         EVENT_PTR(rapl_pkg),
473         EVENT_PTR(rapl_gpu),
474 
475         EVENT_PTR(rapl_cores_unit),
476         EVENT_PTR(rapl_pkg_unit),
477         EVENT_PTR(rapl_gpu_unit),
478 
479         EVENT_PTR(rapl_cores_scale),
480         EVENT_PTR(rapl_pkg_scale),
481         EVENT_PTR(rapl_gpu_scale),
482         NULL,
483 };
484 
485 static struct attribute *rapl_events_hsw_attr[] = {
486         EVENT_PTR(rapl_cores),
487         EVENT_PTR(rapl_pkg),
488         EVENT_PTR(rapl_gpu),
489         EVENT_PTR(rapl_ram),
490 
491         EVENT_PTR(rapl_cores_unit),
492         EVENT_PTR(rapl_pkg_unit),
493         EVENT_PTR(rapl_gpu_unit),
494         EVENT_PTR(rapl_ram_unit),
495 
496         EVENT_PTR(rapl_cores_scale),
497         EVENT_PTR(rapl_pkg_scale),
498         EVENT_PTR(rapl_gpu_scale),
499         EVENT_PTR(rapl_ram_scale),
500         NULL,
501 };
502 
503 static struct attribute *rapl_events_skl_attr[] = {
504         EVENT_PTR(rapl_cores),
505         EVENT_PTR(rapl_pkg),
506         EVENT_PTR(rapl_gpu),
507         EVENT_PTR(rapl_ram),
508         EVENT_PTR(rapl_psys),
509 
510         EVENT_PTR(rapl_cores_unit),
511         EVENT_PTR(rapl_pkg_unit),
512         EVENT_PTR(rapl_gpu_unit),
513         EVENT_PTR(rapl_ram_unit),
514         EVENT_PTR(rapl_psys_unit),
515 
516         EVENT_PTR(rapl_cores_scale),
517         EVENT_PTR(rapl_pkg_scale),
518         EVENT_PTR(rapl_gpu_scale),
519         EVENT_PTR(rapl_ram_scale),
520         EVENT_PTR(rapl_psys_scale),
521         NULL,
522 };
523 
524 static struct attribute *rapl_events_knl_attr[] = {
525         EVENT_PTR(rapl_pkg),
526         EVENT_PTR(rapl_ram),
527 
528         EVENT_PTR(rapl_pkg_unit),
529         EVENT_PTR(rapl_ram_unit),
530 
531         EVENT_PTR(rapl_pkg_scale),
532         EVENT_PTR(rapl_ram_scale),
533         NULL,
534 };
535 
536 static struct attribute_group rapl_pmu_events_group = {
537         .name = "events",
538         .attrs = NULL, /* patched at runtime */
539 };
540 
541 DEFINE_RAPL_FORMAT_ATTR(event, event, "config:0-7");
542 static struct attribute *rapl_formats_attr[] = {
543         &format_attr_event.attr,
544         NULL,
545 };
546 
547 static struct attribute_group rapl_pmu_format_group = {
548         .name = "format",
549         .attrs = rapl_formats_attr,
550 };
551 
552 const struct attribute_group *rapl_attr_groups[] = {
553         &rapl_pmu_attr_group,
554         &rapl_pmu_format_group,
555         &rapl_pmu_events_group,
556         NULL,
557 };
558 
559 static int rapl_cpu_offline(unsigned int cpu)
560 {
561         struct rapl_pmu *pmu = cpu_to_rapl_pmu(cpu);
562         int target;
563 
564         /* Check if exiting cpu is used for collecting rapl events */
565         if (!cpumask_test_and_clear_cpu(cpu, &rapl_cpu_mask))
566                 return 0;
567 
568         pmu->cpu = -1;
569         /* Find a new cpu to collect rapl events */
570         target = cpumask_any_but(topology_core_cpumask(cpu), cpu);
571 
572         /* Migrate rapl events to the new target */
573         if (target < nr_cpu_ids) {
574                 cpumask_set_cpu(target, &rapl_cpu_mask);
575                 pmu->cpu = target;
576                 perf_pmu_migrate_context(pmu->pmu, cpu, target);
577         }
578         return 0;
579 }
580 
581 static int rapl_cpu_online(unsigned int cpu)
582 {
583         struct rapl_pmu *pmu = cpu_to_rapl_pmu(cpu);
584         int target;
585 
586         /*
587          * Check if there is an online cpu in the package which collects rapl
588          * events already.
589          */
590         target = cpumask_any_and(&rapl_cpu_mask, topology_core_cpumask(cpu));
591         if (target < nr_cpu_ids)
592                 return 0;
593 
594         cpumask_set_cpu(cpu, &rapl_cpu_mask);
595         pmu->cpu = cpu;
596         return 0;
597 }
598 
599 static int rapl_cpu_prepare(unsigned int cpu)
600 {
601         struct rapl_pmu *pmu = cpu_to_rapl_pmu(cpu);
602 
603         if (pmu)
604                 return 0;
605 
606         pmu = kzalloc_node(sizeof(*pmu), GFP_KERNEL, cpu_to_node(cpu));
607         if (!pmu)
608                 return -ENOMEM;
609 
610         raw_spin_lock_init(&pmu->lock);
611         INIT_LIST_HEAD(&pmu->active_list);
612         pmu->pmu = &rapl_pmus->pmu;
613         pmu->timer_interval = ms_to_ktime(rapl_timer_ms);
614         pmu->cpu = -1;
615         rapl_hrtimer_init(pmu);
616         rapl_pmus->pmus[topology_logical_package_id(cpu)] = pmu;
617         return 0;
618 }
619 
620 static int rapl_check_hw_unit(bool apply_quirk)
621 {
622         u64 msr_rapl_power_unit_bits;
623         int i;
624 
625         /* protect rdmsrl() to handle virtualization */
626         if (rdmsrl_safe(MSR_RAPL_POWER_UNIT, &msr_rapl_power_unit_bits))
627                 return -1;
628         for (i = 0; i < NR_RAPL_DOMAINS; i++)
629                 rapl_hw_unit[i] = (msr_rapl_power_unit_bits >> 8) & 0x1FULL;
630 
631         /*
632          * DRAM domain on HSW server and KNL has fixed energy unit which can be
633          * different than the unit from power unit MSR. See
634          * "Intel Xeon Processor E5-1600 and E5-2600 v3 Product Families, V2
635          * of 2. Datasheet, September 2014, Reference Number: 330784-001 "
636          */
637         if (apply_quirk)
638                 rapl_hw_unit[RAPL_IDX_RAM_NRG_STAT] = 16;
639 
640         /*
641          * Calculate the timer rate:
642          * Use reference of 200W for scaling the timeout to avoid counter
643          * overflows. 200W = 200 Joules/sec
644          * Divide interval by 2 to avoid lockstep (2 * 100)
645          * if hw unit is 32, then we use 2 ms 1/200/2
646          */
647         rapl_timer_ms = 2;
648         if (rapl_hw_unit[0] < 32) {
649                 rapl_timer_ms = (1000 / (2 * 100));
650                 rapl_timer_ms *= (1ULL << (32 - rapl_hw_unit[0] - 1));
651         }
652         return 0;
653 }
654 
655 static void __init rapl_advertise(void)
656 {
657         int i;
658 
659         pr_info("API unit is 2^-32 Joules, %d fixed counters, %llu ms ovfl timer\n",
660                 hweight32(rapl_cntr_mask), rapl_timer_ms);
661 
662         for (i = 0; i < NR_RAPL_DOMAINS; i++) {
663                 if (rapl_cntr_mask & (1 << i)) {
664                         pr_info("hw unit of domain %s 2^-%d Joules\n",
665                                 rapl_domain_names[i], rapl_hw_unit[i]);
666                 }
667         }
668 }
669 
670 static void cleanup_rapl_pmus(void)
671 {
672         int i;
673 
674         for (i = 0; i < rapl_pmus->maxpkg; i++)
675                 kfree(rapl_pmus->pmus[i]);
676         kfree(rapl_pmus);
677 }
678 
679 static int __init init_rapl_pmus(void)
680 {
681         int maxpkg = topology_max_packages();
682         size_t size;
683 
684         size = sizeof(*rapl_pmus) + maxpkg * sizeof(struct rapl_pmu *);
685         rapl_pmus = kzalloc(size, GFP_KERNEL);
686         if (!rapl_pmus)
687                 return -ENOMEM;
688 
689         rapl_pmus->maxpkg               = maxpkg;
690         rapl_pmus->pmu.attr_groups      = rapl_attr_groups;
691         rapl_pmus->pmu.task_ctx_nr      = perf_invalid_context;
692         rapl_pmus->pmu.event_init       = rapl_pmu_event_init;
693         rapl_pmus->pmu.add              = rapl_pmu_event_add;
694         rapl_pmus->pmu.del              = rapl_pmu_event_del;
695         rapl_pmus->pmu.start            = rapl_pmu_event_start;
696         rapl_pmus->pmu.stop             = rapl_pmu_event_stop;
697         rapl_pmus->pmu.read             = rapl_pmu_event_read;
698         return 0;
699 }
700 
701 #define X86_RAPL_MODEL_MATCH(model, init)       \
702         { X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long)&init }
703 
704 struct intel_rapl_init_fun {
705         bool apply_quirk;
706         int cntr_mask;
707         struct attribute **attrs;
708 };
709 
710 static const struct intel_rapl_init_fun snb_rapl_init __initconst = {
711         .apply_quirk = false,
712         .cntr_mask = RAPL_IDX_CLN,
713         .attrs = rapl_events_cln_attr,
714 };
715 
716 static const struct intel_rapl_init_fun hsx_rapl_init __initconst = {
717         .apply_quirk = true,
718         .cntr_mask = RAPL_IDX_SRV,
719         .attrs = rapl_events_srv_attr,
720 };
721 
722 static const struct intel_rapl_init_fun hsw_rapl_init __initconst = {
723         .apply_quirk = false,
724         .cntr_mask = RAPL_IDX_HSW,
725         .attrs = rapl_events_hsw_attr,
726 };
727 
728 static const struct intel_rapl_init_fun snbep_rapl_init __initconst = {
729         .apply_quirk = false,
730         .cntr_mask = RAPL_IDX_SRV,
731         .attrs = rapl_events_srv_attr,
732 };
733 
734 static const struct intel_rapl_init_fun knl_rapl_init __initconst = {
735         .apply_quirk = true,
736         .cntr_mask = RAPL_IDX_KNL,
737         .attrs = rapl_events_knl_attr,
738 };
739 
740 static const struct intel_rapl_init_fun skl_rapl_init __initconst = {
741         .apply_quirk = false,
742         .cntr_mask = RAPL_IDX_SKL_CLN,
743         .attrs = rapl_events_skl_attr,
744 };
745 
746 static const struct x86_cpu_id rapl_cpu_match[] __initconst = {
747         X86_RAPL_MODEL_MATCH(INTEL_FAM6_SANDYBRIDGE,   snb_rapl_init),
748         X86_RAPL_MODEL_MATCH(INTEL_FAM6_SANDYBRIDGE_X, snbep_rapl_init),
749 
750         X86_RAPL_MODEL_MATCH(INTEL_FAM6_IVYBRIDGE,   snb_rapl_init),
751         X86_RAPL_MODEL_MATCH(INTEL_FAM6_IVYBRIDGE_X, snbep_rapl_init),
752 
753         X86_RAPL_MODEL_MATCH(INTEL_FAM6_HASWELL_CORE, hsw_rapl_init),
754         X86_RAPL_MODEL_MATCH(INTEL_FAM6_HASWELL_X,    hsw_rapl_init),
755         X86_RAPL_MODEL_MATCH(INTEL_FAM6_HASWELL_ULT,  hsw_rapl_init),
756         X86_RAPL_MODEL_MATCH(INTEL_FAM6_HASWELL_GT3E, hsw_rapl_init),
757 
758         X86_RAPL_MODEL_MATCH(INTEL_FAM6_BROADWELL_CORE,   hsw_rapl_init),
759         X86_RAPL_MODEL_MATCH(INTEL_FAM6_BROADWELL_GT3E,   hsw_rapl_init),
760         X86_RAPL_MODEL_MATCH(INTEL_FAM6_BROADWELL_X,      hsw_rapl_init),
761         X86_RAPL_MODEL_MATCH(INTEL_FAM6_BROADWELL_XEON_D, hsw_rapl_init),
762 
763         X86_RAPL_MODEL_MATCH(INTEL_FAM6_XEON_PHI_KNL, knl_rapl_init),
764 
765         X86_RAPL_MODEL_MATCH(INTEL_FAM6_SKYLAKE_MOBILE,  skl_rapl_init),
766         X86_RAPL_MODEL_MATCH(INTEL_FAM6_SKYLAKE_DESKTOP, skl_rapl_init),
767         X86_RAPL_MODEL_MATCH(INTEL_FAM6_SKYLAKE_X,       hsx_rapl_init),
768         {},
769 };
770 
771 MODULE_DEVICE_TABLE(x86cpu, rapl_cpu_match);
772 
773 static int __init rapl_pmu_init(void)
774 {
775         const struct x86_cpu_id *id;
776         struct intel_rapl_init_fun *rapl_init;
777         bool apply_quirk;
778         int ret;
779 
780         id = x86_match_cpu(rapl_cpu_match);
781         if (!id)
782                 return -ENODEV;
783 
784         rapl_init = (struct intel_rapl_init_fun *)id->driver_data;
785         apply_quirk = rapl_init->apply_quirk;
786         rapl_cntr_mask = rapl_init->cntr_mask;
787         rapl_pmu_events_group.attrs = rapl_init->attrs;
788 
789         ret = rapl_check_hw_unit(apply_quirk);
790         if (ret)
791                 return ret;
792 
793         ret = init_rapl_pmus();
794         if (ret)
795                 return ret;
796 
797         /*
798          * Install callbacks. Core will call them for each online cpu.
799          */
800 
801         ret = cpuhp_setup_state(CPUHP_PERF_X86_RAPL_PREP, "PERF_X86_RAPL_PREP",
802                                 rapl_cpu_prepare, NULL);
803         if (ret)
804                 goto out;
805 
806         ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_RAPL_ONLINE,
807                                 "AP_PERF_X86_RAPL_ONLINE",
808                                 rapl_cpu_online, rapl_cpu_offline);
809         if (ret)
810                 goto out1;
811 
812         ret = perf_pmu_register(&rapl_pmus->pmu, "power", -1);
813         if (ret)
814                 goto out2;
815 
816         rapl_advertise();
817         return 0;
818 
819 out2:
820         cpuhp_remove_state(CPUHP_AP_PERF_X86_RAPL_ONLINE);
821 out1:
822         cpuhp_remove_state(CPUHP_PERF_X86_RAPL_PREP);
823 out:
824         pr_warn("Initialization failed (%d), disabled\n", ret);
825         cleanup_rapl_pmus();
826         return ret;
827 }
828 module_init(rapl_pmu_init);
829 
830 static void __exit intel_rapl_exit(void)
831 {
832         cpuhp_remove_state_nocalls(CPUHP_AP_PERF_X86_RAPL_ONLINE);
833         cpuhp_remove_state_nocalls(CPUHP_PERF_X86_RAPL_PREP);
834         perf_pmu_unregister(&rapl_pmus->pmu);
835         cleanup_rapl_pmus();
836 }
837 module_exit(intel_rapl_exit);
838 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp