~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/events/intel/uncore.c

Version: ~ [ linux-5.13-rc5 ] ~ [ linux-5.12.9 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.42 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.124 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.193 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.235 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.271 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.271 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-only
  2 #include <linux/module.h>
  3 
  4 #include <asm/cpu_device_id.h>
  5 #include <asm/intel-family.h>
  6 #include "uncore.h"
  7 
  8 static struct intel_uncore_type *empty_uncore[] = { NULL, };
  9 struct intel_uncore_type **uncore_msr_uncores = empty_uncore;
 10 struct intel_uncore_type **uncore_pci_uncores = empty_uncore;
 11 struct intel_uncore_type **uncore_mmio_uncores = empty_uncore;
 12 
 13 static bool pcidrv_registered;
 14 struct pci_driver *uncore_pci_driver;
 15 /* pci bus to socket mapping */
 16 DEFINE_RAW_SPINLOCK(pci2phy_map_lock);
 17 struct list_head pci2phy_map_head = LIST_HEAD_INIT(pci2phy_map_head);
 18 struct pci_extra_dev *uncore_extra_pci_dev;
 19 static int max_dies;
 20 
 21 /* mask of cpus that collect uncore events */
 22 static cpumask_t uncore_cpu_mask;
 23 
 24 /* constraint for the fixed counter */
 25 static struct event_constraint uncore_constraint_fixed =
 26         EVENT_CONSTRAINT(~0ULL, 1 << UNCORE_PMC_IDX_FIXED, ~0ULL);
 27 struct event_constraint uncore_constraint_empty =
 28         EVENT_CONSTRAINT(0, 0, 0);
 29 
 30 MODULE_LICENSE("GPL");
 31 
 32 int uncore_pcibus_to_physid(struct pci_bus *bus)
 33 {
 34         struct pci2phy_map *map;
 35         int phys_id = -1;
 36 
 37         raw_spin_lock(&pci2phy_map_lock);
 38         list_for_each_entry(map, &pci2phy_map_head, list) {
 39                 if (map->segment == pci_domain_nr(bus)) {
 40                         phys_id = map->pbus_to_physid[bus->number];
 41                         break;
 42                 }
 43         }
 44         raw_spin_unlock(&pci2phy_map_lock);
 45 
 46         return phys_id;
 47 }
 48 
 49 static void uncore_free_pcibus_map(void)
 50 {
 51         struct pci2phy_map *map, *tmp;
 52 
 53         list_for_each_entry_safe(map, tmp, &pci2phy_map_head, list) {
 54                 list_del(&map->list);
 55                 kfree(map);
 56         }
 57 }
 58 
 59 struct pci2phy_map *__find_pci2phy_map(int segment)
 60 {
 61         struct pci2phy_map *map, *alloc = NULL;
 62         int i;
 63 
 64         lockdep_assert_held(&pci2phy_map_lock);
 65 
 66 lookup:
 67         list_for_each_entry(map, &pci2phy_map_head, list) {
 68                 if (map->segment == segment)
 69                         goto end;
 70         }
 71 
 72         if (!alloc) {
 73                 raw_spin_unlock(&pci2phy_map_lock);
 74                 alloc = kmalloc(sizeof(struct pci2phy_map), GFP_KERNEL);
 75                 raw_spin_lock(&pci2phy_map_lock);
 76 
 77                 if (!alloc)
 78                         return NULL;
 79 
 80                 goto lookup;
 81         }
 82 
 83         map = alloc;
 84         alloc = NULL;
 85         map->segment = segment;
 86         for (i = 0; i < 256; i++)
 87                 map->pbus_to_physid[i] = -1;
 88         list_add_tail(&map->list, &pci2phy_map_head);
 89 
 90 end:
 91         kfree(alloc);
 92         return map;
 93 }
 94 
 95 ssize_t uncore_event_show(struct kobject *kobj,
 96                           struct kobj_attribute *attr, char *buf)
 97 {
 98         struct uncore_event_desc *event =
 99                 container_of(attr, struct uncore_event_desc, attr);
100         return sprintf(buf, "%s", event->config);
101 }
102 
103 struct intel_uncore_box *uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
104 {
105         unsigned int dieid = topology_logical_die_id(cpu);
106 
107         /*
108          * The unsigned check also catches the '-1' return value for non
109          * existent mappings in the topology map.
110          */
111         return dieid < max_dies ? pmu->boxes[dieid] : NULL;
112 }
113 
114 u64 uncore_msr_read_counter(struct intel_uncore_box *box, struct perf_event *event)
115 {
116         u64 count;
117 
118         rdmsrl(event->hw.event_base, count);
119 
120         return count;
121 }
122 
123 void uncore_mmio_exit_box(struct intel_uncore_box *box)
124 {
125         if (box->io_addr)
126                 iounmap(box->io_addr);
127 }
128 
129 u64 uncore_mmio_read_counter(struct intel_uncore_box *box,
130                              struct perf_event *event)
131 {
132         if (!box->io_addr)
133                 return 0;
134 
135         return readq(box->io_addr + event->hw.event_base);
136 }
137 
138 /*
139  * generic get constraint function for shared match/mask registers.
140  */
141 struct event_constraint *
142 uncore_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
143 {
144         struct intel_uncore_extra_reg *er;
145         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
146         struct hw_perf_event_extra *reg2 = &event->hw.branch_reg;
147         unsigned long flags;
148         bool ok = false;
149 
150         /*
151          * reg->alloc can be set due to existing state, so for fake box we
152          * need to ignore this, otherwise we might fail to allocate proper
153          * fake state for this extra reg constraint.
154          */
155         if (reg1->idx == EXTRA_REG_NONE ||
156             (!uncore_box_is_fake(box) && reg1->alloc))
157                 return NULL;
158 
159         er = &box->shared_regs[reg1->idx];
160         raw_spin_lock_irqsave(&er->lock, flags);
161         if (!atomic_read(&er->ref) ||
162             (er->config1 == reg1->config && er->config2 == reg2->config)) {
163                 atomic_inc(&er->ref);
164                 er->config1 = reg1->config;
165                 er->config2 = reg2->config;
166                 ok = true;
167         }
168         raw_spin_unlock_irqrestore(&er->lock, flags);
169 
170         if (ok) {
171                 if (!uncore_box_is_fake(box))
172                         reg1->alloc = 1;
173                 return NULL;
174         }
175 
176         return &uncore_constraint_empty;
177 }
178 
179 void uncore_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
180 {
181         struct intel_uncore_extra_reg *er;
182         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
183 
184         /*
185          * Only put constraint if extra reg was actually allocated. Also
186          * takes care of event which do not use an extra shared reg.
187          *
188          * Also, if this is a fake box we shouldn't touch any event state
189          * (reg->alloc) and we don't care about leaving inconsistent box
190          * state either since it will be thrown out.
191          */
192         if (uncore_box_is_fake(box) || !reg1->alloc)
193                 return;
194 
195         er = &box->shared_regs[reg1->idx];
196         atomic_dec(&er->ref);
197         reg1->alloc = 0;
198 }
199 
200 u64 uncore_shared_reg_config(struct intel_uncore_box *box, int idx)
201 {
202         struct intel_uncore_extra_reg *er;
203         unsigned long flags;
204         u64 config;
205 
206         er = &box->shared_regs[idx];
207 
208         raw_spin_lock_irqsave(&er->lock, flags);
209         config = er->config;
210         raw_spin_unlock_irqrestore(&er->lock, flags);
211 
212         return config;
213 }
214 
215 static void uncore_assign_hw_event(struct intel_uncore_box *box,
216                                    struct perf_event *event, int idx)
217 {
218         struct hw_perf_event *hwc = &event->hw;
219 
220         hwc->idx = idx;
221         hwc->last_tag = ++box->tags[idx];
222 
223         if (uncore_pmc_fixed(hwc->idx)) {
224                 hwc->event_base = uncore_fixed_ctr(box);
225                 hwc->config_base = uncore_fixed_ctl(box);
226                 return;
227         }
228 
229         hwc->config_base = uncore_event_ctl(box, hwc->idx);
230         hwc->event_base  = uncore_perf_ctr(box, hwc->idx);
231 }
232 
233 void uncore_perf_event_update(struct intel_uncore_box *box, struct perf_event *event)
234 {
235         u64 prev_count, new_count, delta;
236         int shift;
237 
238         if (uncore_pmc_freerunning(event->hw.idx))
239                 shift = 64 - uncore_freerunning_bits(box, event);
240         else if (uncore_pmc_fixed(event->hw.idx))
241                 shift = 64 - uncore_fixed_ctr_bits(box);
242         else
243                 shift = 64 - uncore_perf_ctr_bits(box);
244 
245         /* the hrtimer might modify the previous event value */
246 again:
247         prev_count = local64_read(&event->hw.prev_count);
248         new_count = uncore_read_counter(box, event);
249         if (local64_xchg(&event->hw.prev_count, new_count) != prev_count)
250                 goto again;
251 
252         delta = (new_count << shift) - (prev_count << shift);
253         delta >>= shift;
254 
255         local64_add(delta, &event->count);
256 }
257 
258 /*
259  * The overflow interrupt is unavailable for SandyBridge-EP, is broken
260  * for SandyBridge. So we use hrtimer to periodically poll the counter
261  * to avoid overflow.
262  */
263 static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer)
264 {
265         struct intel_uncore_box *box;
266         struct perf_event *event;
267         unsigned long flags;
268         int bit;
269 
270         box = container_of(hrtimer, struct intel_uncore_box, hrtimer);
271         if (!box->n_active || box->cpu != smp_processor_id())
272                 return HRTIMER_NORESTART;
273         /*
274          * disable local interrupt to prevent uncore_pmu_event_start/stop
275          * to interrupt the update process
276          */
277         local_irq_save(flags);
278 
279         /*
280          * handle boxes with an active event list as opposed to active
281          * counters
282          */
283         list_for_each_entry(event, &box->active_list, active_entry) {
284                 uncore_perf_event_update(box, event);
285         }
286 
287         for_each_set_bit(bit, box->active_mask, UNCORE_PMC_IDX_MAX)
288                 uncore_perf_event_update(box, box->events[bit]);
289 
290         local_irq_restore(flags);
291 
292         hrtimer_forward_now(hrtimer, ns_to_ktime(box->hrtimer_duration));
293         return HRTIMER_RESTART;
294 }
295 
296 void uncore_pmu_start_hrtimer(struct intel_uncore_box *box)
297 {
298         hrtimer_start(&box->hrtimer, ns_to_ktime(box->hrtimer_duration),
299                       HRTIMER_MODE_REL_PINNED);
300 }
301 
302 void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box)
303 {
304         hrtimer_cancel(&box->hrtimer);
305 }
306 
307 static void uncore_pmu_init_hrtimer(struct intel_uncore_box *box)
308 {
309         hrtimer_init(&box->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
310         box->hrtimer.function = uncore_pmu_hrtimer;
311 }
312 
313 static struct intel_uncore_box *uncore_alloc_box(struct intel_uncore_type *type,
314                                                  int node)
315 {
316         int i, size, numshared = type->num_shared_regs ;
317         struct intel_uncore_box *box;
318 
319         size = sizeof(*box) + numshared * sizeof(struct intel_uncore_extra_reg);
320 
321         box = kzalloc_node(size, GFP_KERNEL, node);
322         if (!box)
323                 return NULL;
324 
325         for (i = 0; i < numshared; i++)
326                 raw_spin_lock_init(&box->shared_regs[i].lock);
327 
328         uncore_pmu_init_hrtimer(box);
329         box->cpu = -1;
330         box->pci_phys_id = -1;
331         box->dieid = -1;
332 
333         /* set default hrtimer timeout */
334         box->hrtimer_duration = UNCORE_PMU_HRTIMER_INTERVAL;
335 
336         INIT_LIST_HEAD(&box->active_list);
337 
338         return box;
339 }
340 
341 /*
342  * Using uncore_pmu_event_init pmu event_init callback
343  * as a detection point for uncore events.
344  */
345 static int uncore_pmu_event_init(struct perf_event *event);
346 
347 static bool is_box_event(struct intel_uncore_box *box, struct perf_event *event)
348 {
349         return &box->pmu->pmu == event->pmu;
350 }
351 
352 static int
353 uncore_collect_events(struct intel_uncore_box *box, struct perf_event *leader,
354                       bool dogrp)
355 {
356         struct perf_event *event;
357         int n, max_count;
358 
359         max_count = box->pmu->type->num_counters;
360         if (box->pmu->type->fixed_ctl)
361                 max_count++;
362 
363         if (box->n_events >= max_count)
364                 return -EINVAL;
365 
366         n = box->n_events;
367 
368         if (is_box_event(box, leader)) {
369                 box->event_list[n] = leader;
370                 n++;
371         }
372 
373         if (!dogrp)
374                 return n;
375 
376         for_each_sibling_event(event, leader) {
377                 if (!is_box_event(box, event) ||
378                     event->state <= PERF_EVENT_STATE_OFF)
379                         continue;
380 
381                 if (n >= max_count)
382                         return -EINVAL;
383 
384                 box->event_list[n] = event;
385                 n++;
386         }
387         return n;
388 }
389 
390 static struct event_constraint *
391 uncore_get_event_constraint(struct intel_uncore_box *box, struct perf_event *event)
392 {
393         struct intel_uncore_type *type = box->pmu->type;
394         struct event_constraint *c;
395 
396         if (type->ops->get_constraint) {
397                 c = type->ops->get_constraint(box, event);
398                 if (c)
399                         return c;
400         }
401 
402         if (event->attr.config == UNCORE_FIXED_EVENT)
403                 return &uncore_constraint_fixed;
404 
405         if (type->constraints) {
406                 for_each_event_constraint(c, type->constraints) {
407                         if ((event->hw.config & c->cmask) == c->code)
408                                 return c;
409                 }
410         }
411 
412         return &type->unconstrainted;
413 }
414 
415 static void uncore_put_event_constraint(struct intel_uncore_box *box,
416                                         struct perf_event *event)
417 {
418         if (box->pmu->type->ops->put_constraint)
419                 box->pmu->type->ops->put_constraint(box, event);
420 }
421 
422 static int uncore_assign_events(struct intel_uncore_box *box, int assign[], int n)
423 {
424         unsigned long used_mask[BITS_TO_LONGS(UNCORE_PMC_IDX_MAX)];
425         struct event_constraint *c;
426         int i, wmin, wmax, ret = 0;
427         struct hw_perf_event *hwc;
428 
429         bitmap_zero(used_mask, UNCORE_PMC_IDX_MAX);
430 
431         for (i = 0, wmin = UNCORE_PMC_IDX_MAX, wmax = 0; i < n; i++) {
432                 c = uncore_get_event_constraint(box, box->event_list[i]);
433                 box->event_constraint[i] = c;
434                 wmin = min(wmin, c->weight);
435                 wmax = max(wmax, c->weight);
436         }
437 
438         /* fastpath, try to reuse previous register */
439         for (i = 0; i < n; i++) {
440                 hwc = &box->event_list[i]->hw;
441                 c = box->event_constraint[i];
442 
443                 /* never assigned */
444                 if (hwc->idx == -1)
445                         break;
446 
447                 /* constraint still honored */
448                 if (!test_bit(hwc->idx, c->idxmsk))
449                         break;
450 
451                 /* not already used */
452                 if (test_bit(hwc->idx, used_mask))
453                         break;
454 
455                 __set_bit(hwc->idx, used_mask);
456                 if (assign)
457                         assign[i] = hwc->idx;
458         }
459         /* slow path */
460         if (i != n)
461                 ret = perf_assign_events(box->event_constraint, n,
462                                          wmin, wmax, n, assign);
463 
464         if (!assign || ret) {
465                 for (i = 0; i < n; i++)
466                         uncore_put_event_constraint(box, box->event_list[i]);
467         }
468         return ret ? -EINVAL : 0;
469 }
470 
471 void uncore_pmu_event_start(struct perf_event *event, int flags)
472 {
473         struct intel_uncore_box *box = uncore_event_to_box(event);
474         int idx = event->hw.idx;
475 
476         if (WARN_ON_ONCE(idx == -1 || idx >= UNCORE_PMC_IDX_MAX))
477                 return;
478 
479         /*
480          * Free running counter is read-only and always active.
481          * Use the current counter value as start point.
482          * There is no overflow interrupt for free running counter.
483          * Use hrtimer to periodically poll the counter to avoid overflow.
484          */
485         if (uncore_pmc_freerunning(event->hw.idx)) {
486                 list_add_tail(&event->active_entry, &box->active_list);
487                 local64_set(&event->hw.prev_count,
488                             uncore_read_counter(box, event));
489                 if (box->n_active++ == 0)
490                         uncore_pmu_start_hrtimer(box);
491                 return;
492         }
493 
494         if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
495                 return;
496 
497         event->hw.state = 0;
498         box->events[idx] = event;
499         box->n_active++;
500         __set_bit(idx, box->active_mask);
501 
502         local64_set(&event->hw.prev_count, uncore_read_counter(box, event));
503         uncore_enable_event(box, event);
504 
505         if (box->n_active == 1)
506                 uncore_pmu_start_hrtimer(box);
507 }
508 
509 void uncore_pmu_event_stop(struct perf_event *event, int flags)
510 {
511         struct intel_uncore_box *box = uncore_event_to_box(event);
512         struct hw_perf_event *hwc = &event->hw;
513 
514         /* Cannot disable free running counter which is read-only */
515         if (uncore_pmc_freerunning(hwc->idx)) {
516                 list_del(&event->active_entry);
517                 if (--box->n_active == 0)
518                         uncore_pmu_cancel_hrtimer(box);
519                 uncore_perf_event_update(box, event);
520                 return;
521         }
522 
523         if (__test_and_clear_bit(hwc->idx, box->active_mask)) {
524                 uncore_disable_event(box, event);
525                 box->n_active--;
526                 box->events[hwc->idx] = NULL;
527                 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
528                 hwc->state |= PERF_HES_STOPPED;
529 
530                 if (box->n_active == 0)
531                         uncore_pmu_cancel_hrtimer(box);
532         }
533 
534         if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
535                 /*
536                  * Drain the remaining delta count out of a event
537                  * that we are disabling:
538                  */
539                 uncore_perf_event_update(box, event);
540                 hwc->state |= PERF_HES_UPTODATE;
541         }
542 }
543 
544 int uncore_pmu_event_add(struct perf_event *event, int flags)
545 {
546         struct intel_uncore_box *box = uncore_event_to_box(event);
547         struct hw_perf_event *hwc = &event->hw;
548         int assign[UNCORE_PMC_IDX_MAX];
549         int i, n, ret;
550 
551         if (!box)
552                 return -ENODEV;
553 
554         /*
555          * The free funning counter is assigned in event_init().
556          * The free running counter event and free running counter
557          * are 1:1 mapped. It doesn't need to be tracked in event_list.
558          */
559         if (uncore_pmc_freerunning(hwc->idx)) {
560                 if (flags & PERF_EF_START)
561                         uncore_pmu_event_start(event, 0);
562                 return 0;
563         }
564 
565         ret = n = uncore_collect_events(box, event, false);
566         if (ret < 0)
567                 return ret;
568 
569         hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
570         if (!(flags & PERF_EF_START))
571                 hwc->state |= PERF_HES_ARCH;
572 
573         ret = uncore_assign_events(box, assign, n);
574         if (ret)
575                 return ret;
576 
577         /* save events moving to new counters */
578         for (i = 0; i < box->n_events; i++) {
579                 event = box->event_list[i];
580                 hwc = &event->hw;
581 
582                 if (hwc->idx == assign[i] &&
583                         hwc->last_tag == box->tags[assign[i]])
584                         continue;
585                 /*
586                  * Ensure we don't accidentally enable a stopped
587                  * counter simply because we rescheduled.
588                  */
589                 if (hwc->state & PERF_HES_STOPPED)
590                         hwc->state |= PERF_HES_ARCH;
591 
592                 uncore_pmu_event_stop(event, PERF_EF_UPDATE);
593         }
594 
595         /* reprogram moved events into new counters */
596         for (i = 0; i < n; i++) {
597                 event = box->event_list[i];
598                 hwc = &event->hw;
599 
600                 if (hwc->idx != assign[i] ||
601                         hwc->last_tag != box->tags[assign[i]])
602                         uncore_assign_hw_event(box, event, assign[i]);
603                 else if (i < box->n_events)
604                         continue;
605 
606                 if (hwc->state & PERF_HES_ARCH)
607                         continue;
608 
609                 uncore_pmu_event_start(event, 0);
610         }
611         box->n_events = n;
612 
613         return 0;
614 }
615 
616 void uncore_pmu_event_del(struct perf_event *event, int flags)
617 {
618         struct intel_uncore_box *box = uncore_event_to_box(event);
619         int i;
620 
621         uncore_pmu_event_stop(event, PERF_EF_UPDATE);
622 
623         /*
624          * The event for free running counter is not tracked by event_list.
625          * It doesn't need to force event->hw.idx = -1 to reassign the counter.
626          * Because the event and the free running counter are 1:1 mapped.
627          */
628         if (uncore_pmc_freerunning(event->hw.idx))
629                 return;
630 
631         for (i = 0; i < box->n_events; i++) {
632                 if (event == box->event_list[i]) {
633                         uncore_put_event_constraint(box, event);
634 
635                         for (++i; i < box->n_events; i++)
636                                 box->event_list[i - 1] = box->event_list[i];
637 
638                         --box->n_events;
639                         break;
640                 }
641         }
642 
643         event->hw.idx = -1;
644         event->hw.last_tag = ~0ULL;
645 }
646 
647 void uncore_pmu_event_read(struct perf_event *event)
648 {
649         struct intel_uncore_box *box = uncore_event_to_box(event);
650         uncore_perf_event_update(box, event);
651 }
652 
653 /*
654  * validation ensures the group can be loaded onto the
655  * PMU if it was the only group available.
656  */
657 static int uncore_validate_group(struct intel_uncore_pmu *pmu,
658                                 struct perf_event *event)
659 {
660         struct perf_event *leader = event->group_leader;
661         struct intel_uncore_box *fake_box;
662         int ret = -EINVAL, n;
663 
664         /* The free running counter is always active. */
665         if (uncore_pmc_freerunning(event->hw.idx))
666                 return 0;
667 
668         fake_box = uncore_alloc_box(pmu->type, NUMA_NO_NODE);
669         if (!fake_box)
670                 return -ENOMEM;
671 
672         fake_box->pmu = pmu;
673         /*
674          * the event is not yet connected with its
675          * siblings therefore we must first collect
676          * existing siblings, then add the new event
677          * before we can simulate the scheduling
678          */
679         n = uncore_collect_events(fake_box, leader, true);
680         if (n < 0)
681                 goto out;
682 
683         fake_box->n_events = n;
684         n = uncore_collect_events(fake_box, event, false);
685         if (n < 0)
686                 goto out;
687 
688         fake_box->n_events = n;
689 
690         ret = uncore_assign_events(fake_box, NULL, n);
691 out:
692         kfree(fake_box);
693         return ret;
694 }
695 
696 static int uncore_pmu_event_init(struct perf_event *event)
697 {
698         struct intel_uncore_pmu *pmu;
699         struct intel_uncore_box *box;
700         struct hw_perf_event *hwc = &event->hw;
701         int ret;
702 
703         if (event->attr.type != event->pmu->type)
704                 return -ENOENT;
705 
706         pmu = uncore_event_to_pmu(event);
707         /* no device found for this pmu */
708         if (pmu->func_id < 0)
709                 return -ENOENT;
710 
711         /* Sampling not supported yet */
712         if (hwc->sample_period)
713                 return -EINVAL;
714 
715         /*
716          * Place all uncore events for a particular physical package
717          * onto a single cpu
718          */
719         if (event->cpu < 0)
720                 return -EINVAL;
721         box = uncore_pmu_to_box(pmu, event->cpu);
722         if (!box || box->cpu < 0)
723                 return -EINVAL;
724         event->cpu = box->cpu;
725         event->pmu_private = box;
726 
727         event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG;
728 
729         event->hw.idx = -1;
730         event->hw.last_tag = ~0ULL;
731         event->hw.extra_reg.idx = EXTRA_REG_NONE;
732         event->hw.branch_reg.idx = EXTRA_REG_NONE;
733 
734         if (event->attr.config == UNCORE_FIXED_EVENT) {
735                 /* no fixed counter */
736                 if (!pmu->type->fixed_ctl)
737                         return -EINVAL;
738                 /*
739                  * if there is only one fixed counter, only the first pmu
740                  * can access the fixed counter
741                  */
742                 if (pmu->type->single_fixed && pmu->pmu_idx > 0)
743                         return -EINVAL;
744 
745                 /* fixed counters have event field hardcoded to zero */
746                 hwc->config = 0ULL;
747         } else if (is_freerunning_event(event)) {
748                 hwc->config = event->attr.config;
749                 if (!check_valid_freerunning_event(box, event))
750                         return -EINVAL;
751                 event->hw.idx = UNCORE_PMC_IDX_FREERUNNING;
752                 /*
753                  * The free running counter event and free running counter
754                  * are always 1:1 mapped.
755                  * The free running counter is always active.
756                  * Assign the free running counter here.
757                  */
758                 event->hw.event_base = uncore_freerunning_counter(box, event);
759         } else {
760                 hwc->config = event->attr.config &
761                               (pmu->type->event_mask | ((u64)pmu->type->event_mask_ext << 32));
762                 if (pmu->type->ops->hw_config) {
763                         ret = pmu->type->ops->hw_config(box, event);
764                         if (ret)
765                                 return ret;
766                 }
767         }
768 
769         if (event->group_leader != event)
770                 ret = uncore_validate_group(pmu, event);
771         else
772                 ret = 0;
773 
774         return ret;
775 }
776 
777 static void uncore_pmu_enable(struct pmu *pmu)
778 {
779         struct intel_uncore_pmu *uncore_pmu;
780         struct intel_uncore_box *box;
781 
782         uncore_pmu = container_of(pmu, struct intel_uncore_pmu, pmu);
783         if (!uncore_pmu)
784                 return;
785 
786         box = uncore_pmu_to_box(uncore_pmu, smp_processor_id());
787         if (!box)
788                 return;
789 
790         if (uncore_pmu->type->ops->enable_box)
791                 uncore_pmu->type->ops->enable_box(box);
792 }
793 
794 static void uncore_pmu_disable(struct pmu *pmu)
795 {
796         struct intel_uncore_pmu *uncore_pmu;
797         struct intel_uncore_box *box;
798 
799         uncore_pmu = container_of(pmu, struct intel_uncore_pmu, pmu);
800         if (!uncore_pmu)
801                 return;
802 
803         box = uncore_pmu_to_box(uncore_pmu, smp_processor_id());
804         if (!box)
805                 return;
806 
807         if (uncore_pmu->type->ops->disable_box)
808                 uncore_pmu->type->ops->disable_box(box);
809 }
810 
811 static ssize_t uncore_get_attr_cpumask(struct device *dev,
812                                 struct device_attribute *attr, char *buf)
813 {
814         return cpumap_print_to_pagebuf(true, buf, &uncore_cpu_mask);
815 }
816 
817 static DEVICE_ATTR(cpumask, S_IRUGO, uncore_get_attr_cpumask, NULL);
818 
819 static struct attribute *uncore_pmu_attrs[] = {
820         &dev_attr_cpumask.attr,
821         NULL,
822 };
823 
824 static const struct attribute_group uncore_pmu_attr_group = {
825         .attrs = uncore_pmu_attrs,
826 };
827 
828 static int uncore_pmu_register(struct intel_uncore_pmu *pmu)
829 {
830         int ret;
831 
832         if (!pmu->type->pmu) {
833                 pmu->pmu = (struct pmu) {
834                         .attr_groups    = pmu->type->attr_groups,
835                         .task_ctx_nr    = perf_invalid_context,
836                         .pmu_enable     = uncore_pmu_enable,
837                         .pmu_disable    = uncore_pmu_disable,
838                         .event_init     = uncore_pmu_event_init,
839                         .add            = uncore_pmu_event_add,
840                         .del            = uncore_pmu_event_del,
841                         .start          = uncore_pmu_event_start,
842                         .stop           = uncore_pmu_event_stop,
843                         .read           = uncore_pmu_event_read,
844                         .module         = THIS_MODULE,
845                         .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
846                 };
847         } else {
848                 pmu->pmu = *pmu->type->pmu;
849                 pmu->pmu.attr_groups = pmu->type->attr_groups;
850         }
851 
852         if (pmu->type->num_boxes == 1) {
853                 if (strlen(pmu->type->name) > 0)
854                         sprintf(pmu->name, "uncore_%s", pmu->type->name);
855                 else
856                         sprintf(pmu->name, "uncore");
857         } else {
858                 sprintf(pmu->name, "uncore_%s_%d", pmu->type->name,
859                         pmu->pmu_idx);
860         }
861 
862         ret = perf_pmu_register(&pmu->pmu, pmu->name, -1);
863         if (!ret)
864                 pmu->registered = true;
865         return ret;
866 }
867 
868 static void uncore_pmu_unregister(struct intel_uncore_pmu *pmu)
869 {
870         if (!pmu->registered)
871                 return;
872         perf_pmu_unregister(&pmu->pmu);
873         pmu->registered = false;
874 }
875 
876 static void uncore_free_boxes(struct intel_uncore_pmu *pmu)
877 {
878         int die;
879 
880         for (die = 0; die < max_dies; die++)
881                 kfree(pmu->boxes[die]);
882         kfree(pmu->boxes);
883 }
884 
885 static void uncore_type_exit(struct intel_uncore_type *type)
886 {
887         struct intel_uncore_pmu *pmu = type->pmus;
888         int i;
889 
890         if (pmu) {
891                 for (i = 0; i < type->num_boxes; i++, pmu++) {
892                         uncore_pmu_unregister(pmu);
893                         uncore_free_boxes(pmu);
894                 }
895                 kfree(type->pmus);
896                 type->pmus = NULL;
897         }
898         kfree(type->events_group);
899         type->events_group = NULL;
900 }
901 
902 static void uncore_types_exit(struct intel_uncore_type **types)
903 {
904         for (; *types; types++)
905                 uncore_type_exit(*types);
906 }
907 
908 static int __init uncore_type_init(struct intel_uncore_type *type, bool setid)
909 {
910         struct intel_uncore_pmu *pmus;
911         size_t size;
912         int i, j;
913 
914         pmus = kcalloc(type->num_boxes, sizeof(*pmus), GFP_KERNEL);
915         if (!pmus)
916                 return -ENOMEM;
917 
918         size = max_dies * sizeof(struct intel_uncore_box *);
919 
920         for (i = 0; i < type->num_boxes; i++) {
921                 pmus[i].func_id = setid ? i : -1;
922                 pmus[i].pmu_idx = i;
923                 pmus[i].type    = type;
924                 pmus[i].boxes   = kzalloc(size, GFP_KERNEL);
925                 if (!pmus[i].boxes)
926                         goto err;
927         }
928 
929         type->pmus = pmus;
930         type->unconstrainted = (struct event_constraint)
931                 __EVENT_CONSTRAINT(0, (1ULL << type->num_counters) - 1,
932                                 0, type->num_counters, 0, 0);
933 
934         if (type->event_descs) {
935                 struct {
936                         struct attribute_group group;
937                         struct attribute *attrs[];
938                 } *attr_group;
939                 for (i = 0; type->event_descs[i].attr.attr.name; i++);
940 
941                 attr_group = kzalloc(struct_size(attr_group, attrs, i + 1),
942                                                                 GFP_KERNEL);
943                 if (!attr_group)
944                         goto err;
945 
946                 attr_group->group.name = "events";
947                 attr_group->group.attrs = attr_group->attrs;
948 
949                 for (j = 0; j < i; j++)
950                         attr_group->attrs[j] = &type->event_descs[j].attr.attr;
951 
952                 type->events_group = &attr_group->group;
953         }
954 
955         type->pmu_group = &uncore_pmu_attr_group;
956 
957         return 0;
958 
959 err:
960         for (i = 0; i < type->num_boxes; i++)
961                 kfree(pmus[i].boxes);
962         kfree(pmus);
963 
964         return -ENOMEM;
965 }
966 
967 static int __init
968 uncore_types_init(struct intel_uncore_type **types, bool setid)
969 {
970         int ret;
971 
972         for (; *types; types++) {
973                 ret = uncore_type_init(*types, setid);
974                 if (ret)
975                         return ret;
976         }
977         return 0;
978 }
979 
980 /*
981  * add a pci uncore device
982  */
983 static int uncore_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
984 {
985         struct intel_uncore_type *type;
986         struct intel_uncore_pmu *pmu = NULL;
987         struct intel_uncore_box *box;
988         int phys_id, die, ret;
989 
990         phys_id = uncore_pcibus_to_physid(pdev->bus);
991         if (phys_id < 0)
992                 return -ENODEV;
993 
994         die = (topology_max_die_per_package() > 1) ? phys_id :
995                                         topology_phys_to_logical_pkg(phys_id);
996         if (die < 0)
997                 return -EINVAL;
998 
999         if (UNCORE_PCI_DEV_TYPE(id->driver_data) == UNCORE_EXTRA_PCI_DEV) {
1000                 int idx = UNCORE_PCI_DEV_IDX(id->driver_data);
1001 
1002                 uncore_extra_pci_dev[die].dev[idx] = pdev;
1003                 pci_set_drvdata(pdev, NULL);
1004                 return 0;
1005         }
1006 
1007         type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(id->driver_data)];
1008 
1009         /*
1010          * Some platforms, e.g.  Knights Landing, use a common PCI device ID
1011          * for multiple instances of an uncore PMU device type. We should check
1012          * PCI slot and func to indicate the uncore box.
1013          */
1014         if (id->driver_data & ~0xffff) {
1015                 struct pci_driver *pci_drv = pdev->driver;
1016                 const struct pci_device_id *ids = pci_drv->id_table;
1017                 unsigned int devfn;
1018 
1019                 while (ids && ids->vendor) {
1020                         if ((ids->vendor == pdev->vendor) &&
1021                             (ids->device == pdev->device)) {
1022                                 devfn = PCI_DEVFN(UNCORE_PCI_DEV_DEV(ids->driver_data),
1023                                                   UNCORE_PCI_DEV_FUNC(ids->driver_data));
1024                                 if (devfn == pdev->devfn) {
1025                                         pmu = &type->pmus[UNCORE_PCI_DEV_IDX(ids->driver_data)];
1026                                         break;
1027                                 }
1028                         }
1029                         ids++;
1030                 }
1031                 if (pmu == NULL)
1032                         return -ENODEV;
1033         } else {
1034                 /*
1035                  * for performance monitoring unit with multiple boxes,
1036                  * each box has a different function id.
1037                  */
1038                 pmu = &type->pmus[UNCORE_PCI_DEV_IDX(id->driver_data)];
1039         }
1040 
1041         if (WARN_ON_ONCE(pmu->boxes[die] != NULL))
1042                 return -EINVAL;
1043 
1044         box = uncore_alloc_box(type, NUMA_NO_NODE);
1045         if (!box)
1046                 return -ENOMEM;
1047 
1048         if (pmu->func_id < 0)
1049                 pmu->func_id = pdev->devfn;
1050         else
1051                 WARN_ON_ONCE(pmu->func_id != pdev->devfn);
1052 
1053         atomic_inc(&box->refcnt);
1054         box->pci_phys_id = phys_id;
1055         box->dieid = die;
1056         box->pci_dev = pdev;
1057         box->pmu = pmu;
1058         uncore_box_init(box);
1059         pci_set_drvdata(pdev, box);
1060 
1061         pmu->boxes[die] = box;
1062         if (atomic_inc_return(&pmu->activeboxes) > 1)
1063                 return 0;
1064 
1065         /* First active box registers the pmu */
1066         ret = uncore_pmu_register(pmu);
1067         if (ret) {
1068                 pci_set_drvdata(pdev, NULL);
1069                 pmu->boxes[die] = NULL;
1070                 uncore_box_exit(box);
1071                 kfree(box);
1072         }
1073         return ret;
1074 }
1075 
1076 static void uncore_pci_remove(struct pci_dev *pdev)
1077 {
1078         struct intel_uncore_box *box;
1079         struct intel_uncore_pmu *pmu;
1080         int i, phys_id, die;
1081 
1082         phys_id = uncore_pcibus_to_physid(pdev->bus);
1083 
1084         box = pci_get_drvdata(pdev);
1085         if (!box) {
1086                 die = (topology_max_die_per_package() > 1) ? phys_id :
1087                                         topology_phys_to_logical_pkg(phys_id);
1088                 for (i = 0; i < UNCORE_EXTRA_PCI_DEV_MAX; i++) {
1089                         if (uncore_extra_pci_dev[die].dev[i] == pdev) {
1090                                 uncore_extra_pci_dev[die].dev[i] = NULL;
1091                                 break;
1092                         }
1093                 }
1094                 WARN_ON_ONCE(i >= UNCORE_EXTRA_PCI_DEV_MAX);
1095                 return;
1096         }
1097 
1098         pmu = box->pmu;
1099         if (WARN_ON_ONCE(phys_id != box->pci_phys_id))
1100                 return;
1101 
1102         pci_set_drvdata(pdev, NULL);
1103         pmu->boxes[box->dieid] = NULL;
1104         if (atomic_dec_return(&pmu->activeboxes) == 0)
1105                 uncore_pmu_unregister(pmu);
1106         uncore_box_exit(box);
1107         kfree(box);
1108 }
1109 
1110 static int __init uncore_pci_init(void)
1111 {
1112         size_t size;
1113         int ret;
1114 
1115         size = max_dies * sizeof(struct pci_extra_dev);
1116         uncore_extra_pci_dev = kzalloc(size, GFP_KERNEL);
1117         if (!uncore_extra_pci_dev) {
1118                 ret = -ENOMEM;
1119                 goto err;
1120         }
1121 
1122         ret = uncore_types_init(uncore_pci_uncores, false);
1123         if (ret)
1124                 goto errtype;
1125 
1126         uncore_pci_driver->probe = uncore_pci_probe;
1127         uncore_pci_driver->remove = uncore_pci_remove;
1128 
1129         ret = pci_register_driver(uncore_pci_driver);
1130         if (ret)
1131                 goto errtype;
1132 
1133         pcidrv_registered = true;
1134         return 0;
1135 
1136 errtype:
1137         uncore_types_exit(uncore_pci_uncores);
1138         kfree(uncore_extra_pci_dev);
1139         uncore_extra_pci_dev = NULL;
1140         uncore_free_pcibus_map();
1141 err:
1142         uncore_pci_uncores = empty_uncore;
1143         return ret;
1144 }
1145 
1146 static void uncore_pci_exit(void)
1147 {
1148         if (pcidrv_registered) {
1149                 pcidrv_registered = false;
1150                 pci_unregister_driver(uncore_pci_driver);
1151                 uncore_types_exit(uncore_pci_uncores);
1152                 kfree(uncore_extra_pci_dev);
1153                 uncore_free_pcibus_map();
1154         }
1155 }
1156 
1157 static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu,
1158                                    int new_cpu)
1159 {
1160         struct intel_uncore_pmu *pmu = type->pmus;
1161         struct intel_uncore_box *box;
1162         int i, die;
1163 
1164         die = topology_logical_die_id(old_cpu < 0 ? new_cpu : old_cpu);
1165         for (i = 0; i < type->num_boxes; i++, pmu++) {
1166                 box = pmu->boxes[die];
1167                 if (!box)
1168                         continue;
1169 
1170                 if (old_cpu < 0) {
1171                         WARN_ON_ONCE(box->cpu != -1);
1172                         box->cpu = new_cpu;
1173                         continue;
1174                 }
1175 
1176                 WARN_ON_ONCE(box->cpu != old_cpu);
1177                 box->cpu = -1;
1178                 if (new_cpu < 0)
1179                         continue;
1180 
1181                 uncore_pmu_cancel_hrtimer(box);
1182                 perf_pmu_migrate_context(&pmu->pmu, old_cpu, new_cpu);
1183                 box->cpu = new_cpu;
1184         }
1185 }
1186 
1187 static void uncore_change_context(struct intel_uncore_type **uncores,
1188                                   int old_cpu, int new_cpu)
1189 {
1190         for (; *uncores; uncores++)
1191                 uncore_change_type_ctx(*uncores, old_cpu, new_cpu);
1192 }
1193 
1194 static void uncore_box_unref(struct intel_uncore_type **types, int id)
1195 {
1196         struct intel_uncore_type *type;
1197         struct intel_uncore_pmu *pmu;
1198         struct intel_uncore_box *box;
1199         int i;
1200 
1201         for (; *types; types++) {
1202                 type = *types;
1203                 pmu = type->pmus;
1204                 for (i = 0; i < type->num_boxes; i++, pmu++) {
1205                         box = pmu->boxes[id];
1206                         if (box && atomic_dec_return(&box->refcnt) == 0)
1207                                 uncore_box_exit(box);
1208                 }
1209         }
1210 }
1211 
1212 static int uncore_event_cpu_offline(unsigned int cpu)
1213 {
1214         int die, target;
1215 
1216         /* Check if exiting cpu is used for collecting uncore events */
1217         if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
1218                 goto unref;
1219         /* Find a new cpu to collect uncore events */
1220         target = cpumask_any_but(topology_die_cpumask(cpu), cpu);
1221 
1222         /* Migrate uncore events to the new target */
1223         if (target < nr_cpu_ids)
1224                 cpumask_set_cpu(target, &uncore_cpu_mask);
1225         else
1226                 target = -1;
1227 
1228         uncore_change_context(uncore_msr_uncores, cpu, target);
1229         uncore_change_context(uncore_mmio_uncores, cpu, target);
1230         uncore_change_context(uncore_pci_uncores, cpu, target);
1231 
1232 unref:
1233         /* Clear the references */
1234         die = topology_logical_die_id(cpu);
1235         uncore_box_unref(uncore_msr_uncores, die);
1236         uncore_box_unref(uncore_mmio_uncores, die);
1237         return 0;
1238 }
1239 
1240 static int allocate_boxes(struct intel_uncore_type **types,
1241                          unsigned int die, unsigned int cpu)
1242 {
1243         struct intel_uncore_box *box, *tmp;
1244         struct intel_uncore_type *type;
1245         struct intel_uncore_pmu *pmu;
1246         LIST_HEAD(allocated);
1247         int i;
1248 
1249         /* Try to allocate all required boxes */
1250         for (; *types; types++) {
1251                 type = *types;
1252                 pmu = type->pmus;
1253                 for (i = 0; i < type->num_boxes; i++, pmu++) {
1254                         if (pmu->boxes[die])
1255                                 continue;
1256                         box = uncore_alloc_box(type, cpu_to_node(cpu));
1257                         if (!box)
1258                                 goto cleanup;
1259                         box->pmu = pmu;
1260                         box->dieid = die;
1261                         list_add(&box->active_list, &allocated);
1262                 }
1263         }
1264         /* Install them in the pmus */
1265         list_for_each_entry_safe(box, tmp, &allocated, active_list) {
1266                 list_del_init(&box->active_list);
1267                 box->pmu->boxes[die] = box;
1268         }
1269         return 0;
1270 
1271 cleanup:
1272         list_for_each_entry_safe(box, tmp, &allocated, active_list) {
1273                 list_del_init(&box->active_list);
1274                 kfree(box);
1275         }
1276         return -ENOMEM;
1277 }
1278 
1279 static int uncore_box_ref(struct intel_uncore_type **types,
1280                           int id, unsigned int cpu)
1281 {
1282         struct intel_uncore_type *type;
1283         struct intel_uncore_pmu *pmu;
1284         struct intel_uncore_box *box;
1285         int i, ret;
1286 
1287         ret = allocate_boxes(types, id, cpu);
1288         if (ret)
1289                 return ret;
1290 
1291         for (; *types; types++) {
1292                 type = *types;
1293                 pmu = type->pmus;
1294                 for (i = 0; i < type->num_boxes; i++, pmu++) {
1295                         box = pmu->boxes[id];
1296                         if (box && atomic_inc_return(&box->refcnt) == 1)
1297                                 uncore_box_init(box);
1298                 }
1299         }
1300         return 0;
1301 }
1302 
1303 static int uncore_event_cpu_online(unsigned int cpu)
1304 {
1305         int die, target, msr_ret, mmio_ret;
1306 
1307         die = topology_logical_die_id(cpu);
1308         msr_ret = uncore_box_ref(uncore_msr_uncores, die, cpu);
1309         mmio_ret = uncore_box_ref(uncore_mmio_uncores, die, cpu);
1310         if (msr_ret && mmio_ret)
1311                 return -ENOMEM;
1312 
1313         /*
1314          * Check if there is an online cpu in the package
1315          * which collects uncore events already.
1316          */
1317         target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu));
1318         if (target < nr_cpu_ids)
1319                 return 0;
1320 
1321         cpumask_set_cpu(cpu, &uncore_cpu_mask);
1322 
1323         if (!msr_ret)
1324                 uncore_change_context(uncore_msr_uncores, -1, cpu);
1325         if (!mmio_ret)
1326                 uncore_change_context(uncore_mmio_uncores, -1, cpu);
1327         uncore_change_context(uncore_pci_uncores, -1, cpu);
1328         return 0;
1329 }
1330 
1331 static int __init type_pmu_register(struct intel_uncore_type *type)
1332 {
1333         int i, ret;
1334 
1335         for (i = 0; i < type->num_boxes; i++) {
1336                 ret = uncore_pmu_register(&type->pmus[i]);
1337                 if (ret)
1338                         return ret;
1339         }
1340         return 0;
1341 }
1342 
1343 static int __init uncore_msr_pmus_register(void)
1344 {
1345         struct intel_uncore_type **types = uncore_msr_uncores;
1346         int ret;
1347 
1348         for (; *types; types++) {
1349                 ret = type_pmu_register(*types);
1350                 if (ret)
1351                         return ret;
1352         }
1353         return 0;
1354 }
1355 
1356 static int __init uncore_cpu_init(void)
1357 {
1358         int ret;
1359 
1360         ret = uncore_types_init(uncore_msr_uncores, true);
1361         if (ret)
1362                 goto err;
1363 
1364         ret = uncore_msr_pmus_register();
1365         if (ret)
1366                 goto err;
1367         return 0;
1368 err:
1369         uncore_types_exit(uncore_msr_uncores);
1370         uncore_msr_uncores = empty_uncore;
1371         return ret;
1372 }
1373 
1374 static int __init uncore_mmio_init(void)
1375 {
1376         struct intel_uncore_type **types = uncore_mmio_uncores;
1377         int ret;
1378 
1379         ret = uncore_types_init(types, true);
1380         if (ret)
1381                 goto err;
1382 
1383         for (; *types; types++) {
1384                 ret = type_pmu_register(*types);
1385                 if (ret)
1386                         goto err;
1387         }
1388         return 0;
1389 err:
1390         uncore_types_exit(uncore_mmio_uncores);
1391         uncore_mmio_uncores = empty_uncore;
1392         return ret;
1393 }
1394 
1395 
1396 #define X86_UNCORE_MODEL_MATCH(model, init)     \
1397         { X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long)&init }
1398 
1399 struct intel_uncore_init_fun {
1400         void    (*cpu_init)(void);
1401         int     (*pci_init)(void);
1402         void    (*mmio_init)(void);
1403 };
1404 
1405 static const struct intel_uncore_init_fun nhm_uncore_init __initconst = {
1406         .cpu_init = nhm_uncore_cpu_init,
1407 };
1408 
1409 static const struct intel_uncore_init_fun snb_uncore_init __initconst = {
1410         .cpu_init = snb_uncore_cpu_init,
1411         .pci_init = snb_uncore_pci_init,
1412 };
1413 
1414 static const struct intel_uncore_init_fun ivb_uncore_init __initconst = {
1415         .cpu_init = snb_uncore_cpu_init,
1416         .pci_init = ivb_uncore_pci_init,
1417 };
1418 
1419 static const struct intel_uncore_init_fun hsw_uncore_init __initconst = {
1420         .cpu_init = snb_uncore_cpu_init,
1421         .pci_init = hsw_uncore_pci_init,
1422 };
1423 
1424 static const struct intel_uncore_init_fun bdw_uncore_init __initconst = {
1425         .cpu_init = snb_uncore_cpu_init,
1426         .pci_init = bdw_uncore_pci_init,
1427 };
1428 
1429 static const struct intel_uncore_init_fun snbep_uncore_init __initconst = {
1430         .cpu_init = snbep_uncore_cpu_init,
1431         .pci_init = snbep_uncore_pci_init,
1432 };
1433 
1434 static const struct intel_uncore_init_fun nhmex_uncore_init __initconst = {
1435         .cpu_init = nhmex_uncore_cpu_init,
1436 };
1437 
1438 static const struct intel_uncore_init_fun ivbep_uncore_init __initconst = {
1439         .cpu_init = ivbep_uncore_cpu_init,
1440         .pci_init = ivbep_uncore_pci_init,
1441 };
1442 
1443 static const struct intel_uncore_init_fun hswep_uncore_init __initconst = {
1444         .cpu_init = hswep_uncore_cpu_init,
1445         .pci_init = hswep_uncore_pci_init,
1446 };
1447 
1448 static const struct intel_uncore_init_fun bdx_uncore_init __initconst = {
1449         .cpu_init = bdx_uncore_cpu_init,
1450         .pci_init = bdx_uncore_pci_init,
1451 };
1452 
1453 static const struct intel_uncore_init_fun knl_uncore_init __initconst = {
1454         .cpu_init = knl_uncore_cpu_init,
1455         .pci_init = knl_uncore_pci_init,
1456 };
1457 
1458 static const struct intel_uncore_init_fun skl_uncore_init __initconst = {
1459         .cpu_init = skl_uncore_cpu_init,
1460         .pci_init = skl_uncore_pci_init,
1461 };
1462 
1463 static const struct intel_uncore_init_fun skx_uncore_init __initconst = {
1464         .cpu_init = skx_uncore_cpu_init,
1465         .pci_init = skx_uncore_pci_init,
1466 };
1467 
1468 static const struct intel_uncore_init_fun icl_uncore_init __initconst = {
1469         .cpu_init = icl_uncore_cpu_init,
1470         .pci_init = skl_uncore_pci_init,
1471 };
1472 
1473 static const struct intel_uncore_init_fun snr_uncore_init __initconst = {
1474         .cpu_init = snr_uncore_cpu_init,
1475         .pci_init = snr_uncore_pci_init,
1476         .mmio_init = snr_uncore_mmio_init,
1477 };
1478 
1479 static const struct x86_cpu_id intel_uncore_match[] __initconst = {
1480         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_NEHALEM_EP,     nhm_uncore_init),
1481         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_NEHALEM,        nhm_uncore_init),
1482         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_WESTMERE,       nhm_uncore_init),
1483         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_WESTMERE_EP,    nhm_uncore_init),
1484         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SANDYBRIDGE,    snb_uncore_init),
1485         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_IVYBRIDGE,      ivb_uncore_init),
1486         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL,        hsw_uncore_init),
1487         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL_L,      hsw_uncore_init),
1488         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL_G,      hsw_uncore_init),
1489         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL,      bdw_uncore_init),
1490         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL_G,    bdw_uncore_init),
1491         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SANDYBRIDGE_X,  snbep_uncore_init),
1492         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_NEHALEM_EX,     nhmex_uncore_init),
1493         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_WESTMERE_EX,    nhmex_uncore_init),
1494         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_IVYBRIDGE_X,    ivbep_uncore_init),
1495         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL_X,      hswep_uncore_init),
1496         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL_X,    bdx_uncore_init),
1497         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL_D,    bdx_uncore_init),
1498         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_XEON_PHI_KNL,   knl_uncore_init),
1499         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_XEON_PHI_KNM,   knl_uncore_init),
1500         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SKYLAKE,        skl_uncore_init),
1501         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SKYLAKE_L,      skl_uncore_init),
1502         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SKYLAKE_X,      skx_uncore_init),
1503         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_KABYLAKE_L,     skl_uncore_init),
1504         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_KABYLAKE,       skl_uncore_init),
1505         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_ICELAKE_L,      icl_uncore_init),
1506         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_ICELAKE_NNPI,   icl_uncore_init),
1507         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_ICELAKE,        icl_uncore_init),
1508         X86_UNCORE_MODEL_MATCH(INTEL_FAM6_ATOM_TREMONT_D, snr_uncore_init),
1509         {},
1510 };
1511 
1512 MODULE_DEVICE_TABLE(x86cpu, intel_uncore_match);
1513 
1514 static int __init intel_uncore_init(void)
1515 {
1516         const struct x86_cpu_id *id;
1517         struct intel_uncore_init_fun *uncore_init;
1518         int pret = 0, cret = 0, mret = 0, ret;
1519 
1520         id = x86_match_cpu(intel_uncore_match);
1521         if (!id)
1522                 return -ENODEV;
1523 
1524         if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
1525                 return -ENODEV;
1526 
1527         max_dies = topology_max_packages() * topology_max_die_per_package();
1528 
1529         uncore_init = (struct intel_uncore_init_fun *)id->driver_data;
1530         if (uncore_init->pci_init) {
1531                 pret = uncore_init->pci_init();
1532                 if (!pret)
1533                         pret = uncore_pci_init();
1534         }
1535 
1536         if (uncore_init->cpu_init) {
1537                 uncore_init->cpu_init();
1538                 cret = uncore_cpu_init();
1539         }
1540 
1541         if (uncore_init->mmio_init) {
1542                 uncore_init->mmio_init();
1543                 mret = uncore_mmio_init();
1544         }
1545 
1546         if (cret && pret && mret)
1547                 return -ENODEV;
1548 
1549         /* Install hotplug callbacks to setup the targets for each package */
1550         ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_UNCORE_ONLINE,
1551                                 "perf/x86/intel/uncore:online",
1552                                 uncore_event_cpu_online,
1553                                 uncore_event_cpu_offline);
1554         if (ret)
1555                 goto err;
1556         return 0;
1557 
1558 err:
1559         uncore_types_exit(uncore_msr_uncores);
1560         uncore_types_exit(uncore_mmio_uncores);
1561         uncore_pci_exit();
1562         return ret;
1563 }
1564 module_init(intel_uncore_init);
1565 
1566 static void __exit intel_uncore_exit(void)
1567 {
1568         cpuhp_remove_state(CPUHP_AP_PERF_X86_UNCORE_ONLINE);
1569         uncore_types_exit(uncore_msr_uncores);
1570         uncore_types_exit(uncore_mmio_uncores);
1571         uncore_pci_exit();
1572 }
1573 module_exit(intel_uncore_exit);
1574 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp