~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/powerpc/perf/core-fsl-emb.c

Version: ~ [ linux-5.4-rc7 ] ~ [ linux-5.3.11 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.84 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.154 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.201 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.201 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.77 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Performance event support - Freescale Embedded Performance Monitor
  3  *
  4  * Copyright 2008-2009 Paul Mackerras, IBM Corporation.
  5  * Copyright 2010 Freescale Semiconductor, Inc.
  6  *
  7  * This program is free software; you can redistribute it and/or
  8  * modify it under the terms of the GNU General Public License
  9  * as published by the Free Software Foundation; either version
 10  * 2 of the License, or (at your option) any later version.
 11  */
 12 #include <linux/kernel.h>
 13 #include <linux/sched.h>
 14 #include <linux/perf_event.h>
 15 #include <linux/percpu.h>
 16 #include <linux/hardirq.h>
 17 #include <asm/reg_fsl_emb.h>
 18 #include <asm/pmc.h>
 19 #include <asm/machdep.h>
 20 #include <asm/firmware.h>
 21 #include <asm/ptrace.h>
 22 
 23 struct cpu_hw_events {
 24         int n_events;
 25         int disabled;
 26         u8  pmcs_enabled;
 27         struct perf_event *event[MAX_HWEVENTS];
 28 };
 29 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
 30 
 31 static struct fsl_emb_pmu *ppmu;
 32 
 33 /* Number of perf_events counting hardware events */
 34 static atomic_t num_events;
 35 /* Used to avoid races in calling reserve/release_pmc_hardware */
 36 static DEFINE_MUTEX(pmc_reserve_mutex);
 37 
 38 /*
 39  * If interrupts were soft-disabled when a PMU interrupt occurs, treat
 40  * it as an NMI.
 41  */
 42 static inline int perf_intr_is_nmi(struct pt_regs *regs)
 43 {
 44 #ifdef __powerpc64__
 45         return !regs->softe;
 46 #else
 47         return 0;
 48 #endif
 49 }
 50 
 51 static void perf_event_interrupt(struct pt_regs *regs);
 52 
 53 /*
 54  * Read one performance monitor counter (PMC).
 55  */
 56 static unsigned long read_pmc(int idx)
 57 {
 58         unsigned long val;
 59 
 60         switch (idx) {
 61         case 0:
 62                 val = mfpmr(PMRN_PMC0);
 63                 break;
 64         case 1:
 65                 val = mfpmr(PMRN_PMC1);
 66                 break;
 67         case 2:
 68                 val = mfpmr(PMRN_PMC2);
 69                 break;
 70         case 3:
 71                 val = mfpmr(PMRN_PMC3);
 72                 break;
 73         default:
 74                 printk(KERN_ERR "oops trying to read PMC%d\n", idx);
 75                 val = 0;
 76         }
 77         return val;
 78 }
 79 
 80 /*
 81  * Write one PMC.
 82  */
 83 static void write_pmc(int idx, unsigned long val)
 84 {
 85         switch (idx) {
 86         case 0:
 87                 mtpmr(PMRN_PMC0, val);
 88                 break;
 89         case 1:
 90                 mtpmr(PMRN_PMC1, val);
 91                 break;
 92         case 2:
 93                 mtpmr(PMRN_PMC2, val);
 94                 break;
 95         case 3:
 96                 mtpmr(PMRN_PMC3, val);
 97                 break;
 98         default:
 99                 printk(KERN_ERR "oops trying to write PMC%d\n", idx);
100         }
101 
102         isync();
103 }
104 
105 /*
106  * Write one local control A register
107  */
108 static void write_pmlca(int idx, unsigned long val)
109 {
110         switch (idx) {
111         case 0:
112                 mtpmr(PMRN_PMLCA0, val);
113                 break;
114         case 1:
115                 mtpmr(PMRN_PMLCA1, val);
116                 break;
117         case 2:
118                 mtpmr(PMRN_PMLCA2, val);
119                 break;
120         case 3:
121                 mtpmr(PMRN_PMLCA3, val);
122                 break;
123         default:
124                 printk(KERN_ERR "oops trying to write PMLCA%d\n", idx);
125         }
126 
127         isync();
128 }
129 
130 /*
131  * Write one local control B register
132  */
133 static void write_pmlcb(int idx, unsigned long val)
134 {
135         switch (idx) {
136         case 0:
137                 mtpmr(PMRN_PMLCB0, val);
138                 break;
139         case 1:
140                 mtpmr(PMRN_PMLCB1, val);
141                 break;
142         case 2:
143                 mtpmr(PMRN_PMLCB2, val);
144                 break;
145         case 3:
146                 mtpmr(PMRN_PMLCB3, val);
147                 break;
148         default:
149                 printk(KERN_ERR "oops trying to write PMLCB%d\n", idx);
150         }
151 
152         isync();
153 }
154 
155 static void fsl_emb_pmu_read(struct perf_event *event)
156 {
157         s64 val, delta, prev;
158 
159         if (event->hw.state & PERF_HES_STOPPED)
160                 return;
161 
162         /*
163          * Performance monitor interrupts come even when interrupts
164          * are soft-disabled, as long as interrupts are hard-enabled.
165          * Therefore we treat them like NMIs.
166          */
167         do {
168                 prev = local64_read(&event->hw.prev_count);
169                 barrier();
170                 val = read_pmc(event->hw.idx);
171         } while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev);
172 
173         /* The counters are only 32 bits wide */
174         delta = (val - prev) & 0xfffffffful;
175         local64_add(delta, &event->count);
176         local64_sub(delta, &event->hw.period_left);
177 }
178 
179 /*
180  * Disable all events to prevent PMU interrupts and to allow
181  * events to be added or removed.
182  */
183 static void fsl_emb_pmu_disable(struct pmu *pmu)
184 {
185         struct cpu_hw_events *cpuhw;
186         unsigned long flags;
187 
188         local_irq_save(flags);
189         cpuhw = &__get_cpu_var(cpu_hw_events);
190 
191         if (!cpuhw->disabled) {
192                 cpuhw->disabled = 1;
193 
194                 /*
195                  * Check if we ever enabled the PMU on this cpu.
196                  */
197                 if (!cpuhw->pmcs_enabled) {
198                         ppc_enable_pmcs();
199                         cpuhw->pmcs_enabled = 1;
200                 }
201 
202                 if (atomic_read(&num_events)) {
203                         /*
204                          * Set the 'freeze all counters' bit, and disable
205                          * interrupts.  The barrier is to make sure the
206                          * mtpmr has been executed and the PMU has frozen
207                          * the events before we return.
208                          */
209 
210                         mtpmr(PMRN_PMGC0, PMGC0_FAC);
211                         isync();
212                 }
213         }
214         local_irq_restore(flags);
215 }
216 
217 /*
218  * Re-enable all events if disable == 0.
219  * If we were previously disabled and events were added, then
220  * put the new config on the PMU.
221  */
222 static void fsl_emb_pmu_enable(struct pmu *pmu)
223 {
224         struct cpu_hw_events *cpuhw;
225         unsigned long flags;
226 
227         local_irq_save(flags);
228         cpuhw = &__get_cpu_var(cpu_hw_events);
229         if (!cpuhw->disabled)
230                 goto out;
231 
232         cpuhw->disabled = 0;
233         ppc_set_pmu_inuse(cpuhw->n_events != 0);
234 
235         if (cpuhw->n_events > 0) {
236                 mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);
237                 isync();
238         }
239 
240  out:
241         local_irq_restore(flags);
242 }
243 
244 static int collect_events(struct perf_event *group, int max_count,
245                           struct perf_event *ctrs[])
246 {
247         int n = 0;
248         struct perf_event *event;
249 
250         if (!is_software_event(group)) {
251                 if (n >= max_count)
252                         return -1;
253                 ctrs[n] = group;
254                 n++;
255         }
256         list_for_each_entry(event, &group->sibling_list, group_entry) {
257                 if (!is_software_event(event) &&
258                     event->state != PERF_EVENT_STATE_OFF) {
259                         if (n >= max_count)
260                                 return -1;
261                         ctrs[n] = event;
262                         n++;
263                 }
264         }
265         return n;
266 }
267 
268 /* context locked on entry */
269 static int fsl_emb_pmu_add(struct perf_event *event, int flags)
270 {
271         struct cpu_hw_events *cpuhw;
272         int ret = -EAGAIN;
273         int num_counters = ppmu->n_counter;
274         u64 val;
275         int i;
276 
277         perf_pmu_disable(event->pmu);
278         cpuhw = &get_cpu_var(cpu_hw_events);
279 
280         if (event->hw.config & FSL_EMB_EVENT_RESTRICTED)
281                 num_counters = ppmu->n_restricted;
282 
283         /*
284          * Allocate counters from top-down, so that restricted-capable
285          * counters are kept free as long as possible.
286          */
287         for (i = num_counters - 1; i >= 0; i--) {
288                 if (cpuhw->event[i])
289                         continue;
290 
291                 break;
292         }
293 
294         if (i < 0)
295                 goto out;
296 
297         event->hw.idx = i;
298         cpuhw->event[i] = event;
299         ++cpuhw->n_events;
300 
301         val = 0;
302         if (event->hw.sample_period) {
303                 s64 left = local64_read(&event->hw.period_left);
304                 if (left < 0x80000000L)
305                         val = 0x80000000L - left;
306         }
307         local64_set(&event->hw.prev_count, val);
308 
309         if (!(flags & PERF_EF_START)) {
310                 event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
311                 val = 0;
312         }
313 
314         write_pmc(i, val);
315         perf_event_update_userpage(event);
316 
317         write_pmlcb(i, event->hw.config >> 32);
318         write_pmlca(i, event->hw.config_base);
319 
320         ret = 0;
321  out:
322         put_cpu_var(cpu_hw_events);
323         perf_pmu_enable(event->pmu);
324         return ret;
325 }
326 
327 /* context locked on entry */
328 static void fsl_emb_pmu_del(struct perf_event *event, int flags)
329 {
330         struct cpu_hw_events *cpuhw;
331         int i = event->hw.idx;
332 
333         perf_pmu_disable(event->pmu);
334         if (i < 0)
335                 goto out;
336 
337         fsl_emb_pmu_read(event);
338 
339         cpuhw = &get_cpu_var(cpu_hw_events);
340 
341         WARN_ON(event != cpuhw->event[event->hw.idx]);
342 
343         write_pmlca(i, 0);
344         write_pmlcb(i, 0);
345         write_pmc(i, 0);
346 
347         cpuhw->event[i] = NULL;
348         event->hw.idx = -1;
349 
350         /*
351          * TODO: if at least one restricted event exists, and we
352          * just freed up a non-restricted-capable counter, and
353          * there is a restricted-capable counter occupied by
354          * a non-restricted event, migrate that event to the
355          * vacated counter.
356          */
357 
358         cpuhw->n_events--;
359 
360  out:
361         perf_pmu_enable(event->pmu);
362         put_cpu_var(cpu_hw_events);
363 }
364 
365 static void fsl_emb_pmu_start(struct perf_event *event, int ef_flags)
366 {
367         unsigned long flags;
368         s64 left;
369 
370         if (event->hw.idx < 0 || !event->hw.sample_period)
371                 return;
372 
373         if (!(event->hw.state & PERF_HES_STOPPED))
374                 return;
375 
376         if (ef_flags & PERF_EF_RELOAD)
377                 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
378 
379         local_irq_save(flags);
380         perf_pmu_disable(event->pmu);
381 
382         event->hw.state = 0;
383         left = local64_read(&event->hw.period_left);
384         write_pmc(event->hw.idx, left);
385 
386         perf_event_update_userpage(event);
387         perf_pmu_enable(event->pmu);
388         local_irq_restore(flags);
389 }
390 
391 static void fsl_emb_pmu_stop(struct perf_event *event, int ef_flags)
392 {
393         unsigned long flags;
394 
395         if (event->hw.idx < 0 || !event->hw.sample_period)
396                 return;
397 
398         if (event->hw.state & PERF_HES_STOPPED)
399                 return;
400 
401         local_irq_save(flags);
402         perf_pmu_disable(event->pmu);
403 
404         fsl_emb_pmu_read(event);
405         event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
406         write_pmc(event->hw.idx, 0);
407 
408         perf_event_update_userpage(event);
409         perf_pmu_enable(event->pmu);
410         local_irq_restore(flags);
411 }
412 
413 /*
414  * Release the PMU if this is the last perf_event.
415  */
416 static void hw_perf_event_destroy(struct perf_event *event)
417 {
418         if (!atomic_add_unless(&num_events, -1, 1)) {
419                 mutex_lock(&pmc_reserve_mutex);
420                 if (atomic_dec_return(&num_events) == 0)
421                         release_pmc_hardware();
422                 mutex_unlock(&pmc_reserve_mutex);
423         }
424 }
425 
426 /*
427  * Translate a generic cache event_id config to a raw event_id code.
428  */
429 static int hw_perf_cache_event(u64 config, u64 *eventp)
430 {
431         unsigned long type, op, result;
432         int ev;
433 
434         if (!ppmu->cache_events)
435                 return -EINVAL;
436 
437         /* unpack config */
438         type = config & 0xff;
439         op = (config >> 8) & 0xff;
440         result = (config >> 16) & 0xff;
441 
442         if (type >= PERF_COUNT_HW_CACHE_MAX ||
443             op >= PERF_COUNT_HW_CACHE_OP_MAX ||
444             result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
445                 return -EINVAL;
446 
447         ev = (*ppmu->cache_events)[type][op][result];
448         if (ev == 0)
449                 return -EOPNOTSUPP;
450         if (ev == -1)
451                 return -EINVAL;
452         *eventp = ev;
453         return 0;
454 }
455 
456 static int fsl_emb_pmu_event_init(struct perf_event *event)
457 {
458         u64 ev;
459         struct perf_event *events[MAX_HWEVENTS];
460         int n;
461         int err;
462         int num_restricted;
463         int i;
464 
465         switch (event->attr.type) {
466         case PERF_TYPE_HARDWARE:
467                 ev = event->attr.config;
468                 if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
469                         return -EOPNOTSUPP;
470                 ev = ppmu->generic_events[ev];
471                 break;
472 
473         case PERF_TYPE_HW_CACHE:
474                 err = hw_perf_cache_event(event->attr.config, &ev);
475                 if (err)
476                         return err;
477                 break;
478 
479         case PERF_TYPE_RAW:
480                 ev = event->attr.config;
481                 break;
482 
483         default:
484                 return -ENOENT;
485         }
486 
487         event->hw.config = ppmu->xlate_event(ev);
488         if (!(event->hw.config & FSL_EMB_EVENT_VALID))
489                 return -EINVAL;
490 
491         /*
492          * If this is in a group, check if it can go on with all the
493          * other hardware events in the group.  We assume the event
494          * hasn't been linked into its leader's sibling list at this point.
495          */
496         n = 0;
497         if (event->group_leader != event) {
498                 n = collect_events(event->group_leader,
499                                    ppmu->n_counter - 1, events);
500                 if (n < 0)
501                         return -EINVAL;
502         }
503 
504         if (event->hw.config & FSL_EMB_EVENT_RESTRICTED) {
505                 num_restricted = 0;
506                 for (i = 0; i < n; i++) {
507                         if (events[i]->hw.config & FSL_EMB_EVENT_RESTRICTED)
508                                 num_restricted++;
509                 }
510 
511                 if (num_restricted >= ppmu->n_restricted)
512                         return -EINVAL;
513         }
514 
515         event->hw.idx = -1;
516 
517         event->hw.config_base = PMLCA_CE | PMLCA_FCM1 |
518                                 (u32)((ev << 16) & PMLCA_EVENT_MASK);
519 
520         if (event->attr.exclude_user)
521                 event->hw.config_base |= PMLCA_FCU;
522         if (event->attr.exclude_kernel)
523                 event->hw.config_base |= PMLCA_FCS;
524         if (event->attr.exclude_idle)
525                 return -ENOTSUPP;
526 
527         event->hw.last_period = event->hw.sample_period;
528         local64_set(&event->hw.period_left, event->hw.last_period);
529 
530         /*
531          * See if we need to reserve the PMU.
532          * If no events are currently in use, then we have to take a
533          * mutex to ensure that we don't race with another task doing
534          * reserve_pmc_hardware or release_pmc_hardware.
535          */
536         err = 0;
537         if (!atomic_inc_not_zero(&num_events)) {
538                 mutex_lock(&pmc_reserve_mutex);
539                 if (atomic_read(&num_events) == 0 &&
540                     reserve_pmc_hardware(perf_event_interrupt))
541                         err = -EBUSY;
542                 else
543                         atomic_inc(&num_events);
544                 mutex_unlock(&pmc_reserve_mutex);
545 
546                 mtpmr(PMRN_PMGC0, PMGC0_FAC);
547                 isync();
548         }
549         event->destroy = hw_perf_event_destroy;
550 
551         return err;
552 }
553 
554 static struct pmu fsl_emb_pmu = {
555         .pmu_enable     = fsl_emb_pmu_enable,
556         .pmu_disable    = fsl_emb_pmu_disable,
557         .event_init     = fsl_emb_pmu_event_init,
558         .add            = fsl_emb_pmu_add,
559         .del            = fsl_emb_pmu_del,
560         .start          = fsl_emb_pmu_start,
561         .stop           = fsl_emb_pmu_stop,
562         .read           = fsl_emb_pmu_read,
563 };
564 
565 /*
566  * A counter has overflowed; update its count and record
567  * things if requested.  Note that interrupts are hard-disabled
568  * here so there is no possibility of being interrupted.
569  */
570 static void record_and_restart(struct perf_event *event, unsigned long val,
571                                struct pt_regs *regs)
572 {
573         u64 period = event->hw.sample_period;
574         s64 prev, delta, left;
575         int record = 0;
576 
577         if (event->hw.state & PERF_HES_STOPPED) {
578                 write_pmc(event->hw.idx, 0);
579                 return;
580         }
581 
582         /* we don't have to worry about interrupts here */
583         prev = local64_read(&event->hw.prev_count);
584         delta = (val - prev) & 0xfffffffful;
585         local64_add(delta, &event->count);
586 
587         /*
588          * See if the total period for this event has expired,
589          * and update for the next period.
590          */
591         val = 0;
592         left = local64_read(&event->hw.period_left) - delta;
593         if (period) {
594                 if (left <= 0) {
595                         left += period;
596                         if (left <= 0)
597                                 left = period;
598                         record = 1;
599                         event->hw.last_period = event->hw.sample_period;
600                 }
601                 if (left < 0x80000000LL)
602                         val = 0x80000000LL - left;
603         }
604 
605         write_pmc(event->hw.idx, val);
606         local64_set(&event->hw.prev_count, val);
607         local64_set(&event->hw.period_left, left);
608         perf_event_update_userpage(event);
609 
610         /*
611          * Finally record data if requested.
612          */
613         if (record) {
614                 struct perf_sample_data data;
615 
616                 perf_sample_data_init(&data, 0, event->hw.last_period);
617 
618                 if (perf_event_overflow(event, &data, regs))
619                         fsl_emb_pmu_stop(event, 0);
620         }
621 }
622 
623 static void perf_event_interrupt(struct pt_regs *regs)
624 {
625         int i;
626         struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
627         struct perf_event *event;
628         unsigned long val;
629         int found = 0;
630         int nmi;
631 
632         nmi = perf_intr_is_nmi(regs);
633         if (nmi)
634                 nmi_enter();
635         else
636                 irq_enter();
637 
638         for (i = 0; i < ppmu->n_counter; ++i) {
639                 event = cpuhw->event[i];
640 
641                 val = read_pmc(i);
642                 if ((int)val < 0) {
643                         if (event) {
644                                 /* event has overflowed */
645                                 found = 1;
646                                 record_and_restart(event, val, regs);
647                         } else {
648                                 /*
649                                  * Disabled counter is negative,
650                                  * reset it just in case.
651                                  */
652                                 write_pmc(i, 0);
653                         }
654                 }
655         }
656 
657         /* PMM will keep counters frozen until we return from the interrupt. */
658         mtmsr(mfmsr() | MSR_PMM);
659         mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);
660         isync();
661 
662         if (nmi)
663                 nmi_exit();
664         else
665                 irq_exit();
666 }
667 
668 void hw_perf_event_setup(int cpu)
669 {
670         struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);
671 
672         memset(cpuhw, 0, sizeof(*cpuhw));
673 }
674 
675 int register_fsl_emb_pmu(struct fsl_emb_pmu *pmu)
676 {
677         if (ppmu)
678                 return -EBUSY;          /* something's already registered */
679 
680         ppmu = pmu;
681         pr_info("%s performance monitor hardware support registered\n",
682                 pmu->name);
683 
684         perf_pmu_register(&fsl_emb_pmu, "cpu", PERF_TYPE_RAW);
685 
686         return 0;
687 }
688 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp