~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm/kernel/perf_event_xscale.c

Version: ~ [ linux-5.4-rc3 ] ~ [ linux-5.3.6 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.79 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.149 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.196 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.196 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.75 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * ARMv5 [xscale] Performance counter handling code.
  3  *
  4  * Copyright (C) 2010, ARM Ltd., Will Deacon <will.deacon@arm.com>
  5  *
  6  * Based on the previous xscale OProfile code.
  7  *
  8  * There are two variants of the xscale PMU that we support:
  9  *      - xscale1pmu: 2 event counters and a cycle counter
 10  *      - xscale2pmu: 4 event counters and a cycle counter
 11  * The two variants share event definitions, but have different
 12  * PMU structures.
 13  */
 14 
 15 #ifdef CONFIG_CPU_XSCALE
 16 enum xscale_perf_types {
 17         XSCALE_PERFCTR_ICACHE_MISS              = 0x00,
 18         XSCALE_PERFCTR_ICACHE_NO_DELIVER        = 0x01,
 19         XSCALE_PERFCTR_DATA_STALL               = 0x02,
 20         XSCALE_PERFCTR_ITLB_MISS                = 0x03,
 21         XSCALE_PERFCTR_DTLB_MISS                = 0x04,
 22         XSCALE_PERFCTR_BRANCH                   = 0x05,
 23         XSCALE_PERFCTR_BRANCH_MISS              = 0x06,
 24         XSCALE_PERFCTR_INSTRUCTION              = 0x07,
 25         XSCALE_PERFCTR_DCACHE_FULL_STALL        = 0x08,
 26         XSCALE_PERFCTR_DCACHE_FULL_STALL_CONTIG = 0x09,
 27         XSCALE_PERFCTR_DCACHE_ACCESS            = 0x0A,
 28         XSCALE_PERFCTR_DCACHE_MISS              = 0x0B,
 29         XSCALE_PERFCTR_DCACHE_WRITE_BACK        = 0x0C,
 30         XSCALE_PERFCTR_PC_CHANGED               = 0x0D,
 31         XSCALE_PERFCTR_BCU_REQUEST              = 0x10,
 32         XSCALE_PERFCTR_BCU_FULL                 = 0x11,
 33         XSCALE_PERFCTR_BCU_DRAIN                = 0x12,
 34         XSCALE_PERFCTR_BCU_ECC_NO_ELOG          = 0x14,
 35         XSCALE_PERFCTR_BCU_1_BIT_ERR            = 0x15,
 36         XSCALE_PERFCTR_RMW                      = 0x16,
 37         /* XSCALE_PERFCTR_CCNT is not hardware defined */
 38         XSCALE_PERFCTR_CCNT                     = 0xFE,
 39         XSCALE_PERFCTR_UNUSED                   = 0xFF,
 40 };
 41 
 42 enum xscale_counters {
 43         XSCALE_CYCLE_COUNTER    = 0,
 44         XSCALE_COUNTER0,
 45         XSCALE_COUNTER1,
 46         XSCALE_COUNTER2,
 47         XSCALE_COUNTER3,
 48 };
 49 
 50 static const unsigned xscale_perf_map[PERF_COUNT_HW_MAX] = {
 51         [PERF_COUNT_HW_CPU_CYCLES]          = XSCALE_PERFCTR_CCNT,
 52         [PERF_COUNT_HW_INSTRUCTIONS]        = XSCALE_PERFCTR_INSTRUCTION,
 53         [PERF_COUNT_HW_CACHE_REFERENCES]    = HW_OP_UNSUPPORTED,
 54         [PERF_COUNT_HW_CACHE_MISSES]        = HW_OP_UNSUPPORTED,
 55         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = XSCALE_PERFCTR_BRANCH,
 56         [PERF_COUNT_HW_BRANCH_MISSES]       = XSCALE_PERFCTR_BRANCH_MISS,
 57         [PERF_COUNT_HW_BUS_CYCLES]          = HW_OP_UNSUPPORTED,
 58 };
 59 
 60 static const unsigned xscale_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 61                                            [PERF_COUNT_HW_CACHE_OP_MAX]
 62                                            [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 63         [C(L1D)] = {
 64                 [C(OP_READ)] = {
 65                         [C(RESULT_ACCESS)]      = XSCALE_PERFCTR_DCACHE_ACCESS,
 66                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DCACHE_MISS,
 67                 },
 68                 [C(OP_WRITE)] = {
 69                         [C(RESULT_ACCESS)]      = XSCALE_PERFCTR_DCACHE_ACCESS,
 70                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DCACHE_MISS,
 71                 },
 72                 [C(OP_PREFETCH)] = {
 73                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 74                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 75                 },
 76         },
 77         [C(L1I)] = {
 78                 [C(OP_READ)] = {
 79                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 80                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ICACHE_MISS,
 81                 },
 82                 [C(OP_WRITE)] = {
 83                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 84                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ICACHE_MISS,
 85                 },
 86                 [C(OP_PREFETCH)] = {
 87                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 88                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 89                 },
 90         },
 91         [C(LL)] = {
 92                 [C(OP_READ)] = {
 93                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 94                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 95                 },
 96                 [C(OP_WRITE)] = {
 97                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 98                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 99                 },
100                 [C(OP_PREFETCH)] = {
101                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
102                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
103                 },
104         },
105         [C(DTLB)] = {
106                 [C(OP_READ)] = {
107                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
108                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DTLB_MISS,
109                 },
110                 [C(OP_WRITE)] = {
111                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
112                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DTLB_MISS,
113                 },
114                 [C(OP_PREFETCH)] = {
115                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
116                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
117                 },
118         },
119         [C(ITLB)] = {
120                 [C(OP_READ)] = {
121                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
122                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ITLB_MISS,
123                 },
124                 [C(OP_WRITE)] = {
125                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
126                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ITLB_MISS,
127                 },
128                 [C(OP_PREFETCH)] = {
129                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
130                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
131                 },
132         },
133         [C(BPU)] = {
134                 [C(OP_READ)] = {
135                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
136                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
137                 },
138                 [C(OP_WRITE)] = {
139                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
140                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
141                 },
142                 [C(OP_PREFETCH)] = {
143                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
144                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
145                 },
146         },
147         [C(NODE)] = {
148                 [C(OP_READ)] = {
149                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
150                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
151                 },
152                 [C(OP_WRITE)] = {
153                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
154                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
155                 },
156                 [C(OP_PREFETCH)] = {
157                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
158                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
159                 },
160         },
161 };
162 
163 #define XSCALE_PMU_ENABLE       0x001
164 #define XSCALE_PMN_RESET        0x002
165 #define XSCALE_CCNT_RESET       0x004
166 #define XSCALE_PMU_RESET        (CCNT_RESET | PMN_RESET)
167 #define XSCALE_PMU_CNT64        0x008
168 
169 #define XSCALE1_OVERFLOWED_MASK 0x700
170 #define XSCALE1_CCOUNT_OVERFLOW 0x400
171 #define XSCALE1_COUNT0_OVERFLOW 0x100
172 #define XSCALE1_COUNT1_OVERFLOW 0x200
173 #define XSCALE1_CCOUNT_INT_EN   0x040
174 #define XSCALE1_COUNT0_INT_EN   0x010
175 #define XSCALE1_COUNT1_INT_EN   0x020
176 #define XSCALE1_COUNT0_EVT_SHFT 12
177 #define XSCALE1_COUNT0_EVT_MASK (0xff << XSCALE1_COUNT0_EVT_SHFT)
178 #define XSCALE1_COUNT1_EVT_SHFT 20
179 #define XSCALE1_COUNT1_EVT_MASK (0xff << XSCALE1_COUNT1_EVT_SHFT)
180 
181 static inline u32
182 xscale1pmu_read_pmnc(void)
183 {
184         u32 val;
185         asm volatile("mrc p14, 0, %0, c0, c0, 0" : "=r" (val));
186         return val;
187 }
188 
189 static inline void
190 xscale1pmu_write_pmnc(u32 val)
191 {
192         /* upper 4bits and 7, 11 are write-as-0 */
193         val &= 0xffff77f;
194         asm volatile("mcr p14, 0, %0, c0, c0, 0" : : "r" (val));
195 }
196 
197 static inline int
198 xscale1_pmnc_counter_has_overflowed(unsigned long pmnc,
199                                         enum xscale_counters counter)
200 {
201         int ret = 0;
202 
203         switch (counter) {
204         case XSCALE_CYCLE_COUNTER:
205                 ret = pmnc & XSCALE1_CCOUNT_OVERFLOW;
206                 break;
207         case XSCALE_COUNTER0:
208                 ret = pmnc & XSCALE1_COUNT0_OVERFLOW;
209                 break;
210         case XSCALE_COUNTER1:
211                 ret = pmnc & XSCALE1_COUNT1_OVERFLOW;
212                 break;
213         default:
214                 WARN_ONCE(1, "invalid counter number (%d)\n", counter);
215         }
216 
217         return ret;
218 }
219 
220 static irqreturn_t
221 xscale1pmu_handle_irq(int irq_num, void *dev)
222 {
223         unsigned long pmnc;
224         struct perf_sample_data data;
225         struct pmu_hw_events *cpuc;
226         struct pt_regs *regs;
227         int idx;
228 
229         /*
230          * NOTE: there's an A stepping erratum that states if an overflow
231          *       bit already exists and another occurs, the previous
232          *       Overflow bit gets cleared. There's no workaround.
233          *       Fixed in B stepping or later.
234          */
235         pmnc = xscale1pmu_read_pmnc();
236 
237         /*
238          * Write the value back to clear the overflow flags. Overflow
239          * flags remain in pmnc for use below. We also disable the PMU
240          * while we process the interrupt.
241          */
242         xscale1pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE);
243 
244         if (!(pmnc & XSCALE1_OVERFLOWED_MASK))
245                 return IRQ_NONE;
246 
247         regs = get_irq_regs();
248 
249         perf_sample_data_init(&data, 0);
250 
251         cpuc = &__get_cpu_var(cpu_hw_events);
252         for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
253                 struct perf_event *event = cpuc->events[idx];
254                 struct hw_perf_event *hwc;
255 
256                 if (!event)
257                         continue;
258 
259                 if (!xscale1_pmnc_counter_has_overflowed(pmnc, idx))
260                         continue;
261 
262                 hwc = &event->hw;
263                 armpmu_event_update(event, hwc, idx);
264                 data.period = event->hw.last_period;
265                 if (!armpmu_event_set_period(event, hwc, idx))
266                         continue;
267 
268                 if (perf_event_overflow(event, &data, regs))
269                         cpu_pmu->disable(hwc, idx);
270         }
271 
272         irq_work_run();
273 
274         /*
275          * Re-enable the PMU.
276          */
277         pmnc = xscale1pmu_read_pmnc() | XSCALE_PMU_ENABLE;
278         xscale1pmu_write_pmnc(pmnc);
279 
280         return IRQ_HANDLED;
281 }
282 
283 static void
284 xscale1pmu_enable_event(struct hw_perf_event *hwc, int idx)
285 {
286         unsigned long val, mask, evt, flags;
287         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
288 
289         switch (idx) {
290         case XSCALE_CYCLE_COUNTER:
291                 mask = 0;
292                 evt = XSCALE1_CCOUNT_INT_EN;
293                 break;
294         case XSCALE_COUNTER0:
295                 mask = XSCALE1_COUNT0_EVT_MASK;
296                 evt = (hwc->config_base << XSCALE1_COUNT0_EVT_SHFT) |
297                         XSCALE1_COUNT0_INT_EN;
298                 break;
299         case XSCALE_COUNTER1:
300                 mask = XSCALE1_COUNT1_EVT_MASK;
301                 evt = (hwc->config_base << XSCALE1_COUNT1_EVT_SHFT) |
302                         XSCALE1_COUNT1_INT_EN;
303                 break;
304         default:
305                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
306                 return;
307         }
308 
309         raw_spin_lock_irqsave(&events->pmu_lock, flags);
310         val = xscale1pmu_read_pmnc();
311         val &= ~mask;
312         val |= evt;
313         xscale1pmu_write_pmnc(val);
314         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
315 }
316 
317 static void
318 xscale1pmu_disable_event(struct hw_perf_event *hwc, int idx)
319 {
320         unsigned long val, mask, evt, flags;
321         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
322 
323         switch (idx) {
324         case XSCALE_CYCLE_COUNTER:
325                 mask = XSCALE1_CCOUNT_INT_EN;
326                 evt = 0;
327                 break;
328         case XSCALE_COUNTER0:
329                 mask = XSCALE1_COUNT0_INT_EN | XSCALE1_COUNT0_EVT_MASK;
330                 evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT0_EVT_SHFT;
331                 break;
332         case XSCALE_COUNTER1:
333                 mask = XSCALE1_COUNT1_INT_EN | XSCALE1_COUNT1_EVT_MASK;
334                 evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT1_EVT_SHFT;
335                 break;
336         default:
337                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
338                 return;
339         }
340 
341         raw_spin_lock_irqsave(&events->pmu_lock, flags);
342         val = xscale1pmu_read_pmnc();
343         val &= ~mask;
344         val |= evt;
345         xscale1pmu_write_pmnc(val);
346         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
347 }
348 
349 static int
350 xscale1pmu_get_event_idx(struct pmu_hw_events *cpuc,
351                         struct hw_perf_event *event)
352 {
353         if (XSCALE_PERFCTR_CCNT == event->config_base) {
354                 if (test_and_set_bit(XSCALE_CYCLE_COUNTER, cpuc->used_mask))
355                         return -EAGAIN;
356 
357                 return XSCALE_CYCLE_COUNTER;
358         } else {
359                 if (!test_and_set_bit(XSCALE_COUNTER1, cpuc->used_mask))
360                         return XSCALE_COUNTER1;
361 
362                 if (!test_and_set_bit(XSCALE_COUNTER0, cpuc->used_mask))
363                         return XSCALE_COUNTER0;
364 
365                 return -EAGAIN;
366         }
367 }
368 
369 static void
370 xscale1pmu_start(void)
371 {
372         unsigned long flags, val;
373         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
374 
375         raw_spin_lock_irqsave(&events->pmu_lock, flags);
376         val = xscale1pmu_read_pmnc();
377         val |= XSCALE_PMU_ENABLE;
378         xscale1pmu_write_pmnc(val);
379         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
380 }
381 
382 static void
383 xscale1pmu_stop(void)
384 {
385         unsigned long flags, val;
386         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
387 
388         raw_spin_lock_irqsave(&events->pmu_lock, flags);
389         val = xscale1pmu_read_pmnc();
390         val &= ~XSCALE_PMU_ENABLE;
391         xscale1pmu_write_pmnc(val);
392         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
393 }
394 
395 static inline u32
396 xscale1pmu_read_counter(int counter)
397 {
398         u32 val = 0;
399 
400         switch (counter) {
401         case XSCALE_CYCLE_COUNTER:
402                 asm volatile("mrc p14, 0, %0, c1, c0, 0" : "=r" (val));
403                 break;
404         case XSCALE_COUNTER0:
405                 asm volatile("mrc p14, 0, %0, c2, c0, 0" : "=r" (val));
406                 break;
407         case XSCALE_COUNTER1:
408                 asm volatile("mrc p14, 0, %0, c3, c0, 0" : "=r" (val));
409                 break;
410         }
411 
412         return val;
413 }
414 
415 static inline void
416 xscale1pmu_write_counter(int counter, u32 val)
417 {
418         switch (counter) {
419         case XSCALE_CYCLE_COUNTER:
420                 asm volatile("mcr p14, 0, %0, c1, c0, 0" : : "r" (val));
421                 break;
422         case XSCALE_COUNTER0:
423                 asm volatile("mcr p14, 0, %0, c2, c0, 0" : : "r" (val));
424                 break;
425         case XSCALE_COUNTER1:
426                 asm volatile("mcr p14, 0, %0, c3, c0, 0" : : "r" (val));
427                 break;
428         }
429 }
430 
431 static int xscale_map_event(struct perf_event *event)
432 {
433         return map_cpu_event(event, &xscale_perf_map,
434                                 &xscale_perf_cache_map, 0xFF);
435 }
436 
437 static struct arm_pmu xscale1pmu = {
438         .id             = ARM_PERF_PMU_ID_XSCALE1,
439         .name           = "xscale1",
440         .handle_irq     = xscale1pmu_handle_irq,
441         .enable         = xscale1pmu_enable_event,
442         .disable        = xscale1pmu_disable_event,
443         .read_counter   = xscale1pmu_read_counter,
444         .write_counter  = xscale1pmu_write_counter,
445         .get_event_idx  = xscale1pmu_get_event_idx,
446         .start          = xscale1pmu_start,
447         .stop           = xscale1pmu_stop,
448         .map_event      = xscale_map_event,
449         .num_events     = 3,
450         .max_period     = (1LLU << 32) - 1,
451 };
452 
453 static struct arm_pmu *__init xscale1pmu_init(void)
454 {
455         return &xscale1pmu;
456 }
457 
458 #define XSCALE2_OVERFLOWED_MASK 0x01f
459 #define XSCALE2_CCOUNT_OVERFLOW 0x001
460 #define XSCALE2_COUNT0_OVERFLOW 0x002
461 #define XSCALE2_COUNT1_OVERFLOW 0x004
462 #define XSCALE2_COUNT2_OVERFLOW 0x008
463 #define XSCALE2_COUNT3_OVERFLOW 0x010
464 #define XSCALE2_CCOUNT_INT_EN   0x001
465 #define XSCALE2_COUNT0_INT_EN   0x002
466 #define XSCALE2_COUNT1_INT_EN   0x004
467 #define XSCALE2_COUNT2_INT_EN   0x008
468 #define XSCALE2_COUNT3_INT_EN   0x010
469 #define XSCALE2_COUNT0_EVT_SHFT 0
470 #define XSCALE2_COUNT0_EVT_MASK (0xff << XSCALE2_COUNT0_EVT_SHFT)
471 #define XSCALE2_COUNT1_EVT_SHFT 8
472 #define XSCALE2_COUNT1_EVT_MASK (0xff << XSCALE2_COUNT1_EVT_SHFT)
473 #define XSCALE2_COUNT2_EVT_SHFT 16
474 #define XSCALE2_COUNT2_EVT_MASK (0xff << XSCALE2_COUNT2_EVT_SHFT)
475 #define XSCALE2_COUNT3_EVT_SHFT 24
476 #define XSCALE2_COUNT3_EVT_MASK (0xff << XSCALE2_COUNT3_EVT_SHFT)
477 
478 static inline u32
479 xscale2pmu_read_pmnc(void)
480 {
481         u32 val;
482         asm volatile("mrc p14, 0, %0, c0, c1, 0" : "=r" (val));
483         /* bits 1-2 and 4-23 are read-unpredictable */
484         return val & 0xff000009;
485 }
486 
487 static inline void
488 xscale2pmu_write_pmnc(u32 val)
489 {
490         /* bits 4-23 are write-as-0, 24-31 are write ignored */
491         val &= 0xf;
492         asm volatile("mcr p14, 0, %0, c0, c1, 0" : : "r" (val));
493 }
494 
495 static inline u32
496 xscale2pmu_read_overflow_flags(void)
497 {
498         u32 val;
499         asm volatile("mrc p14, 0, %0, c5, c1, 0" : "=r" (val));
500         return val;
501 }
502 
503 static inline void
504 xscale2pmu_write_overflow_flags(u32 val)
505 {
506         asm volatile("mcr p14, 0, %0, c5, c1, 0" : : "r" (val));
507 }
508 
509 static inline u32
510 xscale2pmu_read_event_select(void)
511 {
512         u32 val;
513         asm volatile("mrc p14, 0, %0, c8, c1, 0" : "=r" (val));
514         return val;
515 }
516 
517 static inline void
518 xscale2pmu_write_event_select(u32 val)
519 {
520         asm volatile("mcr p14, 0, %0, c8, c1, 0" : : "r"(val));
521 }
522 
523 static inline u32
524 xscale2pmu_read_int_enable(void)
525 {
526         u32 val;
527         asm volatile("mrc p14, 0, %0, c4, c1, 0" : "=r" (val));
528         return val;
529 }
530 
531 static void
532 xscale2pmu_write_int_enable(u32 val)
533 {
534         asm volatile("mcr p14, 0, %0, c4, c1, 0" : : "r" (val));
535 }
536 
537 static inline int
538 xscale2_pmnc_counter_has_overflowed(unsigned long of_flags,
539                                         enum xscale_counters counter)
540 {
541         int ret = 0;
542 
543         switch (counter) {
544         case XSCALE_CYCLE_COUNTER:
545                 ret = of_flags & XSCALE2_CCOUNT_OVERFLOW;
546                 break;
547         case XSCALE_COUNTER0:
548                 ret = of_flags & XSCALE2_COUNT0_OVERFLOW;
549                 break;
550         case XSCALE_COUNTER1:
551                 ret = of_flags & XSCALE2_COUNT1_OVERFLOW;
552                 break;
553         case XSCALE_COUNTER2:
554                 ret = of_flags & XSCALE2_COUNT2_OVERFLOW;
555                 break;
556         case XSCALE_COUNTER3:
557                 ret = of_flags & XSCALE2_COUNT3_OVERFLOW;
558                 break;
559         default:
560                 WARN_ONCE(1, "invalid counter number (%d)\n", counter);
561         }
562 
563         return ret;
564 }
565 
566 static irqreturn_t
567 xscale2pmu_handle_irq(int irq_num, void *dev)
568 {
569         unsigned long pmnc, of_flags;
570         struct perf_sample_data data;
571         struct pmu_hw_events *cpuc;
572         struct pt_regs *regs;
573         int idx;
574 
575         /* Disable the PMU. */
576         pmnc = xscale2pmu_read_pmnc();
577         xscale2pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE);
578 
579         /* Check the overflow flag register. */
580         of_flags = xscale2pmu_read_overflow_flags();
581         if (!(of_flags & XSCALE2_OVERFLOWED_MASK))
582                 return IRQ_NONE;
583 
584         /* Clear the overflow bits. */
585         xscale2pmu_write_overflow_flags(of_flags);
586 
587         regs = get_irq_regs();
588 
589         perf_sample_data_init(&data, 0);
590 
591         cpuc = &__get_cpu_var(cpu_hw_events);
592         for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
593                 struct perf_event *event = cpuc->events[idx];
594                 struct hw_perf_event *hwc;
595 
596                 if (!event)
597                         continue;
598 
599                 if (!xscale2_pmnc_counter_has_overflowed(of_flags, idx))
600                         continue;
601 
602                 hwc = &event->hw;
603                 armpmu_event_update(event, hwc, idx);
604                 data.period = event->hw.last_period;
605                 if (!armpmu_event_set_period(event, hwc, idx))
606                         continue;
607 
608                 if (perf_event_overflow(event, &data, regs))
609                         cpu_pmu->disable(hwc, idx);
610         }
611 
612         irq_work_run();
613 
614         /*
615          * Re-enable the PMU.
616          */
617         pmnc = xscale2pmu_read_pmnc() | XSCALE_PMU_ENABLE;
618         xscale2pmu_write_pmnc(pmnc);
619 
620         return IRQ_HANDLED;
621 }
622 
623 static void
624 xscale2pmu_enable_event(struct hw_perf_event *hwc, int idx)
625 {
626         unsigned long flags, ien, evtsel;
627         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
628 
629         ien = xscale2pmu_read_int_enable();
630         evtsel = xscale2pmu_read_event_select();
631 
632         switch (idx) {
633         case XSCALE_CYCLE_COUNTER:
634                 ien |= XSCALE2_CCOUNT_INT_EN;
635                 break;
636         case XSCALE_COUNTER0:
637                 ien |= XSCALE2_COUNT0_INT_EN;
638                 evtsel &= ~XSCALE2_COUNT0_EVT_MASK;
639                 evtsel |= hwc->config_base << XSCALE2_COUNT0_EVT_SHFT;
640                 break;
641         case XSCALE_COUNTER1:
642                 ien |= XSCALE2_COUNT1_INT_EN;
643                 evtsel &= ~XSCALE2_COUNT1_EVT_MASK;
644                 evtsel |= hwc->config_base << XSCALE2_COUNT1_EVT_SHFT;
645                 break;
646         case XSCALE_COUNTER2:
647                 ien |= XSCALE2_COUNT2_INT_EN;
648                 evtsel &= ~XSCALE2_COUNT2_EVT_MASK;
649                 evtsel |= hwc->config_base << XSCALE2_COUNT2_EVT_SHFT;
650                 break;
651         case XSCALE_COUNTER3:
652                 ien |= XSCALE2_COUNT3_INT_EN;
653                 evtsel &= ~XSCALE2_COUNT3_EVT_MASK;
654                 evtsel |= hwc->config_base << XSCALE2_COUNT3_EVT_SHFT;
655                 break;
656         default:
657                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
658                 return;
659         }
660 
661         raw_spin_lock_irqsave(&events->pmu_lock, flags);
662         xscale2pmu_write_event_select(evtsel);
663         xscale2pmu_write_int_enable(ien);
664         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
665 }
666 
667 static void
668 xscale2pmu_disable_event(struct hw_perf_event *hwc, int idx)
669 {
670         unsigned long flags, ien, evtsel, of_flags;
671         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
672 
673         ien = xscale2pmu_read_int_enable();
674         evtsel = xscale2pmu_read_event_select();
675 
676         switch (idx) {
677         case XSCALE_CYCLE_COUNTER:
678                 ien &= ~XSCALE2_CCOUNT_INT_EN;
679                 of_flags = XSCALE2_CCOUNT_OVERFLOW;
680                 break;
681         case XSCALE_COUNTER0:
682                 ien &= ~XSCALE2_COUNT0_INT_EN;
683                 evtsel &= ~XSCALE2_COUNT0_EVT_MASK;
684                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT0_EVT_SHFT;
685                 of_flags = XSCALE2_COUNT0_OVERFLOW;
686                 break;
687         case XSCALE_COUNTER1:
688                 ien &= ~XSCALE2_COUNT1_INT_EN;
689                 evtsel &= ~XSCALE2_COUNT1_EVT_MASK;
690                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT1_EVT_SHFT;
691                 of_flags = XSCALE2_COUNT1_OVERFLOW;
692                 break;
693         case XSCALE_COUNTER2:
694                 ien &= ~XSCALE2_COUNT2_INT_EN;
695                 evtsel &= ~XSCALE2_COUNT2_EVT_MASK;
696                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT2_EVT_SHFT;
697                 of_flags = XSCALE2_COUNT2_OVERFLOW;
698                 break;
699         case XSCALE_COUNTER3:
700                 ien &= ~XSCALE2_COUNT3_INT_EN;
701                 evtsel &= ~XSCALE2_COUNT3_EVT_MASK;
702                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT3_EVT_SHFT;
703                 of_flags = XSCALE2_COUNT3_OVERFLOW;
704                 break;
705         default:
706                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
707                 return;
708         }
709 
710         raw_spin_lock_irqsave(&events->pmu_lock, flags);
711         xscale2pmu_write_event_select(evtsel);
712         xscale2pmu_write_int_enable(ien);
713         xscale2pmu_write_overflow_flags(of_flags);
714         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
715 }
716 
717 static int
718 xscale2pmu_get_event_idx(struct pmu_hw_events *cpuc,
719                         struct hw_perf_event *event)
720 {
721         int idx = xscale1pmu_get_event_idx(cpuc, event);
722         if (idx >= 0)
723                 goto out;
724 
725         if (!test_and_set_bit(XSCALE_COUNTER3, cpuc->used_mask))
726                 idx = XSCALE_COUNTER3;
727         else if (!test_and_set_bit(XSCALE_COUNTER2, cpuc->used_mask))
728                 idx = XSCALE_COUNTER2;
729 out:
730         return idx;
731 }
732 
733 static void
734 xscale2pmu_start(void)
735 {
736         unsigned long flags, val;
737         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
738 
739         raw_spin_lock_irqsave(&events->pmu_lock, flags);
740         val = xscale2pmu_read_pmnc() & ~XSCALE_PMU_CNT64;
741         val |= XSCALE_PMU_ENABLE;
742         xscale2pmu_write_pmnc(val);
743         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
744 }
745 
746 static void
747 xscale2pmu_stop(void)
748 {
749         unsigned long flags, val;
750         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
751 
752         raw_spin_lock_irqsave(&events->pmu_lock, flags);
753         val = xscale2pmu_read_pmnc();
754         val &= ~XSCALE_PMU_ENABLE;
755         xscale2pmu_write_pmnc(val);
756         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
757 }
758 
759 static inline u32
760 xscale2pmu_read_counter(int counter)
761 {
762         u32 val = 0;
763 
764         switch (counter) {
765         case XSCALE_CYCLE_COUNTER:
766                 asm volatile("mrc p14, 0, %0, c1, c1, 0" : "=r" (val));
767                 break;
768         case XSCALE_COUNTER0:
769                 asm volatile("mrc p14, 0, %0, c0, c2, 0" : "=r" (val));
770                 break;
771         case XSCALE_COUNTER1:
772                 asm volatile("mrc p14, 0, %0, c1, c2, 0" : "=r" (val));
773                 break;
774         case XSCALE_COUNTER2:
775                 asm volatile("mrc p14, 0, %0, c2, c2, 0" : "=r" (val));
776                 break;
777         case XSCALE_COUNTER3:
778                 asm volatile("mrc p14, 0, %0, c3, c2, 0" : "=r" (val));
779                 break;
780         }
781 
782         return val;
783 }
784 
785 static inline void
786 xscale2pmu_write_counter(int counter, u32 val)
787 {
788         switch (counter) {
789         case XSCALE_CYCLE_COUNTER:
790                 asm volatile("mcr p14, 0, %0, c1, c1, 0" : : "r" (val));
791                 break;
792         case XSCALE_COUNTER0:
793                 asm volatile("mcr p14, 0, %0, c0, c2, 0" : : "r" (val));
794                 break;
795         case XSCALE_COUNTER1:
796                 asm volatile("mcr p14, 0, %0, c1, c2, 0" : : "r" (val));
797                 break;
798         case XSCALE_COUNTER2:
799                 asm volatile("mcr p14, 0, %0, c2, c2, 0" : : "r" (val));
800                 break;
801         case XSCALE_COUNTER3:
802                 asm volatile("mcr p14, 0, %0, c3, c2, 0" : : "r" (val));
803                 break;
804         }
805 }
806 
807 static struct arm_pmu xscale2pmu = {
808         .id             = ARM_PERF_PMU_ID_XSCALE2,
809         .name           = "xscale2",
810         .handle_irq     = xscale2pmu_handle_irq,
811         .enable         = xscale2pmu_enable_event,
812         .disable        = xscale2pmu_disable_event,
813         .read_counter   = xscale2pmu_read_counter,
814         .write_counter  = xscale2pmu_write_counter,
815         .get_event_idx  = xscale2pmu_get_event_idx,
816         .start          = xscale2pmu_start,
817         .stop           = xscale2pmu_stop,
818         .map_event      = xscale_map_event,
819         .num_events     = 5,
820         .max_period     = (1LLU << 32) - 1,
821 };
822 
823 static struct arm_pmu *__init xscale2pmu_init(void)
824 {
825         return &xscale2pmu;
826 }
827 #else
828 static struct arm_pmu *__init xscale1pmu_init(void)
829 {
830         return NULL;
831 }
832 
833 static struct arm_pmu *__init xscale2pmu_init(void)
834 {
835         return NULL;
836 }
837 #endif  /* CONFIG_CPU_XSCALE */
838 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp