~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/sparc/kernel/perf_event.c

Version: ~ [ linux-5.11-rc3 ] ~ [ linux-5.10.7 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.89 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.167 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.215 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.251 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.251 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /* Performance event support for sparc64.
  3  *
  4  * Copyright (C) 2009, 2010 David S. Miller <davem@davemloft.net>
  5  *
  6  * This code is based almost entirely upon the x86 perf event
  7  * code, which is:
  8  *
  9  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
 10  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
 11  *  Copyright (C) 2009 Jaswinder Singh Rajput
 12  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
 13  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
 14  */
 15 
 16 #include <linux/perf_event.h>
 17 #include <linux/kprobes.h>
 18 #include <linux/ftrace.h>
 19 #include <linux/kernel.h>
 20 #include <linux/kdebug.h>
 21 #include <linux/mutex.h>
 22 
 23 #include <asm/stacktrace.h>
 24 #include <asm/cpudata.h>
 25 #include <linux/uaccess.h>
 26 #include <linux/atomic.h>
 27 #include <asm/nmi.h>
 28 #include <asm/pcr.h>
 29 #include <asm/cacheflush.h>
 30 
 31 #include "kernel.h"
 32 #include "kstack.h"
 33 
 34 /* Two classes of sparc64 chips currently exist.  All of which have
 35  * 32-bit counters which can generate overflow interrupts on the
 36  * transition from 0xffffffff to 0.
 37  *
 38  * All chips upto and including SPARC-T3 have two performance
 39  * counters.  The two 32-bit counters are accessed in one go using a
 40  * single 64-bit register.
 41  *
 42  * On these older chips both counters are controlled using a single
 43  * control register.  The only way to stop all sampling is to clear
 44  * all of the context (user, supervisor, hypervisor) sampling enable
 45  * bits.  But these bits apply to both counters, thus the two counters
 46  * can't be enabled/disabled individually.
 47  *
 48  * Furthermore, the control register on these older chips have two
 49  * event fields, one for each of the two counters.  It's thus nearly
 50  * impossible to have one counter going while keeping the other one
 51  * stopped.  Therefore it is possible to get overflow interrupts for
 52  * counters not currently "in use" and that condition must be checked
 53  * in the overflow interrupt handler.
 54  *
 55  * So we use a hack, in that we program inactive counters with the
 56  * "sw_count0" and "sw_count1" events.  These count how many times
 57  * the instruction "sethi %hi(0xfc000), %g0" is executed.  It's an
 58  * unusual way to encode a NOP and therefore will not trigger in
 59  * normal code.
 60  *
 61  * Starting with SPARC-T4 we have one control register per counter.
 62  * And the counters are stored in individual registers.  The registers
 63  * for the counters are 64-bit but only a 32-bit counter is
 64  * implemented.  The event selections on SPARC-T4 lack any
 65  * restrictions, therefore we can elide all of the complicated
 66  * conflict resolution code we have for SPARC-T3 and earlier chips.
 67  */
 68 
 69 #define MAX_HWEVENTS                    4
 70 #define MAX_PCRS                        4
 71 #define MAX_PERIOD                      ((1UL << 32) - 1)
 72 
 73 #define PIC_UPPER_INDEX                 0
 74 #define PIC_LOWER_INDEX                 1
 75 #define PIC_NO_INDEX                    -1
 76 
 77 struct cpu_hw_events {
 78         /* Number of events currently scheduled onto this cpu.
 79          * This tells how many entries in the arrays below
 80          * are valid.
 81          */
 82         int                     n_events;
 83 
 84         /* Number of new events added since the last hw_perf_disable().
 85          * This works because the perf event layer always adds new
 86          * events inside of a perf_{disable,enable}() sequence.
 87          */
 88         int                     n_added;
 89 
 90         /* Array of events current scheduled on this cpu.  */
 91         struct perf_event       *event[MAX_HWEVENTS];
 92 
 93         /* Array of encoded longs, specifying the %pcr register
 94          * encoding and the mask of PIC counters this even can
 95          * be scheduled on.  See perf_event_encode() et al.
 96          */
 97         unsigned long           events[MAX_HWEVENTS];
 98 
 99         /* The current counter index assigned to an event.  When the
100          * event hasn't been programmed into the cpu yet, this will
101          * hold PIC_NO_INDEX.  The event->hw.idx value tells us where
102          * we ought to schedule the event.
103          */
104         int                     current_idx[MAX_HWEVENTS];
105 
106         /* Software copy of %pcr register(s) on this cpu.  */
107         u64                     pcr[MAX_HWEVENTS];
108 
109         /* Enabled/disable state.  */
110         int                     enabled;
111 
112         unsigned int            txn_flags;
113 };
114 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { .enabled = 1, };
115 
116 /* An event map describes the characteristics of a performance
117  * counter event.  In particular it gives the encoding as well as
118  * a mask telling which counters the event can be measured on.
119  *
120  * The mask is unused on SPARC-T4 and later.
121  */
122 struct perf_event_map {
123         u16     encoding;
124         u8      pic_mask;
125 #define PIC_NONE        0x00
126 #define PIC_UPPER       0x01
127 #define PIC_LOWER       0x02
128 };
129 
130 /* Encode a perf_event_map entry into a long.  */
131 static unsigned long perf_event_encode(const struct perf_event_map *pmap)
132 {
133         return ((unsigned long) pmap->encoding << 16) | pmap->pic_mask;
134 }
135 
136 static u8 perf_event_get_msk(unsigned long val)
137 {
138         return val & 0xff;
139 }
140 
141 static u64 perf_event_get_enc(unsigned long val)
142 {
143         return val >> 16;
144 }
145 
146 #define C(x) PERF_COUNT_HW_CACHE_##x
147 
148 #define CACHE_OP_UNSUPPORTED    0xfffe
149 #define CACHE_OP_NONSENSE       0xffff
150 
151 typedef struct perf_event_map cache_map_t
152                                 [PERF_COUNT_HW_CACHE_MAX]
153                                 [PERF_COUNT_HW_CACHE_OP_MAX]
154                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
155 
156 struct sparc_pmu {
157         const struct perf_event_map     *(*event_map)(int);
158         const cache_map_t               *cache_map;
159         int                             max_events;
160         u32                             (*read_pmc)(int);
161         void                            (*write_pmc)(int, u64);
162         int                             upper_shift;
163         int                             lower_shift;
164         int                             event_mask;
165         int                             user_bit;
166         int                             priv_bit;
167         int                             hv_bit;
168         int                             irq_bit;
169         int                             upper_nop;
170         int                             lower_nop;
171         unsigned int                    flags;
172 #define SPARC_PMU_ALL_EXCLUDES_SAME     0x00000001
173 #define SPARC_PMU_HAS_CONFLICTS         0x00000002
174         int                             max_hw_events;
175         int                             num_pcrs;
176         int                             num_pic_regs;
177 };
178 
179 static u32 sparc_default_read_pmc(int idx)
180 {
181         u64 val;
182 
183         val = pcr_ops->read_pic(0);
184         if (idx == PIC_UPPER_INDEX)
185                 val >>= 32;
186 
187         return val & 0xffffffff;
188 }
189 
190 static void sparc_default_write_pmc(int idx, u64 val)
191 {
192         u64 shift, mask, pic;
193 
194         shift = 0;
195         if (idx == PIC_UPPER_INDEX)
196                 shift = 32;
197 
198         mask = ((u64) 0xffffffff) << shift;
199         val <<= shift;
200 
201         pic = pcr_ops->read_pic(0);
202         pic &= ~mask;
203         pic |= val;
204         pcr_ops->write_pic(0, pic);
205 }
206 
207 static const struct perf_event_map ultra3_perfmon_event_map[] = {
208         [PERF_COUNT_HW_CPU_CYCLES] = { 0x0000, PIC_UPPER | PIC_LOWER },
209         [PERF_COUNT_HW_INSTRUCTIONS] = { 0x0001, PIC_UPPER | PIC_LOWER },
210         [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0009, PIC_LOWER },
211         [PERF_COUNT_HW_CACHE_MISSES] = { 0x0009, PIC_UPPER },
212 };
213 
214 static const struct perf_event_map *ultra3_event_map(int event_id)
215 {
216         return &ultra3_perfmon_event_map[event_id];
217 }
218 
219 static const cache_map_t ultra3_cache_map = {
220 [C(L1D)] = {
221         [C(OP_READ)] = {
222                 [C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, },
223                 [C(RESULT_MISS)] = { 0x09, PIC_UPPER, },
224         },
225         [C(OP_WRITE)] = {
226                 [C(RESULT_ACCESS)] = { 0x0a, PIC_LOWER },
227                 [C(RESULT_MISS)] = { 0x0a, PIC_UPPER },
228         },
229         [C(OP_PREFETCH)] = {
230                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
231                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
232         },
233 },
234 [C(L1I)] = {
235         [C(OP_READ)] = {
236                 [C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, },
237                 [C(RESULT_MISS)] = { 0x09, PIC_UPPER, },
238         },
239         [ C(OP_WRITE) ] = {
240                 [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
241                 [ C(RESULT_MISS)   ] = { CACHE_OP_NONSENSE },
242         },
243         [ C(OP_PREFETCH) ] = {
244                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
245                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
246         },
247 },
248 [C(LL)] = {
249         [C(OP_READ)] = {
250                 [C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER, },
251                 [C(RESULT_MISS)] = { 0x0c, PIC_UPPER, },
252         },
253         [C(OP_WRITE)] = {
254                 [C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER },
255                 [C(RESULT_MISS)] = { 0x0c, PIC_UPPER },
256         },
257         [C(OP_PREFETCH)] = {
258                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
259                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
260         },
261 },
262 [C(DTLB)] = {
263         [C(OP_READ)] = {
264                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
265                 [C(RESULT_MISS)] = { 0x12, PIC_UPPER, },
266         },
267         [ C(OP_WRITE) ] = {
268                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
269                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
270         },
271         [ C(OP_PREFETCH) ] = {
272                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
273                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
274         },
275 },
276 [C(ITLB)] = {
277         [C(OP_READ)] = {
278                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
279                 [C(RESULT_MISS)] = { 0x11, PIC_UPPER, },
280         },
281         [ C(OP_WRITE) ] = {
282                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
283                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
284         },
285         [ C(OP_PREFETCH) ] = {
286                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
287                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
288         },
289 },
290 [C(BPU)] = {
291         [C(OP_READ)] = {
292                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
293                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
294         },
295         [ C(OP_WRITE) ] = {
296                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
297                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
298         },
299         [ C(OP_PREFETCH) ] = {
300                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
301                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
302         },
303 },
304 [C(NODE)] = {
305         [C(OP_READ)] = {
306                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
307                 [C(RESULT_MISS)  ] = { CACHE_OP_UNSUPPORTED },
308         },
309         [ C(OP_WRITE) ] = {
310                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
311                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
312         },
313         [ C(OP_PREFETCH) ] = {
314                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
315                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
316         },
317 },
318 };
319 
320 static const struct sparc_pmu ultra3_pmu = {
321         .event_map      = ultra3_event_map,
322         .cache_map      = &ultra3_cache_map,
323         .max_events     = ARRAY_SIZE(ultra3_perfmon_event_map),
324         .read_pmc       = sparc_default_read_pmc,
325         .write_pmc      = sparc_default_write_pmc,
326         .upper_shift    = 11,
327         .lower_shift    = 4,
328         .event_mask     = 0x3f,
329         .user_bit       = PCR_UTRACE,
330         .priv_bit       = PCR_STRACE,
331         .upper_nop      = 0x1c,
332         .lower_nop      = 0x14,
333         .flags          = (SPARC_PMU_ALL_EXCLUDES_SAME |
334                            SPARC_PMU_HAS_CONFLICTS),
335         .max_hw_events  = 2,
336         .num_pcrs       = 1,
337         .num_pic_regs   = 1,
338 };
339 
340 /* Niagara1 is very limited.  The upper PIC is hard-locked to count
341  * only instructions, so it is free running which creates all kinds of
342  * problems.  Some hardware designs make one wonder if the creator
343  * even looked at how this stuff gets used by software.
344  */
345 static const struct perf_event_map niagara1_perfmon_event_map[] = {
346         [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, PIC_UPPER },
347         [PERF_COUNT_HW_INSTRUCTIONS] = { 0x00, PIC_UPPER },
348         [PERF_COUNT_HW_CACHE_REFERENCES] = { 0, PIC_NONE },
349         [PERF_COUNT_HW_CACHE_MISSES] = { 0x03, PIC_LOWER },
350 };
351 
352 static const struct perf_event_map *niagara1_event_map(int event_id)
353 {
354         return &niagara1_perfmon_event_map[event_id];
355 }
356 
357 static const cache_map_t niagara1_cache_map = {
358 [C(L1D)] = {
359         [C(OP_READ)] = {
360                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
361                 [C(RESULT_MISS)] = { 0x03, PIC_LOWER, },
362         },
363         [C(OP_WRITE)] = {
364                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
365                 [C(RESULT_MISS)] = { 0x03, PIC_LOWER, },
366         },
367         [C(OP_PREFETCH)] = {
368                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
369                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
370         },
371 },
372 [C(L1I)] = {
373         [C(OP_READ)] = {
374                 [C(RESULT_ACCESS)] = { 0x00, PIC_UPPER },
375                 [C(RESULT_MISS)] = { 0x02, PIC_LOWER, },
376         },
377         [ C(OP_WRITE) ] = {
378                 [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
379                 [ C(RESULT_MISS)   ] = { CACHE_OP_NONSENSE },
380         },
381         [ C(OP_PREFETCH) ] = {
382                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
383                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
384         },
385 },
386 [C(LL)] = {
387         [C(OP_READ)] = {
388                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
389                 [C(RESULT_MISS)] = { 0x07, PIC_LOWER, },
390         },
391         [C(OP_WRITE)] = {
392                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
393                 [C(RESULT_MISS)] = { 0x07, PIC_LOWER, },
394         },
395         [C(OP_PREFETCH)] = {
396                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
397                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
398         },
399 },
400 [C(DTLB)] = {
401         [C(OP_READ)] = {
402                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
403                 [C(RESULT_MISS)] = { 0x05, PIC_LOWER, },
404         },
405         [ C(OP_WRITE) ] = {
406                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
407                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
408         },
409         [ C(OP_PREFETCH) ] = {
410                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
411                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
412         },
413 },
414 [C(ITLB)] = {
415         [C(OP_READ)] = {
416                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
417                 [C(RESULT_MISS)] = { 0x04, PIC_LOWER, },
418         },
419         [ C(OP_WRITE) ] = {
420                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
421                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
422         },
423         [ C(OP_PREFETCH) ] = {
424                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
425                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
426         },
427 },
428 [C(BPU)] = {
429         [C(OP_READ)] = {
430                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
431                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
432         },
433         [ C(OP_WRITE) ] = {
434                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
435                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
436         },
437         [ C(OP_PREFETCH) ] = {
438                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
439                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
440         },
441 },
442 [C(NODE)] = {
443         [C(OP_READ)] = {
444                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
445                 [C(RESULT_MISS)  ] = { CACHE_OP_UNSUPPORTED },
446         },
447         [ C(OP_WRITE) ] = {
448                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
449                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
450         },
451         [ C(OP_PREFETCH) ] = {
452                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
453                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
454         },
455 },
456 };
457 
458 static const struct sparc_pmu niagara1_pmu = {
459         .event_map      = niagara1_event_map,
460         .cache_map      = &niagara1_cache_map,
461         .max_events     = ARRAY_SIZE(niagara1_perfmon_event_map),
462         .read_pmc       = sparc_default_read_pmc,
463         .write_pmc      = sparc_default_write_pmc,
464         .upper_shift    = 0,
465         .lower_shift    = 4,
466         .event_mask     = 0x7,
467         .user_bit       = PCR_UTRACE,
468         .priv_bit       = PCR_STRACE,
469         .upper_nop      = 0x0,
470         .lower_nop      = 0x0,
471         .flags          = (SPARC_PMU_ALL_EXCLUDES_SAME |
472                            SPARC_PMU_HAS_CONFLICTS),
473         .max_hw_events  = 2,
474         .num_pcrs       = 1,
475         .num_pic_regs   = 1,
476 };
477 
478 static const struct perf_event_map niagara2_perfmon_event_map[] = {
479         [PERF_COUNT_HW_CPU_CYCLES] = { 0x02ff, PIC_UPPER | PIC_LOWER },
480         [PERF_COUNT_HW_INSTRUCTIONS] = { 0x02ff, PIC_UPPER | PIC_LOWER },
481         [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0208, PIC_UPPER | PIC_LOWER },
482         [PERF_COUNT_HW_CACHE_MISSES] = { 0x0302, PIC_UPPER | PIC_LOWER },
483         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x0201, PIC_UPPER | PIC_LOWER },
484         [PERF_COUNT_HW_BRANCH_MISSES] = { 0x0202, PIC_UPPER | PIC_LOWER },
485 };
486 
487 static const struct perf_event_map *niagara2_event_map(int event_id)
488 {
489         return &niagara2_perfmon_event_map[event_id];
490 }
491 
492 static const cache_map_t niagara2_cache_map = {
493 [C(L1D)] = {
494         [C(OP_READ)] = {
495                 [C(RESULT_ACCESS)] = { 0x0208, PIC_UPPER | PIC_LOWER, },
496                 [C(RESULT_MISS)] = { 0x0302, PIC_UPPER | PIC_LOWER, },
497         },
498         [C(OP_WRITE)] = {
499                 [C(RESULT_ACCESS)] = { 0x0210, PIC_UPPER | PIC_LOWER, },
500                 [C(RESULT_MISS)] = { 0x0302, PIC_UPPER | PIC_LOWER, },
501         },
502         [C(OP_PREFETCH)] = {
503                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
504                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
505         },
506 },
507 [C(L1I)] = {
508         [C(OP_READ)] = {
509                 [C(RESULT_ACCESS)] = { 0x02ff, PIC_UPPER | PIC_LOWER, },
510                 [C(RESULT_MISS)] = { 0x0301, PIC_UPPER | PIC_LOWER, },
511         },
512         [ C(OP_WRITE) ] = {
513                 [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
514                 [ C(RESULT_MISS)   ] = { CACHE_OP_NONSENSE },
515         },
516         [ C(OP_PREFETCH) ] = {
517                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
518                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
519         },
520 },
521 [C(LL)] = {
522         [C(OP_READ)] = {
523                 [C(RESULT_ACCESS)] = { 0x0208, PIC_UPPER | PIC_LOWER, },
524                 [C(RESULT_MISS)] = { 0x0330, PIC_UPPER | PIC_LOWER, },
525         },
526         [C(OP_WRITE)] = {
527                 [C(RESULT_ACCESS)] = { 0x0210, PIC_UPPER | PIC_LOWER, },
528                 [C(RESULT_MISS)] = { 0x0320, PIC_UPPER | PIC_LOWER, },
529         },
530         [C(OP_PREFETCH)] = {
531                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
532                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
533         },
534 },
535 [C(DTLB)] = {
536         [C(OP_READ)] = {
537                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
538                 [C(RESULT_MISS)] = { 0x0b08, PIC_UPPER | PIC_LOWER, },
539         },
540         [ C(OP_WRITE) ] = {
541                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
542                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
543         },
544         [ C(OP_PREFETCH) ] = {
545                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
546                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
547         },
548 },
549 [C(ITLB)] = {
550         [C(OP_READ)] = {
551                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
552                 [C(RESULT_MISS)] = { 0xb04, PIC_UPPER | PIC_LOWER, },
553         },
554         [ C(OP_WRITE) ] = {
555                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
556                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
557         },
558         [ C(OP_PREFETCH) ] = {
559                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
560                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
561         },
562 },
563 [C(BPU)] = {
564         [C(OP_READ)] = {
565                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
566                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
567         },
568         [ C(OP_WRITE) ] = {
569                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
570                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
571         },
572         [ C(OP_PREFETCH) ] = {
573                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
574                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
575         },
576 },
577 [C(NODE)] = {
578         [C(OP_READ)] = {
579                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
580                 [C(RESULT_MISS)  ] = { CACHE_OP_UNSUPPORTED },
581         },
582         [ C(OP_WRITE) ] = {
583                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
584                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
585         },
586         [ C(OP_PREFETCH) ] = {
587                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
588                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
589         },
590 },
591 };
592 
593 static const struct sparc_pmu niagara2_pmu = {
594         .event_map      = niagara2_event_map,
595         .cache_map      = &niagara2_cache_map,
596         .max_events     = ARRAY_SIZE(niagara2_perfmon_event_map),
597         .read_pmc       = sparc_default_read_pmc,
598         .write_pmc      = sparc_default_write_pmc,
599         .upper_shift    = 19,
600         .lower_shift    = 6,
601         .event_mask     = 0xfff,
602         .user_bit       = PCR_UTRACE,
603         .priv_bit       = PCR_STRACE,
604         .hv_bit         = PCR_N2_HTRACE,
605         .irq_bit        = 0x30,
606         .upper_nop      = 0x220,
607         .lower_nop      = 0x220,
608         .flags          = (SPARC_PMU_ALL_EXCLUDES_SAME |
609                            SPARC_PMU_HAS_CONFLICTS),
610         .max_hw_events  = 2,
611         .num_pcrs       = 1,
612         .num_pic_regs   = 1,
613 };
614 
615 static const struct perf_event_map niagara4_perfmon_event_map[] = {
616         [PERF_COUNT_HW_CPU_CYCLES] = { (26 << 6) },
617         [PERF_COUNT_HW_INSTRUCTIONS] = { (3 << 6) | 0x3f },
618         [PERF_COUNT_HW_CACHE_REFERENCES] = { (3 << 6) | 0x04 },
619         [PERF_COUNT_HW_CACHE_MISSES] = { (16 << 6) | 0x07 },
620         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { (4 << 6) | 0x01 },
621         [PERF_COUNT_HW_BRANCH_MISSES] = { (25 << 6) | 0x0f },
622 };
623 
624 static const struct perf_event_map *niagara4_event_map(int event_id)
625 {
626         return &niagara4_perfmon_event_map[event_id];
627 }
628 
629 static const cache_map_t niagara4_cache_map = {
630 [C(L1D)] = {
631         [C(OP_READ)] = {
632                 [C(RESULT_ACCESS)] = { (3 << 6) | 0x04 },
633                 [C(RESULT_MISS)] = { (16 << 6) | 0x07 },
634         },
635         [C(OP_WRITE)] = {
636                 [C(RESULT_ACCESS)] = { (3 << 6) | 0x08 },
637                 [C(RESULT_MISS)] = { (16 << 6) | 0x07 },
638         },
639         [C(OP_PREFETCH)] = {
640                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
641                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
642         },
643 },
644 [C(L1I)] = {
645         [C(OP_READ)] = {
646                 [C(RESULT_ACCESS)] = { (3 << 6) | 0x3f },
647                 [C(RESULT_MISS)] = { (11 << 6) | 0x03 },
648         },
649         [ C(OP_WRITE) ] = {
650                 [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
651                 [ C(RESULT_MISS)   ] = { CACHE_OP_NONSENSE },
652         },
653         [ C(OP_PREFETCH) ] = {
654                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
655                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
656         },
657 },
658 [C(LL)] = {
659         [C(OP_READ)] = {
660                 [C(RESULT_ACCESS)] = { (3 << 6) | 0x04 },
661                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
662         },
663         [C(OP_WRITE)] = {
664                 [C(RESULT_ACCESS)] = { (3 << 6) | 0x08 },
665                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
666         },
667         [C(OP_PREFETCH)] = {
668                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
669                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
670         },
671 },
672 [C(DTLB)] = {
673         [C(OP_READ)] = {
674                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
675                 [C(RESULT_MISS)] = { (17 << 6) | 0x3f },
676         },
677         [ C(OP_WRITE) ] = {
678                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
679                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
680         },
681         [ C(OP_PREFETCH) ] = {
682                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
683                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
684         },
685 },
686 [C(ITLB)] = {
687         [C(OP_READ)] = {
688                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
689                 [C(RESULT_MISS)] = { (6 << 6) | 0x3f },
690         },
691         [ C(OP_WRITE) ] = {
692                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
693                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
694         },
695         [ C(OP_PREFETCH) ] = {
696                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
697                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
698         },
699 },
700 [C(BPU)] = {
701         [C(OP_READ)] = {
702                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
703                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
704         },
705         [ C(OP_WRITE) ] = {
706                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
707                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
708         },
709         [ C(OP_PREFETCH) ] = {
710                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
711                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
712         },
713 },
714 [C(NODE)] = {
715         [C(OP_READ)] = {
716                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
717                 [C(RESULT_MISS)  ] = { CACHE_OP_UNSUPPORTED },
718         },
719         [ C(OP_WRITE) ] = {
720                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
721                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
722         },
723         [ C(OP_PREFETCH) ] = {
724                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
725                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
726         },
727 },
728 };
729 
730 static u32 sparc_vt_read_pmc(int idx)
731 {
732         u64 val = pcr_ops->read_pic(idx);
733 
734         return val & 0xffffffff;
735 }
736 
737 static void sparc_vt_write_pmc(int idx, u64 val)
738 {
739         u64 pcr;
740 
741         pcr = pcr_ops->read_pcr(idx);
742         /* ensure ov and ntc are reset */
743         pcr &= ~(PCR_N4_OV | PCR_N4_NTC);
744 
745         pcr_ops->write_pic(idx, val & 0xffffffff);
746 
747         pcr_ops->write_pcr(idx, pcr);
748 }
749 
750 static const struct sparc_pmu niagara4_pmu = {
751         .event_map      = niagara4_event_map,
752         .cache_map      = &niagara4_cache_map,
753         .max_events     = ARRAY_SIZE(niagara4_perfmon_event_map),
754         .read_pmc       = sparc_vt_read_pmc,
755         .write_pmc      = sparc_vt_write_pmc,
756         .upper_shift    = 5,
757         .lower_shift    = 5,
758         .event_mask     = 0x7ff,
759         .user_bit       = PCR_N4_UTRACE,
760         .priv_bit       = PCR_N4_STRACE,
761 
762         /* We explicitly don't support hypervisor tracing.  The T4
763          * generates the overflow event for precise events via a trap
764          * which will not be generated (ie. it's completely lost) if
765          * we happen to be in the hypervisor when the event triggers.
766          * Essentially, the overflow event reporting is completely
767          * unusable when you have hypervisor mode tracing enabled.
768          */
769         .hv_bit         = 0,
770 
771         .irq_bit        = PCR_N4_TOE,
772         .upper_nop      = 0,
773         .lower_nop      = 0,
774         .flags          = 0,
775         .max_hw_events  = 4,
776         .num_pcrs       = 4,
777         .num_pic_regs   = 4,
778 };
779 
780 static const struct sparc_pmu sparc_m7_pmu = {
781         .event_map      = niagara4_event_map,
782         .cache_map      = &niagara4_cache_map,
783         .max_events     = ARRAY_SIZE(niagara4_perfmon_event_map),
784         .read_pmc       = sparc_vt_read_pmc,
785         .write_pmc      = sparc_vt_write_pmc,
786         .upper_shift    = 5,
787         .lower_shift    = 5,
788         .event_mask     = 0x7ff,
789         .user_bit       = PCR_N4_UTRACE,
790         .priv_bit       = PCR_N4_STRACE,
791 
792         /* We explicitly don't support hypervisor tracing. */
793         .hv_bit         = 0,
794 
795         .irq_bit        = PCR_N4_TOE,
796         .upper_nop      = 0,
797         .lower_nop      = 0,
798         .flags          = 0,
799         .max_hw_events  = 4,
800         .num_pcrs       = 4,
801         .num_pic_regs   = 4,
802 };
803 static const struct sparc_pmu *sparc_pmu __read_mostly;
804 
805 static u64 event_encoding(u64 event_id, int idx)
806 {
807         if (idx == PIC_UPPER_INDEX)
808                 event_id <<= sparc_pmu->upper_shift;
809         else
810                 event_id <<= sparc_pmu->lower_shift;
811         return event_id;
812 }
813 
814 static u64 mask_for_index(int idx)
815 {
816         return event_encoding(sparc_pmu->event_mask, idx);
817 }
818 
819 static u64 nop_for_index(int idx)
820 {
821         return event_encoding(idx == PIC_UPPER_INDEX ?
822                               sparc_pmu->upper_nop :
823                               sparc_pmu->lower_nop, idx);
824 }
825 
826 static inline void sparc_pmu_enable_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc, int idx)
827 {
828         u64 enc, val, mask = mask_for_index(idx);
829         int pcr_index = 0;
830 
831         if (sparc_pmu->num_pcrs > 1)
832                 pcr_index = idx;
833 
834         enc = perf_event_get_enc(cpuc->events[idx]);
835 
836         val = cpuc->pcr[pcr_index];
837         val &= ~mask;
838         val |= event_encoding(enc, idx);
839         cpuc->pcr[pcr_index] = val;
840 
841         pcr_ops->write_pcr(pcr_index, cpuc->pcr[pcr_index]);
842 }
843 
844 static inline void sparc_pmu_disable_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc, int idx)
845 {
846         u64 mask = mask_for_index(idx);
847         u64 nop = nop_for_index(idx);
848         int pcr_index = 0;
849         u64 val;
850 
851         if (sparc_pmu->num_pcrs > 1)
852                 pcr_index = idx;
853 
854         val = cpuc->pcr[pcr_index];
855         val &= ~mask;
856         val |= nop;
857         cpuc->pcr[pcr_index] = val;
858 
859         pcr_ops->write_pcr(pcr_index, cpuc->pcr[pcr_index]);
860 }
861 
862 static u64 sparc_perf_event_update(struct perf_event *event,
863                                    struct hw_perf_event *hwc, int idx)
864 {
865         int shift = 64 - 32;
866         u64 prev_raw_count, new_raw_count;
867         s64 delta;
868 
869 again:
870         prev_raw_count = local64_read(&hwc->prev_count);
871         new_raw_count = sparc_pmu->read_pmc(idx);
872 
873         if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
874                              new_raw_count) != prev_raw_count)
875                 goto again;
876 
877         delta = (new_raw_count << shift) - (prev_raw_count << shift);
878         delta >>= shift;
879 
880         local64_add(delta, &event->count);
881         local64_sub(delta, &hwc->period_left);
882 
883         return new_raw_count;
884 }
885 
886 static int sparc_perf_event_set_period(struct perf_event *event,
887                                        struct hw_perf_event *hwc, int idx)
888 {
889         s64 left = local64_read(&hwc->period_left);
890         s64 period = hwc->sample_period;
891         int ret = 0;
892 
893         if (unlikely(left <= -period)) {
894                 left = period;
895                 local64_set(&hwc->period_left, left);
896                 hwc->last_period = period;
897                 ret = 1;
898         }
899 
900         if (unlikely(left <= 0)) {
901                 left += period;
902                 local64_set(&hwc->period_left, left);
903                 hwc->last_period = period;
904                 ret = 1;
905         }
906         if (left > MAX_PERIOD)
907                 left = MAX_PERIOD;
908 
909         local64_set(&hwc->prev_count, (u64)-left);
910 
911         sparc_pmu->write_pmc(idx, (u64)(-left) & 0xffffffff);
912 
913         perf_event_update_userpage(event);
914 
915         return ret;
916 }
917 
918 static void read_in_all_counters(struct cpu_hw_events *cpuc)
919 {
920         int i;
921 
922         for (i = 0; i < cpuc->n_events; i++) {
923                 struct perf_event *cp = cpuc->event[i];
924 
925                 if (cpuc->current_idx[i] != PIC_NO_INDEX &&
926                     cpuc->current_idx[i] != cp->hw.idx) {
927                         sparc_perf_event_update(cp, &cp->hw,
928                                                 cpuc->current_idx[i]);
929                         cpuc->current_idx[i] = PIC_NO_INDEX;
930                 }
931         }
932 }
933 
934 /* On this PMU all PICs are programmed using a single PCR.  Calculate
935  * the combined control register value.
936  *
937  * For such chips we require that all of the events have the same
938  * configuration, so just fetch the settings from the first entry.
939  */
940 static void calculate_single_pcr(struct cpu_hw_events *cpuc)
941 {
942         int i;
943 
944         if (!cpuc->n_added)
945                 goto out;
946 
947         /* Assign to counters all unassigned events.  */
948         for (i = 0; i < cpuc->n_events; i++) {
949                 struct perf_event *cp = cpuc->event[i];
950                 struct hw_perf_event *hwc = &cp->hw;
951                 int idx = hwc->idx;
952                 u64 enc;
953 
954                 if (cpuc->current_idx[i] != PIC_NO_INDEX)
955                         continue;
956 
957                 sparc_perf_event_set_period(cp, hwc, idx);
958                 cpuc->current_idx[i] = idx;
959 
960                 enc = perf_event_get_enc(cpuc->events[i]);
961                 cpuc->pcr[0] &= ~mask_for_index(idx);
962                 if (hwc->state & PERF_HES_STOPPED)
963                         cpuc->pcr[0] |= nop_for_index(idx);
964                 else
965                         cpuc->pcr[0] |= event_encoding(enc, idx);
966         }
967 out:
968         cpuc->pcr[0] |= cpuc->event[0]->hw.config_base;
969 }
970 
971 static void sparc_pmu_start(struct perf_event *event, int flags);
972 
973 /* On this PMU each PIC has it's own PCR control register.  */
974 static void calculate_multiple_pcrs(struct cpu_hw_events *cpuc)
975 {
976         int i;
977 
978         if (!cpuc->n_added)
979                 goto out;
980 
981         for (i = 0; i < cpuc->n_events; i++) {
982                 struct perf_event *cp = cpuc->event[i];
983                 struct hw_perf_event *hwc = &cp->hw;
984                 int idx = hwc->idx;
985 
986                 if (cpuc->current_idx[i] != PIC_NO_INDEX)
987                         continue;
988 
989                 cpuc->current_idx[i] = idx;
990 
991                 sparc_pmu_start(cp, PERF_EF_RELOAD);
992         }
993 out:
994         for (i = 0; i < cpuc->n_events; i++) {
995                 struct perf_event *cp = cpuc->event[i];
996                 int idx = cp->hw.idx;
997 
998                 cpuc->pcr[idx] |= cp->hw.config_base;
999         }
1000 }
1001 
1002 /* If performance event entries have been added, move existing events
1003  * around (if necessary) and then assign new entries to counters.
1004  */
1005 static void update_pcrs_for_enable(struct cpu_hw_events *cpuc)
1006 {
1007         if (cpuc->n_added)
1008                 read_in_all_counters(cpuc);
1009 
1010         if (sparc_pmu->num_pcrs == 1) {
1011                 calculate_single_pcr(cpuc);
1012         } else {
1013                 calculate_multiple_pcrs(cpuc);
1014         }
1015 }
1016 
1017 static void sparc_pmu_enable(struct pmu *pmu)
1018 {
1019         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1020         int i;
1021 
1022         if (cpuc->enabled)
1023                 return;
1024 
1025         cpuc->enabled = 1;
1026         barrier();
1027 
1028         if (cpuc->n_events)
1029                 update_pcrs_for_enable(cpuc);
1030 
1031         for (i = 0; i < sparc_pmu->num_pcrs; i++)
1032                 pcr_ops->write_pcr(i, cpuc->pcr[i]);
1033 }
1034 
1035 static void sparc_pmu_disable(struct pmu *pmu)
1036 {
1037         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1038         int i;
1039 
1040         if (!cpuc->enabled)
1041                 return;
1042 
1043         cpuc->enabled = 0;
1044         cpuc->n_added = 0;
1045 
1046         for (i = 0; i < sparc_pmu->num_pcrs; i++) {
1047                 u64 val = cpuc->pcr[i];
1048 
1049                 val &= ~(sparc_pmu->user_bit | sparc_pmu->priv_bit |
1050                          sparc_pmu->hv_bit | sparc_pmu->irq_bit);
1051                 cpuc->pcr[i] = val;
1052                 pcr_ops->write_pcr(i, cpuc->pcr[i]);
1053         }
1054 }
1055 
1056 static int active_event_index(struct cpu_hw_events *cpuc,
1057                               struct perf_event *event)
1058 {
1059         int i;
1060 
1061         for (i = 0; i < cpuc->n_events; i++) {
1062                 if (cpuc->event[i] == event)
1063                         break;
1064         }
1065         BUG_ON(i == cpuc->n_events);
1066         return cpuc->current_idx[i];
1067 }
1068 
1069 static void sparc_pmu_start(struct perf_event *event, int flags)
1070 {
1071         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1072         int idx = active_event_index(cpuc, event);
1073 
1074         if (flags & PERF_EF_RELOAD) {
1075                 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
1076                 sparc_perf_event_set_period(event, &event->hw, idx);
1077         }
1078 
1079         event->hw.state = 0;
1080 
1081         sparc_pmu_enable_event(cpuc, &event->hw, idx);
1082 }
1083 
1084 static void sparc_pmu_stop(struct perf_event *event, int flags)
1085 {
1086         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1087         int idx = active_event_index(cpuc, event);
1088 
1089         if (!(event->hw.state & PERF_HES_STOPPED)) {
1090                 sparc_pmu_disable_event(cpuc, &event->hw, idx);
1091                 event->hw.state |= PERF_HES_STOPPED;
1092         }
1093 
1094         if (!(event->hw.state & PERF_HES_UPTODATE) && (flags & PERF_EF_UPDATE)) {
1095                 sparc_perf_event_update(event, &event->hw, idx);
1096                 event->hw.state |= PERF_HES_UPTODATE;
1097         }
1098 }
1099 
1100 static void sparc_pmu_del(struct perf_event *event, int _flags)
1101 {
1102         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1103         unsigned long flags;
1104         int i;
1105 
1106         local_irq_save(flags);
1107 
1108         for (i = 0; i < cpuc->n_events; i++) {
1109                 if (event == cpuc->event[i]) {
1110                         /* Absorb the final count and turn off the
1111                          * event.
1112                          */
1113                         sparc_pmu_stop(event, PERF_EF_UPDATE);
1114 
1115                         /* Shift remaining entries down into
1116                          * the existing slot.
1117                          */
1118                         while (++i < cpuc->n_events) {
1119                                 cpuc->event[i - 1] = cpuc->event[i];
1120                                 cpuc->events[i - 1] = cpuc->events[i];
1121                                 cpuc->current_idx[i - 1] =
1122                                         cpuc->current_idx[i];
1123                         }
1124 
1125                         perf_event_update_userpage(event);
1126 
1127                         cpuc->n_events--;
1128                         break;
1129                 }
1130         }
1131 
1132         local_irq_restore(flags);
1133 }
1134 
1135 static void sparc_pmu_read(struct perf_event *event)
1136 {
1137         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1138         int idx = active_event_index(cpuc, event);
1139         struct hw_perf_event *hwc = &event->hw;
1140 
1141         sparc_perf_event_update(event, hwc, idx);
1142 }
1143 
1144 static atomic_t active_events = ATOMIC_INIT(0);
1145 static DEFINE_MUTEX(pmc_grab_mutex);
1146 
1147 static void perf_stop_nmi_watchdog(void *unused)
1148 {
1149         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1150         int i;
1151 
1152         stop_nmi_watchdog(NULL);
1153         for (i = 0; i < sparc_pmu->num_pcrs; i++)
1154                 cpuc->pcr[i] = pcr_ops->read_pcr(i);
1155 }
1156 
1157 static void perf_event_grab_pmc(void)
1158 {
1159         if (atomic_inc_not_zero(&active_events))
1160                 return;
1161 
1162         mutex_lock(&pmc_grab_mutex);
1163         if (atomic_read(&active_events) == 0) {
1164                 if (atomic_read(&nmi_active) > 0) {
1165                         on_each_cpu(perf_stop_nmi_watchdog, NULL, 1);
1166                         BUG_ON(atomic_read(&nmi_active) != 0);
1167                 }
1168                 atomic_inc(&active_events);
1169         }
1170         mutex_unlock(&pmc_grab_mutex);
1171 }
1172 
1173 static void perf_event_release_pmc(void)
1174 {
1175         if (atomic_dec_and_mutex_lock(&active_events, &pmc_grab_mutex)) {
1176                 if (atomic_read(&nmi_active) == 0)
1177                         on_each_cpu(start_nmi_watchdog, NULL, 1);
1178                 mutex_unlock(&pmc_grab_mutex);
1179         }
1180 }
1181 
1182 static const struct perf_event_map *sparc_map_cache_event(u64 config)
1183 {
1184         unsigned int cache_type, cache_op, cache_result;
1185         const struct perf_event_map *pmap;
1186 
1187         if (!sparc_pmu->cache_map)
1188                 return ERR_PTR(-ENOENT);
1189 
1190         cache_type = (config >>  0) & 0xff;
1191         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
1192                 return ERR_PTR(-EINVAL);
1193 
1194         cache_op = (config >>  8) & 0xff;
1195         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
1196                 return ERR_PTR(-EINVAL);
1197 
1198         cache_result = (config >> 16) & 0xff;
1199         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
1200                 return ERR_PTR(-EINVAL);
1201 
1202         pmap = &((*sparc_pmu->cache_map)[cache_type][cache_op][cache_result]);
1203 
1204         if (pmap->encoding == CACHE_OP_UNSUPPORTED)
1205                 return ERR_PTR(-ENOENT);
1206 
1207         if (pmap->encoding == CACHE_OP_NONSENSE)
1208                 return ERR_PTR(-EINVAL);
1209 
1210         return pmap;
1211 }
1212 
1213 static void hw_perf_event_destroy(struct perf_event *event)
1214 {
1215         perf_event_release_pmc();
1216 }
1217 
1218 /* Make sure all events can be scheduled into the hardware at
1219  * the same time.  This is simplified by the fact that we only
1220  * need to support 2 simultaneous HW events.
1221  *
1222  * As a side effect, the evts[]->hw.idx values will be assigned
1223  * on success.  These are pending indexes.  When the events are
1224  * actually programmed into the chip, these values will propagate
1225  * to the per-cpu cpuc->current_idx[] slots, see the code in
1226  * maybe_change_configuration() for details.
1227  */
1228 static int sparc_check_constraints(struct perf_event **evts,
1229                                    unsigned long *events, int n_ev)
1230 {
1231         u8 msk0 = 0, msk1 = 0;
1232         int idx0 = 0;
1233 
1234         /* This case is possible when we are invoked from
1235          * hw_perf_group_sched_in().
1236          */
1237         if (!n_ev)
1238                 return 0;
1239 
1240         if (n_ev > sparc_pmu->max_hw_events)
1241                 return -1;
1242 
1243         if (!(sparc_pmu->flags & SPARC_PMU_HAS_CONFLICTS)) {
1244                 int i;
1245 
1246                 for (i = 0; i < n_ev; i++)
1247                         evts[i]->hw.idx = i;
1248                 return 0;
1249         }
1250 
1251         msk0 = perf_event_get_msk(events[0]);
1252         if (n_ev == 1) {
1253                 if (msk0 & PIC_LOWER)
1254                         idx0 = 1;
1255                 goto success;
1256         }
1257         BUG_ON(n_ev != 2);
1258         msk1 = perf_event_get_msk(events[1]);
1259 
1260         /* If both events can go on any counter, OK.  */
1261         if (msk0 == (PIC_UPPER | PIC_LOWER) &&
1262             msk1 == (PIC_UPPER | PIC_LOWER))
1263                 goto success;
1264 
1265         /* If one event is limited to a specific counter,
1266          * and the other can go on both, OK.
1267          */
1268         if ((msk0 == PIC_UPPER || msk0 == PIC_LOWER) &&
1269             msk1 == (PIC_UPPER | PIC_LOWER)) {
1270                 if (msk0 & PIC_LOWER)
1271                         idx0 = 1;
1272                 goto success;
1273         }
1274 
1275         if ((msk1 == PIC_UPPER || msk1 == PIC_LOWER) &&
1276             msk0 == (PIC_UPPER | PIC_LOWER)) {
1277                 if (msk1 & PIC_UPPER)
1278                         idx0 = 1;
1279                 goto success;
1280         }
1281 
1282         /* If the events are fixed to different counters, OK.  */
1283         if ((msk0 == PIC_UPPER && msk1 == PIC_LOWER) ||
1284             (msk0 == PIC_LOWER && msk1 == PIC_UPPER)) {
1285                 if (msk0 & PIC_LOWER)
1286                         idx0 = 1;
1287                 goto success;
1288         }
1289 
1290         /* Otherwise, there is a conflict.  */
1291         return -1;
1292 
1293 success:
1294         evts[0]->hw.idx = idx0;
1295         if (n_ev == 2)
1296                 evts[1]->hw.idx = idx0 ^ 1;
1297         return 0;
1298 }
1299 
1300 static int check_excludes(struct perf_event **evts, int n_prev, int n_new)
1301 {
1302         int eu = 0, ek = 0, eh = 0;
1303         struct perf_event *event;
1304         int i, n, first;
1305 
1306         if (!(sparc_pmu->flags & SPARC_PMU_ALL_EXCLUDES_SAME))
1307                 return 0;
1308 
1309         n = n_prev + n_new;
1310         if (n <= 1)
1311                 return 0;
1312 
1313         first = 1;
1314         for (i = 0; i < n; i++) {
1315                 event = evts[i];
1316                 if (first) {
1317                         eu = event->attr.exclude_user;
1318                         ek = event->attr.exclude_kernel;
1319                         eh = event->attr.exclude_hv;
1320                         first = 0;
1321                 } else if (event->attr.exclude_user != eu ||
1322                            event->attr.exclude_kernel != ek ||
1323                            event->attr.exclude_hv != eh) {
1324                         return -EAGAIN;
1325                 }
1326         }
1327 
1328         return 0;
1329 }
1330 
1331 static int collect_events(struct perf_event *group, int max_count,
1332                           struct perf_event *evts[], unsigned long *events,
1333                           int *current_idx)
1334 {
1335         struct perf_event *event;
1336         int n = 0;
1337 
1338         if (!is_software_event(group)) {
1339                 if (n >= max_count)
1340                         return -1;
1341                 evts[n] = group;
1342                 events[n] = group->hw.event_base;
1343                 current_idx[n++] = PIC_NO_INDEX;
1344         }
1345         for_each_sibling_event(event, group) {
1346                 if (!is_software_event(event) &&
1347                     event->state != PERF_EVENT_STATE_OFF) {
1348                         if (n >= max_count)
1349                                 return -1;
1350                         evts[n] = event;
1351                         events[n] = event->hw.event_base;
1352                         current_idx[n++] = PIC_NO_INDEX;
1353                 }
1354         }
1355         return n;
1356 }
1357 
1358 static int sparc_pmu_add(struct perf_event *event, int ef_flags)
1359 {
1360         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1361         int n0, ret = -EAGAIN;
1362         unsigned long flags;
1363 
1364         local_irq_save(flags);
1365 
1366         n0 = cpuc->n_events;
1367         if (n0 >= sparc_pmu->max_hw_events)
1368                 goto out;
1369 
1370         cpuc->event[n0] = event;
1371         cpuc->events[n0] = event->hw.event_base;
1372         cpuc->current_idx[n0] = PIC_NO_INDEX;
1373 
1374         event->hw.state = PERF_HES_UPTODATE;
1375         if (!(ef_flags & PERF_EF_START))
1376                 event->hw.state |= PERF_HES_STOPPED;
1377 
1378         /*
1379          * If group events scheduling transaction was started,
1380          * skip the schedulability test here, it will be performed
1381          * at commit time(->commit_txn) as a whole
1382          */
1383         if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
1384                 goto nocheck;
1385 
1386         if (check_excludes(cpuc->event, n0, 1))
1387                 goto out;
1388         if (sparc_check_constraints(cpuc->event, cpuc->events, n0 + 1))
1389                 goto out;
1390 
1391 nocheck:
1392         cpuc->n_events++;
1393         cpuc->n_added++;
1394 
1395         ret = 0;
1396 out:
1397         local_irq_restore(flags);
1398         return ret;
1399 }
1400 
1401 static int sparc_pmu_event_init(struct perf_event *event)
1402 {
1403         struct perf_event_attr *attr = &event->attr;
1404         struct perf_event *evts[MAX_HWEVENTS];
1405         struct hw_perf_event *hwc = &event->hw;
1406         unsigned long events[MAX_HWEVENTS];
1407         int current_idx_dmy[MAX_HWEVENTS];
1408         const struct perf_event_map *pmap;
1409         int n;
1410 
1411         if (atomic_read(&nmi_active) < 0)
1412                 return -ENODEV;
1413 
1414         /* does not support taken branch sampling */
1415         if (has_branch_stack(event))
1416                 return -EOPNOTSUPP;
1417 
1418         switch (attr->type) {
1419         case PERF_TYPE_HARDWARE:
1420                 if (attr->config >= sparc_pmu->max_events)
1421                         return -EINVAL;
1422                 pmap = sparc_pmu->event_map(attr->config);
1423                 break;
1424 
1425         case PERF_TYPE_HW_CACHE:
1426                 pmap = sparc_map_cache_event(attr->config);
1427                 if (IS_ERR(pmap))
1428                         return PTR_ERR(pmap);
1429                 break;
1430 
1431         case PERF_TYPE_RAW:
1432                 pmap = NULL;
1433                 break;
1434 
1435         default:
1436                 return -ENOENT;
1437 
1438         }
1439 
1440         if (pmap) {
1441                 hwc->event_base = perf_event_encode(pmap);
1442         } else {
1443                 /*
1444                  * User gives us "(encoding << 16) | pic_mask" for
1445                  * PERF_TYPE_RAW events.
1446                  */
1447                 hwc->event_base = attr->config;
1448         }
1449 
1450         /* We save the enable bits in the config_base.  */
1451         hwc->config_base = sparc_pmu->irq_bit;
1452         if (!attr->exclude_user)
1453                 hwc->config_base |= sparc_pmu->user_bit;
1454         if (!attr->exclude_kernel)
1455                 hwc->config_base |= sparc_pmu->priv_bit;
1456         if (!attr->exclude_hv)
1457                 hwc->config_base |= sparc_pmu->hv_bit;
1458 
1459         n = 0;
1460         if (event->group_leader != event) {
1461                 n = collect_events(event->group_leader,
1462                                    sparc_pmu->max_hw_events - 1,
1463                                    evts, events, current_idx_dmy);
1464                 if (n < 0)
1465                         return -EINVAL;
1466         }
1467         events[n] = hwc->event_base;
1468         evts[n] = event;
1469 
1470         if (check_excludes(evts, n, 1))
1471                 return -EINVAL;
1472 
1473         if (sparc_check_constraints(evts, events, n + 1))
1474                 return -EINVAL;
1475 
1476         hwc->idx = PIC_NO_INDEX;
1477 
1478         /* Try to do all error checking before this point, as unwinding
1479          * state after grabbing the PMC is difficult.
1480          */
1481         perf_event_grab_pmc();
1482         event->destroy = hw_perf_event_destroy;
1483 
1484         if (!hwc->sample_period) {
1485                 hwc->sample_period = MAX_PERIOD;
1486                 hwc->last_period = hwc->sample_period;
1487                 local64_set(&hwc->period_left, hwc->sample_period);
1488         }
1489 
1490         return 0;
1491 }
1492 
1493 /*
1494  * Start group events scheduling transaction
1495  * Set the flag to make pmu::enable() not perform the
1496  * schedulability test, it will be performed at commit time
1497  */
1498 static void sparc_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags)
1499 {
1500         struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
1501 
1502         WARN_ON_ONCE(cpuhw->txn_flags);         /* txn already in flight */
1503 
1504         cpuhw->txn_flags = txn_flags;
1505         if (txn_flags & ~PERF_PMU_TXN_ADD)
1506                 return;
1507 
1508         perf_pmu_disable(pmu);
1509 }
1510 
1511 /*
1512  * Stop group events scheduling transaction
1513  * Clear the flag and pmu::enable() will perform the
1514  * schedulability test.
1515  */
1516 static void sparc_pmu_cancel_txn(struct pmu *pmu)
1517 {
1518         struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
1519         unsigned int txn_flags;
1520 
1521         WARN_ON_ONCE(!cpuhw->txn_flags);        /* no txn in flight */
1522 
1523         txn_flags = cpuhw->txn_flags;
1524         cpuhw->txn_flags = 0;
1525         if (txn_flags & ~PERF_PMU_TXN_ADD)
1526                 return;
1527 
1528         perf_pmu_enable(pmu);
1529 }
1530 
1531 /*
1532  * Commit group events scheduling transaction
1533  * Perform the group schedulability test as a whole
1534  * Return 0 if success
1535  */
1536 static int sparc_pmu_commit_txn(struct pmu *pmu)
1537 {
1538         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1539         int n;
1540 
1541         if (!sparc_pmu)
1542                 return -EINVAL;
1543 
1544         WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */
1545 
1546         if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) {
1547                 cpuc->txn_flags = 0;
1548                 return 0;
1549         }
1550 
1551         n = cpuc->n_events;
1552         if (check_excludes(cpuc->event, 0, n))
1553                 return -EINVAL;
1554         if (sparc_check_constraints(cpuc->event, cpuc->events, n))
1555                 return -EAGAIN;
1556 
1557         cpuc->txn_flags = 0;
1558         perf_pmu_enable(pmu);
1559         return 0;
1560 }
1561 
1562 static struct pmu pmu = {
1563         .pmu_enable     = sparc_pmu_enable,
1564         .pmu_disable    = sparc_pmu_disable,
1565         .event_init     = sparc_pmu_event_init,
1566         .add            = sparc_pmu_add,
1567         .del            = sparc_pmu_del,
1568         .start          = sparc_pmu_start,
1569         .stop           = sparc_pmu_stop,
1570         .read           = sparc_pmu_read,
1571         .start_txn      = sparc_pmu_start_txn,
1572         .cancel_txn     = sparc_pmu_cancel_txn,
1573         .commit_txn     = sparc_pmu_commit_txn,
1574 };
1575 
1576 void perf_event_print_debug(void)
1577 {
1578         unsigned long flags;
1579         int cpu, i;
1580 
1581         if (!sparc_pmu)
1582                 return;
1583 
1584         local_irq_save(flags);
1585 
1586         cpu = smp_processor_id();
1587 
1588         pr_info("\n");
1589         for (i = 0; i < sparc_pmu->num_pcrs; i++)
1590                 pr_info("CPU#%d: PCR%d[%016llx]\n",
1591                         cpu, i, pcr_ops->read_pcr(i));
1592         for (i = 0; i < sparc_pmu->num_pic_regs; i++)
1593                 pr_info("CPU#%d: PIC%d[%016llx]\n",
1594                         cpu, i, pcr_ops->read_pic(i));
1595 
1596         local_irq_restore(flags);
1597 }
1598 
1599 static int __kprobes perf_event_nmi_handler(struct notifier_block *self,
1600                                             unsigned long cmd, void *__args)
1601 {
1602         struct die_args *args = __args;
1603         struct perf_sample_data data;
1604         struct cpu_hw_events *cpuc;
1605         struct pt_regs *regs;
1606         int i;
1607 
1608         if (!atomic_read(&active_events))
1609                 return NOTIFY_DONE;
1610 
1611         switch (cmd) {
1612         case DIE_NMI:
1613                 break;
1614 
1615         default:
1616                 return NOTIFY_DONE;
1617         }
1618 
1619         regs = args->regs;
1620 
1621         cpuc = this_cpu_ptr(&cpu_hw_events);
1622 
1623         /* If the PMU has the TOE IRQ enable bits, we need to do a
1624          * dummy write to the %pcr to clear the overflow bits and thus
1625          * the interrupt.
1626          *
1627          * Do this before we peek at the counters to determine
1628          * overflow so we don't lose any events.
1629          */
1630         if (sparc_pmu->irq_bit &&
1631             sparc_pmu->num_pcrs == 1)
1632                 pcr_ops->write_pcr(0, cpuc->pcr[0]);
1633 
1634         for (i = 0; i < cpuc->n_events; i++) {
1635                 struct perf_event *event = cpuc->event[i];
1636                 int idx = cpuc->current_idx[i];
1637                 struct hw_perf_event *hwc;
1638                 u64 val;
1639 
1640                 if (sparc_pmu->irq_bit &&
1641                     sparc_pmu->num_pcrs > 1)
1642                         pcr_ops->write_pcr(idx, cpuc->pcr[idx]);
1643 
1644                 hwc = &event->hw;
1645                 val = sparc_perf_event_update(event, hwc, idx);
1646                 if (val & (1ULL << 31))
1647                         continue;
1648 
1649                 perf_sample_data_init(&data, 0, hwc->last_period);
1650                 if (!sparc_perf_event_set_period(event, hwc, idx))
1651                         continue;
1652 
1653                 if (perf_event_overflow(event, &data, regs))
1654                         sparc_pmu_stop(event, 0);
1655         }
1656 
1657         return NOTIFY_STOP;
1658 }
1659 
1660 static __read_mostly struct notifier_block perf_event_nmi_notifier = {
1661         .notifier_call          = perf_event_nmi_handler,
1662 };
1663 
1664 static bool __init supported_pmu(void)
1665 {
1666         if (!strcmp(sparc_pmu_type, "ultra3") ||
1667             !strcmp(sparc_pmu_type, "ultra3+") ||
1668             !strcmp(sparc_pmu_type, "ultra3i") ||
1669             !strcmp(sparc_pmu_type, "ultra4+")) {
1670                 sparc_pmu = &ultra3_pmu;
1671                 return true;
1672         }
1673         if (!strcmp(sparc_pmu_type, "niagara")) {
1674                 sparc_pmu = &niagara1_pmu;
1675                 return true;
1676         }
1677         if (!strcmp(sparc_pmu_type, "niagara2") ||
1678             !strcmp(sparc_pmu_type, "niagara3")) {
1679                 sparc_pmu = &niagara2_pmu;
1680                 return true;
1681         }
1682         if (!strcmp(sparc_pmu_type, "niagara4") ||
1683             !strcmp(sparc_pmu_type, "niagara5")) {
1684                 sparc_pmu = &niagara4_pmu;
1685                 return true;
1686         }
1687         if (!strcmp(sparc_pmu_type, "sparc-m7")) {
1688                 sparc_pmu = &sparc_m7_pmu;
1689                 return true;
1690         }
1691         return false;
1692 }
1693 
1694 static int __init init_hw_perf_events(void)
1695 {
1696         int err;
1697 
1698         pr_info("Performance events: ");
1699 
1700         err = pcr_arch_init();
1701         if (err || !supported_pmu()) {
1702                 pr_cont("No support for PMU type '%s'\n", sparc_pmu_type);
1703                 return 0;
1704         }
1705 
1706         pr_cont("Supported PMU type is '%s'\n", sparc_pmu_type);
1707 
1708         perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
1709         register_die_notifier(&perf_event_nmi_notifier);
1710 
1711         return 0;
1712 }
1713 pure_initcall(init_hw_perf_events);
1714 
1715 void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
1716                            struct pt_regs *regs)
1717 {
1718         unsigned long ksp, fp;
1719 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1720         int graph = 0;
1721 #endif
1722 
1723         stack_trace_flush();
1724 
1725         perf_callchain_store(entry, regs->tpc);
1726 
1727         ksp = regs->u_regs[UREG_I6];
1728         fp = ksp + STACK_BIAS;
1729         do {
1730                 struct sparc_stackf *sf;
1731                 struct pt_regs *regs;
1732                 unsigned long pc;
1733 
1734                 if (!kstack_valid(current_thread_info(), fp))
1735                         break;
1736 
1737                 sf = (struct sparc_stackf *) fp;
1738                 regs = (struct pt_regs *) (sf + 1);
1739 
1740                 if (kstack_is_trap_frame(current_thread_info(), regs)) {
1741                         if (user_mode(regs))
1742                                 break;
1743                         pc = regs->tpc;
1744                         fp = regs->u_regs[UREG_I6] + STACK_BIAS;
1745                 } else {
1746                         pc = sf->callers_pc;
1747                         fp = (unsigned long)sf->fp + STACK_BIAS;
1748                 }
1749                 perf_callchain_store(entry, pc);
1750 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1751                 if ((pc + 8UL) == (unsigned long) &return_to_handler) {
1752                         int index = current->curr_ret_stack;
1753                         if (current->ret_stack && index >= graph) {
1754                                 pc = current->ret_stack[index - graph].ret;
1755                                 perf_callchain_store(entry, pc);
1756                                 graph++;
1757                         }
1758                 }
1759 #endif
1760         } while (entry->nr < entry->max_stack);
1761 }
1762 
1763 static inline int
1764 valid_user_frame(const void __user *fp, unsigned long size)
1765 {
1766         /* addresses should be at least 4-byte aligned */
1767         if (((unsigned long) fp) & 3)
1768                 return 0;
1769 
1770         return (__range_not_ok(fp, size, TASK_SIZE) == 0);
1771 }
1772 
1773 static void perf_callchain_user_64(struct perf_callchain_entry_ctx *entry,
1774                                    struct pt_regs *regs)
1775 {
1776         unsigned long ufp;
1777 
1778         ufp = regs->u_regs[UREG_FP] + STACK_BIAS;
1779         do {
1780                 struct sparc_stackf __user *usf;
1781                 struct sparc_stackf sf;
1782                 unsigned long pc;
1783 
1784                 usf = (struct sparc_stackf __user *)ufp;
1785                 if (!valid_user_frame(usf, sizeof(sf)))
1786                         break;
1787 
1788                 if (__copy_from_user_inatomic(&sf, usf, sizeof(sf)))
1789                         break;
1790 
1791                 pc = sf.callers_pc;
1792                 ufp = (unsigned long)sf.fp + STACK_BIAS;
1793                 perf_callchain_store(entry, pc);
1794         } while (entry->nr < entry->max_stack);
1795 }
1796 
1797 static void perf_callchain_user_32(struct perf_callchain_entry_ctx *entry,
1798                                    struct pt_regs *regs)
1799 {
1800         unsigned long ufp;
1801 
1802         ufp = regs->u_regs[UREG_FP] & 0xffffffffUL;
1803         do {
1804                 unsigned long pc;
1805 
1806                 if (thread32_stack_is_64bit(ufp)) {
1807                         struct sparc_stackf __user *usf;
1808                         struct sparc_stackf sf;
1809 
1810                         ufp += STACK_BIAS;
1811                         usf = (struct sparc_stackf __user *)ufp;
1812                         if (__copy_from_user_inatomic(&sf, usf, sizeof(sf)))
1813                                 break;
1814                         pc = sf.callers_pc & 0xffffffff;
1815                         ufp = ((unsigned long) sf.fp) & 0xffffffff;
1816                 } else {
1817                         struct sparc_stackf32 __user *usf;
1818                         struct sparc_stackf32 sf;
1819                         usf = (struct sparc_stackf32 __user *)ufp;
1820                         if (__copy_from_user_inatomic(&sf, usf, sizeof(sf)))
1821                                 break;
1822                         pc = sf.callers_pc;
1823                         ufp = (unsigned long)sf.fp;
1824                 }
1825                 perf_callchain_store(entry, pc);
1826         } while (entry->nr < entry->max_stack);
1827 }
1828 
1829 void
1830 perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
1831 {
1832         u64 saved_fault_address = current_thread_info()->fault_address;
1833         u8 saved_fault_code = get_thread_fault_code();
1834         mm_segment_t old_fs;
1835 
1836         perf_callchain_store(entry, regs->tpc);
1837 
1838         if (!current->mm)
1839                 return;
1840 
1841         old_fs = get_fs();
1842         set_fs(USER_DS);
1843 
1844         flushw_user();
1845 
1846         pagefault_disable();
1847 
1848         if (test_thread_flag(TIF_32BIT))
1849                 perf_callchain_user_32(entry, regs);
1850         else
1851                 perf_callchain_user_64(entry, regs);
1852 
1853         pagefault_enable();
1854 
1855         set_fs(old_fs);
1856         set_thread_fault_code(saved_fault_code);
1857         current_thread_info()->fault_address = saved_fault_address;
1858 }
1859 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp