~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/ia64/kernel/perfmon.c

Version: ~ [ linux-5.1-rc2 ] ~ [ linux-5.0.4 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.31 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.108 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.165 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.177 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.137 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.63 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * This file implements the perfmon-2 subsystem which is used
  3  * to program the IA-64 Performance Monitoring Unit (PMU).
  4  *
  5  * The initial version of perfmon.c was written by
  6  * Ganesh Venkitachalam, IBM Corp.
  7  *
  8  * Then it was modified for perfmon-1.x by Stephane Eranian and
  9  * David Mosberger, Hewlett Packard Co.
 10  *
 11  * Version Perfmon-2.x is a rewrite of perfmon-1.x
 12  * by Stephane Eranian, Hewlett Packard Co.
 13  *
 14  * Copyright (C) 1999-2005  Hewlett Packard Co
 15  *               Stephane Eranian <eranian@hpl.hp.com>
 16  *               David Mosberger-Tang <davidm@hpl.hp.com>
 17  *
 18  * More information about perfmon available at:
 19  *      http://www.hpl.hp.com/research/linux/perfmon
 20  */
 21 
 22 #include <linux/module.h>
 23 #include <linux/kernel.h>
 24 #include <linux/sched.h>
 25 #include <linux/sched/task.h>
 26 #include <linux/sched/task_stack.h>
 27 #include <linux/interrupt.h>
 28 #include <linux/proc_fs.h>
 29 #include <linux/seq_file.h>
 30 #include <linux/init.h>
 31 #include <linux/vmalloc.h>
 32 #include <linux/mm.h>
 33 #include <linux/sysctl.h>
 34 #include <linux/list.h>
 35 #include <linux/file.h>
 36 #include <linux/poll.h>
 37 #include <linux/vfs.h>
 38 #include <linux/smp.h>
 39 #include <linux/pagemap.h>
 40 #include <linux/mount.h>
 41 #include <linux/bitops.h>
 42 #include <linux/capability.h>
 43 #include <linux/rcupdate.h>
 44 #include <linux/completion.h>
 45 #include <linux/tracehook.h>
 46 #include <linux/slab.h>
 47 #include <linux/cpu.h>
 48 
 49 #include <asm/errno.h>
 50 #include <asm/intrinsics.h>
 51 #include <asm/page.h>
 52 #include <asm/perfmon.h>
 53 #include <asm/processor.h>
 54 #include <asm/signal.h>
 55 #include <linux/uaccess.h>
 56 #include <asm/delay.h>
 57 
 58 #ifdef CONFIG_PERFMON
 59 /*
 60  * perfmon context state
 61  */
 62 #define PFM_CTX_UNLOADED        1       /* context is not loaded onto any task */
 63 #define PFM_CTX_LOADED          2       /* context is loaded onto a task */
 64 #define PFM_CTX_MASKED          3       /* context is loaded but monitoring is masked due to overflow */
 65 #define PFM_CTX_ZOMBIE          4       /* owner of the context is closing it */
 66 
 67 #define PFM_INVALID_ACTIVATION  (~0UL)
 68 
 69 #define PFM_NUM_PMC_REGS        64      /* PMC save area for ctxsw */
 70 #define PFM_NUM_PMD_REGS        64      /* PMD save area for ctxsw */
 71 
 72 /*
 73  * depth of message queue
 74  */
 75 #define PFM_MAX_MSGS            32
 76 #define PFM_CTXQ_EMPTY(g)       ((g)->ctx_msgq_head == (g)->ctx_msgq_tail)
 77 
 78 /*
 79  * type of a PMU register (bitmask).
 80  * bitmask structure:
 81  *      bit0   : register implemented
 82  *      bit1   : end marker
 83  *      bit2-3 : reserved
 84  *      bit4   : pmc has pmc.pm
 85  *      bit5   : pmc controls a counter (has pmc.oi), pmd is used as counter
 86  *      bit6-7 : register type
 87  *      bit8-31: reserved
 88  */
 89 #define PFM_REG_NOTIMPL         0x0 /* not implemented at all */
 90 #define PFM_REG_IMPL            0x1 /* register implemented */
 91 #define PFM_REG_END             0x2 /* end marker */
 92 #define PFM_REG_MONITOR         (0x1<<4|PFM_REG_IMPL) /* a PMC with a pmc.pm field only */
 93 #define PFM_REG_COUNTING        (0x2<<4|PFM_REG_MONITOR) /* a monitor + pmc.oi+ PMD used as a counter */
 94 #define PFM_REG_CONTROL         (0x4<<4|PFM_REG_IMPL) /* PMU control register */
 95 #define PFM_REG_CONFIG          (0x8<<4|PFM_REG_IMPL) /* configuration register */
 96 #define PFM_REG_BUFFER          (0xc<<4|PFM_REG_IMPL) /* PMD used as buffer */
 97 
 98 #define PMC_IS_LAST(i)  (pmu_conf->pmc_desc[i].type & PFM_REG_END)
 99 #define PMD_IS_LAST(i)  (pmu_conf->pmd_desc[i].type & PFM_REG_END)
100 
101 #define PMC_OVFL_NOTIFY(ctx, i) ((ctx)->ctx_pmds[i].flags &  PFM_REGFL_OVFL_NOTIFY)
102 
103 /* i assumed unsigned */
104 #define PMC_IS_IMPL(i)    (i< PMU_MAX_PMCS && (pmu_conf->pmc_desc[i].type & PFM_REG_IMPL))
105 #define PMD_IS_IMPL(i)    (i< PMU_MAX_PMDS && (pmu_conf->pmd_desc[i].type & PFM_REG_IMPL))
106 
107 /* XXX: these assume that register i is implemented */
108 #define PMD_IS_COUNTING(i) ((pmu_conf->pmd_desc[i].type & PFM_REG_COUNTING) == PFM_REG_COUNTING)
109 #define PMC_IS_COUNTING(i) ((pmu_conf->pmc_desc[i].type & PFM_REG_COUNTING) == PFM_REG_COUNTING)
110 #define PMC_IS_MONITOR(i)  ((pmu_conf->pmc_desc[i].type & PFM_REG_MONITOR)  == PFM_REG_MONITOR)
111 #define PMC_IS_CONTROL(i)  ((pmu_conf->pmc_desc[i].type & PFM_REG_CONTROL)  == PFM_REG_CONTROL)
112 
113 #define PMC_DFL_VAL(i)     pmu_conf->pmc_desc[i].default_value
114 #define PMC_RSVD_MASK(i)   pmu_conf->pmc_desc[i].reserved_mask
115 #define PMD_PMD_DEP(i)     pmu_conf->pmd_desc[i].dep_pmd[0]
116 #define PMC_PMD_DEP(i)     pmu_conf->pmc_desc[i].dep_pmd[0]
117 
118 #define PFM_NUM_IBRS      IA64_NUM_DBG_REGS
119 #define PFM_NUM_DBRS      IA64_NUM_DBG_REGS
120 
121 #define CTX_OVFL_NOBLOCK(c)     ((c)->ctx_fl_block == 0)
122 #define CTX_HAS_SMPL(c)         ((c)->ctx_fl_is_sampling)
123 #define PFM_CTX_TASK(h)         (h)->ctx_task
124 
125 #define PMU_PMC_OI              5 /* position of pmc.oi bit */
126 
127 /* XXX: does not support more than 64 PMDs */
128 #define CTX_USED_PMD(ctx, mask) (ctx)->ctx_used_pmds[0] |= (mask)
129 #define CTX_IS_USED_PMD(ctx, c) (((ctx)->ctx_used_pmds[0] & (1UL << (c))) != 0UL)
130 
131 #define CTX_USED_MONITOR(ctx, mask) (ctx)->ctx_used_monitors[0] |= (mask)
132 
133 #define CTX_USED_IBR(ctx,n)     (ctx)->ctx_used_ibrs[(n)>>6] |= 1UL<< ((n) % 64)
134 #define CTX_USED_DBR(ctx,n)     (ctx)->ctx_used_dbrs[(n)>>6] |= 1UL<< ((n) % 64)
135 #define CTX_USES_DBREGS(ctx)    (((pfm_context_t *)(ctx))->ctx_fl_using_dbreg==1)
136 #define PFM_CODE_RR     0       /* requesting code range restriction */
137 #define PFM_DATA_RR     1       /* requestion data range restriction */
138 
139 #define PFM_CPUINFO_CLEAR(v)    pfm_get_cpu_var(pfm_syst_info) &= ~(v)
140 #define PFM_CPUINFO_SET(v)      pfm_get_cpu_var(pfm_syst_info) |= (v)
141 #define PFM_CPUINFO_GET()       pfm_get_cpu_var(pfm_syst_info)
142 
143 #define RDEP(x) (1UL<<(x))
144 
145 /*
146  * context protection macros
147  * in SMP:
148  *      - we need to protect against CPU concurrency (spin_lock)
149  *      - we need to protect against PMU overflow interrupts (local_irq_disable)
150  * in UP:
151  *      - we need to protect against PMU overflow interrupts (local_irq_disable)
152  *
153  * spin_lock_irqsave()/spin_unlock_irqrestore():
154  *      in SMP: local_irq_disable + spin_lock
155  *      in UP : local_irq_disable
156  *
157  * spin_lock()/spin_lock():
158  *      in UP : removed automatically
159  *      in SMP: protect against context accesses from other CPU. interrupts
160  *              are not masked. This is useful for the PMU interrupt handler
161  *              because we know we will not get PMU concurrency in that code.
162  */
163 #define PROTECT_CTX(c, f) \
164         do {  \
165                 DPRINT(("spinlock_irq_save ctx %p by [%d]\n", c, task_pid_nr(current))); \
166                 spin_lock_irqsave(&(c)->ctx_lock, f); \
167                 DPRINT(("spinlocked ctx %p  by [%d]\n", c, task_pid_nr(current))); \
168         } while(0)
169 
170 #define UNPROTECT_CTX(c, f) \
171         do { \
172                 DPRINT(("spinlock_irq_restore ctx %p by [%d]\n", c, task_pid_nr(current))); \
173                 spin_unlock_irqrestore(&(c)->ctx_lock, f); \
174         } while(0)
175 
176 #define PROTECT_CTX_NOPRINT(c, f) \
177         do {  \
178                 spin_lock_irqsave(&(c)->ctx_lock, f); \
179         } while(0)
180 
181 
182 #define UNPROTECT_CTX_NOPRINT(c, f) \
183         do { \
184                 spin_unlock_irqrestore(&(c)->ctx_lock, f); \
185         } while(0)
186 
187 
188 #define PROTECT_CTX_NOIRQ(c) \
189         do {  \
190                 spin_lock(&(c)->ctx_lock); \
191         } while(0)
192 
193 #define UNPROTECT_CTX_NOIRQ(c) \
194         do { \
195                 spin_unlock(&(c)->ctx_lock); \
196         } while(0)
197 
198 
199 #ifdef CONFIG_SMP
200 
201 #define GET_ACTIVATION()        pfm_get_cpu_var(pmu_activation_number)
202 #define INC_ACTIVATION()        pfm_get_cpu_var(pmu_activation_number)++
203 #define SET_ACTIVATION(c)       (c)->ctx_last_activation = GET_ACTIVATION()
204 
205 #else /* !CONFIG_SMP */
206 #define SET_ACTIVATION(t)       do {} while(0)
207 #define GET_ACTIVATION(t)       do {} while(0)
208 #define INC_ACTIVATION(t)       do {} while(0)
209 #endif /* CONFIG_SMP */
210 
211 #define SET_PMU_OWNER(t, c)     do { pfm_get_cpu_var(pmu_owner) = (t); pfm_get_cpu_var(pmu_ctx) = (c); } while(0)
212 #define GET_PMU_OWNER()         pfm_get_cpu_var(pmu_owner)
213 #define GET_PMU_CTX()           pfm_get_cpu_var(pmu_ctx)
214 
215 #define LOCK_PFS(g)             spin_lock_irqsave(&pfm_sessions.pfs_lock, g)
216 #define UNLOCK_PFS(g)           spin_unlock_irqrestore(&pfm_sessions.pfs_lock, g)
217 
218 #define PFM_REG_RETFLAG_SET(flags, val) do { flags &= ~PFM_REG_RETFL_MASK; flags |= (val); } while(0)
219 
220 /*
221  * cmp0 must be the value of pmc0
222  */
223 #define PMC0_HAS_OVFL(cmp0)  (cmp0 & ~0x1UL)
224 
225 #define PFMFS_MAGIC 0xa0b4d889
226 
227 /*
228  * debugging
229  */
230 #define PFM_DEBUGGING 1
231 #ifdef PFM_DEBUGGING
232 #define DPRINT(a) \
233         do { \
234                 if (unlikely(pfm_sysctl.debug >0)) { printk("%s.%d: CPU%d [%d] ", __func__, __LINE__, smp_processor_id(), task_pid_nr(current)); printk a; } \
235         } while (0)
236 
237 #define DPRINT_ovfl(a) \
238         do { \
239                 if (unlikely(pfm_sysctl.debug > 0 && pfm_sysctl.debug_ovfl >0)) { printk("%s.%d: CPU%d [%d] ", __func__, __LINE__, smp_processor_id(), task_pid_nr(current)); printk a; } \
240         } while (0)
241 #endif
242 
243 /*
244  * 64-bit software counter structure
245  *
246  * the next_reset_type is applied to the next call to pfm_reset_regs()
247  */
248 typedef struct {
249         unsigned long   val;            /* virtual 64bit counter value */
250         unsigned long   lval;           /* last reset value */
251         unsigned long   long_reset;     /* reset value on sampling overflow */
252         unsigned long   short_reset;    /* reset value on overflow */
253         unsigned long   reset_pmds[4];  /* which other pmds to reset when this counter overflows */
254         unsigned long   smpl_pmds[4];   /* which pmds are accessed when counter overflow */
255         unsigned long   seed;           /* seed for random-number generator */
256         unsigned long   mask;           /* mask for random-number generator */
257         unsigned int    flags;          /* notify/do not notify */
258         unsigned long   eventid;        /* overflow event identifier */
259 } pfm_counter_t;
260 
261 /*
262  * context flags
263  */
264 typedef struct {
265         unsigned int block:1;           /* when 1, task will blocked on user notifications */
266         unsigned int system:1;          /* do system wide monitoring */
267         unsigned int using_dbreg:1;     /* using range restrictions (debug registers) */
268         unsigned int is_sampling:1;     /* true if using a custom format */
269         unsigned int excl_idle:1;       /* exclude idle task in system wide session */
270         unsigned int going_zombie:1;    /* context is zombie (MASKED+blocking) */
271         unsigned int trap_reason:2;     /* reason for going into pfm_handle_work() */
272         unsigned int no_msg:1;          /* no message sent on overflow */
273         unsigned int can_restart:1;     /* allowed to issue a PFM_RESTART */
274         unsigned int reserved:22;
275 } pfm_context_flags_t;
276 
277 #define PFM_TRAP_REASON_NONE            0x0     /* default value */
278 #define PFM_TRAP_REASON_BLOCK           0x1     /* we need to block on overflow */
279 #define PFM_TRAP_REASON_RESET           0x2     /* we need to reset PMDs */
280 
281 
282 /*
283  * perfmon context: encapsulates all the state of a monitoring session
284  */
285 
286 typedef struct pfm_context {
287         spinlock_t              ctx_lock;               /* context protection */
288 
289         pfm_context_flags_t     ctx_flags;              /* bitmask of flags  (block reason incl.) */
290         unsigned int            ctx_state;              /* state: active/inactive (no bitfield) */
291 
292         struct task_struct      *ctx_task;              /* task to which context is attached */
293 
294         unsigned long           ctx_ovfl_regs[4];       /* which registers overflowed (notification) */
295 
296         struct completion       ctx_restart_done;       /* use for blocking notification mode */
297 
298         unsigned long           ctx_used_pmds[4];       /* bitmask of PMD used            */
299         unsigned long           ctx_all_pmds[4];        /* bitmask of all accessible PMDs */
300         unsigned long           ctx_reload_pmds[4];     /* bitmask of force reload PMD on ctxsw in */
301 
302         unsigned long           ctx_all_pmcs[4];        /* bitmask of all accessible PMCs */
303         unsigned long           ctx_reload_pmcs[4];     /* bitmask of force reload PMC on ctxsw in */
304         unsigned long           ctx_used_monitors[4];   /* bitmask of monitor PMC being used */
305 
306         unsigned long           ctx_pmcs[PFM_NUM_PMC_REGS];     /*  saved copies of PMC values */
307 
308         unsigned int            ctx_used_ibrs[1];               /* bitmask of used IBR (speedup ctxsw in) */
309         unsigned int            ctx_used_dbrs[1];               /* bitmask of used DBR (speedup ctxsw in) */
310         unsigned long           ctx_dbrs[IA64_NUM_DBG_REGS];    /* DBR values (cache) when not loaded */
311         unsigned long           ctx_ibrs[IA64_NUM_DBG_REGS];    /* IBR values (cache) when not loaded */
312 
313         pfm_counter_t           ctx_pmds[PFM_NUM_PMD_REGS]; /* software state for PMDS */
314 
315         unsigned long           th_pmcs[PFM_NUM_PMC_REGS];      /* PMC thread save state */
316         unsigned long           th_pmds[PFM_NUM_PMD_REGS];      /* PMD thread save state */
317 
318         unsigned long           ctx_saved_psr_up;       /* only contains psr.up value */
319 
320         unsigned long           ctx_last_activation;    /* context last activation number for last_cpu */
321         unsigned int            ctx_last_cpu;           /* CPU id of current or last CPU used (SMP only) */
322         unsigned int            ctx_cpu;                /* cpu to which perfmon is applied (system wide) */
323 
324         int                     ctx_fd;                 /* file descriptor used my this context */
325         pfm_ovfl_arg_t          ctx_ovfl_arg;           /* argument to custom buffer format handler */
326 
327         pfm_buffer_fmt_t        *ctx_buf_fmt;           /* buffer format callbacks */
328         void                    *ctx_smpl_hdr;          /* points to sampling buffer header kernel vaddr */
329         unsigned long           ctx_smpl_size;          /* size of sampling buffer */
330         void                    *ctx_smpl_vaddr;        /* user level virtual address of smpl buffer */
331 
332         wait_queue_head_t       ctx_msgq_wait;
333         pfm_msg_t               ctx_msgq[PFM_MAX_MSGS];
334         int                     ctx_msgq_head;
335         int                     ctx_msgq_tail;
336         struct fasync_struct    *ctx_async_queue;
337 
338         wait_queue_head_t       ctx_zombieq;            /* termination cleanup wait queue */
339 } pfm_context_t;
340 
341 /*
342  * magic number used to verify that structure is really
343  * a perfmon context
344  */
345 #define PFM_IS_FILE(f)          ((f)->f_op == &pfm_file_ops)
346 
347 #define PFM_GET_CTX(t)          ((pfm_context_t *)(t)->thread.pfm_context)
348 
349 #ifdef CONFIG_SMP
350 #define SET_LAST_CPU(ctx, v)    (ctx)->ctx_last_cpu = (v)
351 #define GET_LAST_CPU(ctx)       (ctx)->ctx_last_cpu
352 #else
353 #define SET_LAST_CPU(ctx, v)    do {} while(0)
354 #define GET_LAST_CPU(ctx)       do {} while(0)
355 #endif
356 
357 
358 #define ctx_fl_block            ctx_flags.block
359 #define ctx_fl_system           ctx_flags.system
360 #define ctx_fl_using_dbreg      ctx_flags.using_dbreg
361 #define ctx_fl_is_sampling      ctx_flags.is_sampling
362 #define ctx_fl_excl_idle        ctx_flags.excl_idle
363 #define ctx_fl_going_zombie     ctx_flags.going_zombie
364 #define ctx_fl_trap_reason      ctx_flags.trap_reason
365 #define ctx_fl_no_msg           ctx_flags.no_msg
366 #define ctx_fl_can_restart      ctx_flags.can_restart
367 
368 #define PFM_SET_WORK_PENDING(t, v)      do { (t)->thread.pfm_needs_checking = v; } while(0);
369 #define PFM_GET_WORK_PENDING(t)         (t)->thread.pfm_needs_checking
370 
371 /*
372  * global information about all sessions
373  * mostly used to synchronize between system wide and per-process
374  */
375 typedef struct {
376         spinlock_t              pfs_lock;                  /* lock the structure */
377 
378         unsigned int            pfs_task_sessions;         /* number of per task sessions */
379         unsigned int            pfs_sys_sessions;          /* number of per system wide sessions */
380         unsigned int            pfs_sys_use_dbregs;        /* incremented when a system wide session uses debug regs */
381         unsigned int            pfs_ptrace_use_dbregs;     /* incremented when a process uses debug regs */
382         struct task_struct      *pfs_sys_session[NR_CPUS]; /* point to task owning a system-wide session */
383 } pfm_session_t;
384 
385 /*
386  * information about a PMC or PMD.
387  * dep_pmd[]: a bitmask of dependent PMD registers
388  * dep_pmc[]: a bitmask of dependent PMC registers
389  */
390 typedef int (*pfm_reg_check_t)(struct task_struct *task, pfm_context_t *ctx, unsigned int cnum, unsigned long *val, struct pt_regs *regs);
391 typedef struct {
392         unsigned int            type;
393         int                     pm_pos;
394         unsigned long           default_value;  /* power-on default value */
395         unsigned long           reserved_mask;  /* bitmask of reserved bits */
396         pfm_reg_check_t         read_check;
397         pfm_reg_check_t         write_check;
398         unsigned long           dep_pmd[4];
399         unsigned long           dep_pmc[4];
400 } pfm_reg_desc_t;
401 
402 /* assume cnum is a valid monitor */
403 #define PMC_PM(cnum, val)       (((val) >> (pmu_conf->pmc_desc[cnum].pm_pos)) & 0x1)
404 
405 /*
406  * This structure is initialized at boot time and contains
407  * a description of the PMU main characteristics.
408  *
409  * If the probe function is defined, detection is based
410  * on its return value: 
411  *      - 0 means recognized PMU
412  *      - anything else means not supported
413  * When the probe function is not defined, then the pmu_family field
414  * is used and it must match the host CPU family such that:
415  *      - cpu->family & config->pmu_family != 0
416  */
417 typedef struct {
418         unsigned long  ovfl_val;        /* overflow value for counters */
419 
420         pfm_reg_desc_t *pmc_desc;       /* detailed PMC register dependencies descriptions */
421         pfm_reg_desc_t *pmd_desc;       /* detailed PMD register dependencies descriptions */
422 
423         unsigned int   num_pmcs;        /* number of PMCS: computed at init time */
424         unsigned int   num_pmds;        /* number of PMDS: computed at init time */
425         unsigned long  impl_pmcs[4];    /* bitmask of implemented PMCS */
426         unsigned long  impl_pmds[4];    /* bitmask of implemented PMDS */
427 
428         char          *pmu_name;        /* PMU family name */
429         unsigned int  pmu_family;       /* cpuid family pattern used to identify pmu */
430         unsigned int  flags;            /* pmu specific flags */
431         unsigned int  num_ibrs;         /* number of IBRS: computed at init time */
432         unsigned int  num_dbrs;         /* number of DBRS: computed at init time */
433         unsigned int  num_counters;     /* PMC/PMD counting pairs : computed at init time */
434         int           (*probe)(void);   /* customized probe routine */
435         unsigned int  use_rr_dbregs:1;  /* set if debug registers used for range restriction */
436 } pmu_config_t;
437 /*
438  * PMU specific flags
439  */
440 #define PFM_PMU_IRQ_RESEND      1       /* PMU needs explicit IRQ resend */
441 
442 /*
443  * debug register related type definitions
444  */
445 typedef struct {
446         unsigned long ibr_mask:56;
447         unsigned long ibr_plm:4;
448         unsigned long ibr_ig:3;
449         unsigned long ibr_x:1;
450 } ibr_mask_reg_t;
451 
452 typedef struct {
453         unsigned long dbr_mask:56;
454         unsigned long dbr_plm:4;
455         unsigned long dbr_ig:2;
456         unsigned long dbr_w:1;
457         unsigned long dbr_r:1;
458 } dbr_mask_reg_t;
459 
460 typedef union {
461         unsigned long  val;
462         ibr_mask_reg_t ibr;
463         dbr_mask_reg_t dbr;
464 } dbreg_t;
465 
466 
467 /*
468  * perfmon command descriptions
469  */
470 typedef struct {
471         int             (*cmd_func)(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
472         char            *cmd_name;
473         int             cmd_flags;
474         unsigned int    cmd_narg;
475         size_t          cmd_argsize;
476         int             (*cmd_getsize)(void *arg, size_t *sz);
477 } pfm_cmd_desc_t;
478 
479 #define PFM_CMD_FD              0x01    /* command requires a file descriptor */
480 #define PFM_CMD_ARG_READ        0x02    /* command must read argument(s) */
481 #define PFM_CMD_ARG_RW          0x04    /* command must read/write argument(s) */
482 #define PFM_CMD_STOP            0x08    /* command does not work on zombie context */
483 
484 
485 #define PFM_CMD_NAME(cmd)       pfm_cmd_tab[(cmd)].cmd_name
486 #define PFM_CMD_READ_ARG(cmd)   (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_ARG_READ)
487 #define PFM_CMD_RW_ARG(cmd)     (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_ARG_RW)
488 #define PFM_CMD_USE_FD(cmd)     (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_FD)
489 #define PFM_CMD_STOPPED(cmd)    (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_STOP)
490 
491 #define PFM_CMD_ARG_MANY        -1 /* cannot be zero */
492 
493 typedef struct {
494         unsigned long pfm_spurious_ovfl_intr_count;     /* keep track of spurious ovfl interrupts */
495         unsigned long pfm_replay_ovfl_intr_count;       /* keep track of replayed ovfl interrupts */
496         unsigned long pfm_ovfl_intr_count;              /* keep track of ovfl interrupts */
497         unsigned long pfm_ovfl_intr_cycles;             /* cycles spent processing ovfl interrupts */
498         unsigned long pfm_ovfl_intr_cycles_min;         /* min cycles spent processing ovfl interrupts */
499         unsigned long pfm_ovfl_intr_cycles_max;         /* max cycles spent processing ovfl interrupts */
500         unsigned long pfm_smpl_handler_calls;
501         unsigned long pfm_smpl_handler_cycles;
502         char pad[SMP_CACHE_BYTES] ____cacheline_aligned;
503 } pfm_stats_t;
504 
505 /*
506  * perfmon internal variables
507  */
508 static pfm_stats_t              pfm_stats[NR_CPUS];
509 static pfm_session_t            pfm_sessions;   /* global sessions information */
510 
511 static DEFINE_SPINLOCK(pfm_alt_install_check);
512 static pfm_intr_handler_desc_t  *pfm_alt_intr_handler;
513 
514 static struct proc_dir_entry    *perfmon_dir;
515 static pfm_uuid_t               pfm_null_uuid = {0,};
516 
517 static spinlock_t               pfm_buffer_fmt_lock;
518 static LIST_HEAD(pfm_buffer_fmt_list);
519 
520 static pmu_config_t             *pmu_conf;
521 
522 /* sysctl() controls */
523 pfm_sysctl_t pfm_sysctl;
524 EXPORT_SYMBOL(pfm_sysctl);
525 
526 static struct ctl_table pfm_ctl_table[] = {
527         {
528                 .procname       = "debug",
529                 .data           = &pfm_sysctl.debug,
530                 .maxlen         = sizeof(int),
531                 .mode           = 0666,
532                 .proc_handler   = proc_dointvec,
533         },
534         {
535                 .procname       = "debug_ovfl",
536                 .data           = &pfm_sysctl.debug_ovfl,
537                 .maxlen         = sizeof(int),
538                 .mode           = 0666,
539                 .proc_handler   = proc_dointvec,
540         },
541         {
542                 .procname       = "fastctxsw",
543                 .data           = &pfm_sysctl.fastctxsw,
544                 .maxlen         = sizeof(int),
545                 .mode           = 0600,
546                 .proc_handler   = proc_dointvec,
547         },
548         {
549                 .procname       = "expert_mode",
550                 .data           = &pfm_sysctl.expert_mode,
551                 .maxlen         = sizeof(int),
552                 .mode           = 0600,
553                 .proc_handler   = proc_dointvec,
554         },
555         {}
556 };
557 static struct ctl_table pfm_sysctl_dir[] = {
558         {
559                 .procname       = "perfmon",
560                 .mode           = 0555,
561                 .child          = pfm_ctl_table,
562         },
563         {}
564 };
565 static struct ctl_table pfm_sysctl_root[] = {
566         {
567                 .procname       = "kernel",
568                 .mode           = 0555,
569                 .child          = pfm_sysctl_dir,
570         },
571         {}
572 };
573 static struct ctl_table_header *pfm_sysctl_header;
574 
575 static int pfm_context_unload(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
576 
577 #define pfm_get_cpu_var(v)              __ia64_per_cpu_var(v)
578 #define pfm_get_cpu_data(a,b)           per_cpu(a, b)
579 
580 static inline void
581 pfm_put_task(struct task_struct *task)
582 {
583         if (task != current) put_task_struct(task);
584 }
585 
586 static inline unsigned long
587 pfm_protect_ctx_ctxsw(pfm_context_t *x)
588 {
589         spin_lock(&(x)->ctx_lock);
590         return 0UL;
591 }
592 
593 static inline void
594 pfm_unprotect_ctx_ctxsw(pfm_context_t *x, unsigned long f)
595 {
596         spin_unlock(&(x)->ctx_lock);
597 }
598 
599 /* forward declaration */
600 static const struct dentry_operations pfmfs_dentry_operations;
601 
602 static struct dentry *
603 pfmfs_mount(struct file_system_type *fs_type, int flags, const char *dev_name, void *data)
604 {
605         return mount_pseudo(fs_type, "pfm:", NULL, &pfmfs_dentry_operations,
606                         PFMFS_MAGIC);
607 }
608 
609 static struct file_system_type pfm_fs_type = {
610         .name     = "pfmfs",
611         .mount    = pfmfs_mount,
612         .kill_sb  = kill_anon_super,
613 };
614 MODULE_ALIAS_FS("pfmfs");
615 
616 DEFINE_PER_CPU(unsigned long, pfm_syst_info);
617 DEFINE_PER_CPU(struct task_struct *, pmu_owner);
618 DEFINE_PER_CPU(pfm_context_t  *, pmu_ctx);
619 DEFINE_PER_CPU(unsigned long, pmu_activation_number);
620 EXPORT_PER_CPU_SYMBOL_GPL(pfm_syst_info);
621 
622 
623 /* forward declaration */
624 static const struct file_operations pfm_file_ops;
625 
626 /*
627  * forward declarations
628  */
629 #ifndef CONFIG_SMP
630 static void pfm_lazy_save_regs (struct task_struct *ta);
631 #endif
632 
633 void dump_pmu_state(const char *);
634 static int pfm_write_ibr_dbr(int mode, pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
635 
636 #include "perfmon_itanium.h"
637 #include "perfmon_mckinley.h"
638 #include "perfmon_montecito.h"
639 #include "perfmon_generic.h"
640 
641 static pmu_config_t *pmu_confs[]={
642         &pmu_conf_mont,
643         &pmu_conf_mck,
644         &pmu_conf_ita,
645         &pmu_conf_gen, /* must be last */
646         NULL
647 };
648 
649 
650 static int pfm_end_notify_user(pfm_context_t *ctx);
651 
652 static inline void
653 pfm_clear_psr_pp(void)
654 {
655         ia64_rsm(IA64_PSR_PP);
656         ia64_srlz_i();
657 }
658 
659 static inline void
660 pfm_set_psr_pp(void)
661 {
662         ia64_ssm(IA64_PSR_PP);
663         ia64_srlz_i();
664 }
665 
666 static inline void
667 pfm_clear_psr_up(void)
668 {
669         ia64_rsm(IA64_PSR_UP);
670         ia64_srlz_i();
671 }
672 
673 static inline void
674 pfm_set_psr_up(void)
675 {
676         ia64_ssm(IA64_PSR_UP);
677         ia64_srlz_i();
678 }
679 
680 static inline unsigned long
681 pfm_get_psr(void)
682 {
683         unsigned long tmp;
684         tmp = ia64_getreg(_IA64_REG_PSR);
685         ia64_srlz_i();
686         return tmp;
687 }
688 
689 static inline void
690 pfm_set_psr_l(unsigned long val)
691 {
692         ia64_setreg(_IA64_REG_PSR_L, val);
693         ia64_srlz_i();
694 }
695 
696 static inline void
697 pfm_freeze_pmu(void)
698 {
699         ia64_set_pmc(0,1UL);
700         ia64_srlz_d();
701 }
702 
703 static inline void
704 pfm_unfreeze_pmu(void)
705 {
706         ia64_set_pmc(0,0UL);
707         ia64_srlz_d();
708 }
709 
710 static inline void
711 pfm_restore_ibrs(unsigned long *ibrs, unsigned int nibrs)
712 {
713         int i;
714 
715         for (i=0; i < nibrs; i++) {
716                 ia64_set_ibr(i, ibrs[i]);
717                 ia64_dv_serialize_instruction();
718         }
719         ia64_srlz_i();
720 }
721 
722 static inline void
723 pfm_restore_dbrs(unsigned long *dbrs, unsigned int ndbrs)
724 {
725         int i;
726 
727         for (i=0; i < ndbrs; i++) {
728                 ia64_set_dbr(i, dbrs[i]);
729                 ia64_dv_serialize_data();
730         }
731         ia64_srlz_d();
732 }
733 
734 /*
735  * PMD[i] must be a counter. no check is made
736  */
737 static inline unsigned long
738 pfm_read_soft_counter(pfm_context_t *ctx, int i)
739 {
740         return ctx->ctx_pmds[i].val + (ia64_get_pmd(i) & pmu_conf->ovfl_val);
741 }
742 
743 /*
744  * PMD[i] must be a counter. no check is made
745  */
746 static inline void
747 pfm_write_soft_counter(pfm_context_t *ctx, int i, unsigned long val)
748 {
749         unsigned long ovfl_val = pmu_conf->ovfl_val;
750 
751         ctx->ctx_pmds[i].val = val  & ~ovfl_val;
752         /*
753          * writing to unimplemented part is ignore, so we do not need to
754          * mask off top part
755          */
756         ia64_set_pmd(i, val & ovfl_val);
757 }
758 
759 static pfm_msg_t *
760 pfm_get_new_msg(pfm_context_t *ctx)
761 {
762         int idx, next;
763 
764         next = (ctx->ctx_msgq_tail+1) % PFM_MAX_MSGS;
765 
766         DPRINT(("ctx_fd=%p head=%d tail=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail));
767         if (next == ctx->ctx_msgq_head) return NULL;
768 
769         idx =   ctx->ctx_msgq_tail;
770         ctx->ctx_msgq_tail = next;
771 
772         DPRINT(("ctx=%p head=%d tail=%d msg=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail, idx));
773 
774         return ctx->ctx_msgq+idx;
775 }
776 
777 static pfm_msg_t *
778 pfm_get_next_msg(pfm_context_t *ctx)
779 {
780         pfm_msg_t *msg;
781 
782         DPRINT(("ctx=%p head=%d tail=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail));
783 
784         if (PFM_CTXQ_EMPTY(ctx)) return NULL;
785 
786         /*
787          * get oldest message
788          */
789         msg = ctx->ctx_msgq+ctx->ctx_msgq_head;
790 
791         /*
792          * and move forward
793          */
794         ctx->ctx_msgq_head = (ctx->ctx_msgq_head+1) % PFM_MAX_MSGS;
795 
796         DPRINT(("ctx=%p head=%d tail=%d type=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail, msg->pfm_gen_msg.msg_type));
797 
798         return msg;
799 }
800 
801 static void
802 pfm_reset_msgq(pfm_context_t *ctx)
803 {
804         ctx->ctx_msgq_head = ctx->ctx_msgq_tail = 0;
805         DPRINT(("ctx=%p msgq reset\n", ctx));
806 }
807 
808 static pfm_context_t *
809 pfm_context_alloc(int ctx_flags)
810 {
811         pfm_context_t *ctx;
812 
813         /* 
814          * allocate context descriptor 
815          * must be able to free with interrupts disabled
816          */
817         ctx = kzalloc(sizeof(pfm_context_t), GFP_KERNEL);
818         if (ctx) {
819                 DPRINT(("alloc ctx @%p\n", ctx));
820 
821                 /*
822                  * init context protection lock
823                  */
824                 spin_lock_init(&ctx->ctx_lock);
825 
826                 /*
827                  * context is unloaded
828                  */
829                 ctx->ctx_state = PFM_CTX_UNLOADED;
830 
831                 /*
832                  * initialization of context's flags
833                  */
834                 ctx->ctx_fl_block       = (ctx_flags & PFM_FL_NOTIFY_BLOCK) ? 1 : 0;
835                 ctx->ctx_fl_system      = (ctx_flags & PFM_FL_SYSTEM_WIDE) ? 1: 0;
836                 ctx->ctx_fl_no_msg      = (ctx_flags & PFM_FL_OVFL_NO_MSG) ? 1: 0;
837                 /*
838                  * will move to set properties
839                  * ctx->ctx_fl_excl_idle   = (ctx_flags & PFM_FL_EXCL_IDLE) ? 1: 0;
840                  */
841 
842                 /*
843                  * init restart semaphore to locked
844                  */
845                 init_completion(&ctx->ctx_restart_done);
846 
847                 /*
848                  * activation is used in SMP only
849                  */
850                 ctx->ctx_last_activation = PFM_INVALID_ACTIVATION;
851                 SET_LAST_CPU(ctx, -1);
852 
853                 /*
854                  * initialize notification message queue
855                  */
856                 ctx->ctx_msgq_head = ctx->ctx_msgq_tail = 0;
857                 init_waitqueue_head(&ctx->ctx_msgq_wait);
858                 init_waitqueue_head(&ctx->ctx_zombieq);
859 
860         }
861         return ctx;
862 }
863 
864 static void
865 pfm_context_free(pfm_context_t *ctx)
866 {
867         if (ctx) {
868                 DPRINT(("free ctx @%p\n", ctx));
869                 kfree(ctx);
870         }
871 }
872 
873 static void
874 pfm_mask_monitoring(struct task_struct *task)
875 {
876         pfm_context_t *ctx = PFM_GET_CTX(task);
877         unsigned long mask, val, ovfl_mask;
878         int i;
879 
880         DPRINT_ovfl(("masking monitoring for [%d]\n", task_pid_nr(task)));
881 
882         ovfl_mask = pmu_conf->ovfl_val;
883         /*
884          * monitoring can only be masked as a result of a valid
885          * counter overflow. In UP, it means that the PMU still
886          * has an owner. Note that the owner can be different
887          * from the current task. However the PMU state belongs
888          * to the owner.
889          * In SMP, a valid overflow only happens when task is
890          * current. Therefore if we come here, we know that
891          * the PMU state belongs to the current task, therefore
892          * we can access the live registers.
893          *
894          * So in both cases, the live register contains the owner's
895          * state. We can ONLY touch the PMU registers and NOT the PSR.
896          *
897          * As a consequence to this call, the ctx->th_pmds[] array
898          * contains stale information which must be ignored
899          * when context is reloaded AND monitoring is active (see
900          * pfm_restart).
901          */
902         mask = ctx->ctx_used_pmds[0];
903         for (i = 0; mask; i++, mask>>=1) {
904                 /* skip non used pmds */
905                 if ((mask & 0x1) == 0) continue;
906                 val = ia64_get_pmd(i);
907 
908                 if (PMD_IS_COUNTING(i)) {
909                         /*
910                          * we rebuild the full 64 bit value of the counter
911                          */
912                         ctx->ctx_pmds[i].val += (val & ovfl_mask);
913                 } else {
914                         ctx->ctx_pmds[i].val = val;
915                 }
916                 DPRINT_ovfl(("pmd[%d]=0x%lx hw_pmd=0x%lx\n",
917                         i,
918                         ctx->ctx_pmds[i].val,
919                         val & ovfl_mask));
920         }
921         /*
922          * mask monitoring by setting the privilege level to 0
923          * we cannot use psr.pp/psr.up for this, it is controlled by
924          * the user
925          *
926          * if task is current, modify actual registers, otherwise modify
927          * thread save state, i.e., what will be restored in pfm_load_regs()
928          */
929         mask = ctx->ctx_used_monitors[0] >> PMU_FIRST_COUNTER;
930         for(i= PMU_FIRST_COUNTER; mask; i++, mask>>=1) {
931                 if ((mask & 0x1) == 0UL) continue;
932                 ia64_set_pmc(i, ctx->th_pmcs[i] & ~0xfUL);
933                 ctx->th_pmcs[i] &= ~0xfUL;
934                 DPRINT_ovfl(("pmc[%d]=0x%lx\n", i, ctx->th_pmcs[i]));
935         }
936         /*
937          * make all of this visible
938          */
939         ia64_srlz_d();
940 }
941 
942 /*
943  * must always be done with task == current
944  *
945  * context must be in MASKED state when calling
946  */
947 static void
948 pfm_restore_monitoring(struct task_struct *task)
949 {
950         pfm_context_t *ctx = PFM_GET_CTX(task);
951         unsigned long mask, ovfl_mask;
952         unsigned long psr, val;
953         int i, is_system;
954 
955         is_system = ctx->ctx_fl_system;
956         ovfl_mask = pmu_conf->ovfl_val;
957 
958         if (task != current) {
959                 printk(KERN_ERR "perfmon.%d: invalid task[%d] current[%d]\n", __LINE__, task_pid_nr(task), task_pid_nr(current));
960                 return;
961         }
962         if (ctx->ctx_state != PFM_CTX_MASKED) {
963                 printk(KERN_ERR "perfmon.%d: task[%d] current[%d] invalid state=%d\n", __LINE__,
964                         task_pid_nr(task), task_pid_nr(current), ctx->ctx_state);
965                 return;
966         }
967         psr = pfm_get_psr();
968         /*
969          * monitoring is masked via the PMC.
970          * As we restore their value, we do not want each counter to
971          * restart right away. We stop monitoring using the PSR,
972          * restore the PMC (and PMD) and then re-establish the psr
973          * as it was. Note that there can be no pending overflow at
974          * this point, because monitoring was MASKED.
975          *
976          * system-wide session are pinned and self-monitoring
977          */
978         if (is_system && (PFM_CPUINFO_GET() & PFM_CPUINFO_DCR_PP)) {
979                 /* disable dcr pp */
980                 ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) & ~IA64_DCR_PP);
981                 pfm_clear_psr_pp();
982         } else {
983                 pfm_clear_psr_up();
984         }
985         /*
986          * first, we restore the PMD
987          */
988         mask = ctx->ctx_used_pmds[0];
989         for (i = 0; mask; i++, mask>>=1) {
990                 /* skip non used pmds */
991                 if ((mask & 0x1) == 0) continue;
992 
993                 if (PMD_IS_COUNTING(i)) {
994                         /*
995                          * we split the 64bit value according to
996                          * counter width
997                          */
998                         val = ctx->ctx_pmds[i].val & ovfl_mask;
999                         ctx->ctx_pmds[i].val &= ~ovfl_mask;
1000                 } else {
1001                         val = ctx->ctx_pmds[i].val;
1002                 }
1003                 ia64_set_pmd(i, val);
1004 
1005                 DPRINT(("pmd[%d]=0x%lx hw_pmd=0x%lx\n",
1006                         i,
1007                         ctx->ctx_pmds[i].val,
1008                         val));
1009         }
1010         /*
1011          * restore the PMCs
1012          */
1013         mask = ctx->ctx_used_monitors[0] >> PMU_FIRST_COUNTER;
1014         for(i= PMU_FIRST_COUNTER; mask; i++, mask>>=1) {
1015                 if ((mask & 0x1) == 0UL) continue;
1016                 ctx->th_pmcs[i] = ctx->ctx_pmcs[i];
1017                 ia64_set_pmc(i, ctx->th_pmcs[i]);
1018                 DPRINT(("[%d] pmc[%d]=0x%lx\n",
1019                                         task_pid_nr(task), i, ctx->th_pmcs[i]));
1020         }
1021         ia64_srlz_d();
1022 
1023         /*
1024          * must restore DBR/IBR because could be modified while masked
1025          * XXX: need to optimize 
1026          */
1027         if (ctx->ctx_fl_using_dbreg) {
1028                 pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
1029                 pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
1030         }
1031 
1032         /*
1033          * now restore PSR
1034          */
1035         if (is_system && (PFM_CPUINFO_GET() & PFM_CPUINFO_DCR_PP)) {
1036                 /* enable dcr pp */
1037                 ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) | IA64_DCR_PP);
1038                 ia64_srlz_i();
1039         }
1040         pfm_set_psr_l(psr);
1041 }
1042 
1043 static inline void
1044 pfm_save_pmds(unsigned long *pmds, unsigned long mask)
1045 {
1046         int i;
1047 
1048         ia64_srlz_d();
1049 
1050         for (i=0; mask; i++, mask>>=1) {
1051                 if (mask & 0x1) pmds[i] = ia64_get_pmd(i);
1052         }
1053 }
1054 
1055 /*
1056  * reload from thread state (used for ctxw only)
1057  */
1058 static inline void
1059 pfm_restore_pmds(unsigned long *pmds, unsigned long mask)
1060 {
1061         int i;
1062         unsigned long val, ovfl_val = pmu_conf->ovfl_val;
1063 
1064         for (i=0; mask; i++, mask>>=1) {
1065                 if ((mask & 0x1) == 0) continue;
1066                 val = PMD_IS_COUNTING(i) ? pmds[i] & ovfl_val : pmds[i];
1067                 ia64_set_pmd(i, val);
1068         }
1069         ia64_srlz_d();
1070 }
1071 
1072 /*
1073  * propagate PMD from context to thread-state
1074  */
1075 static inline void
1076 pfm_copy_pmds(struct task_struct *task, pfm_context_t *ctx)
1077 {
1078         unsigned long ovfl_val = pmu_conf->ovfl_val;
1079         unsigned long mask = ctx->ctx_all_pmds[0];
1080         unsigned long val;
1081         int i;
1082 
1083         DPRINT(("mask=0x%lx\n", mask));
1084 
1085         for (i=0; mask; i++, mask>>=1) {
1086 
1087                 val = ctx->ctx_pmds[i].val;
1088 
1089                 /*
1090                  * We break up the 64 bit value into 2 pieces
1091                  * the lower bits go to the machine state in the
1092                  * thread (will be reloaded on ctxsw in).
1093                  * The upper part stays in the soft-counter.
1094                  */
1095                 if (PMD_IS_COUNTING(i)) {
1096                         ctx->ctx_pmds[i].val = val & ~ovfl_val;
1097                          val &= ovfl_val;
1098                 }
1099                 ctx->th_pmds[i] = val;
1100 
1101                 DPRINT(("pmd[%d]=0x%lx soft_val=0x%lx\n",
1102                         i,
1103                         ctx->th_pmds[i],
1104                         ctx->ctx_pmds[i].val));
1105         }
1106 }
1107 
1108 /*
1109  * propagate PMC from context to thread-state
1110  */
1111 static inline void
1112 pfm_copy_pmcs(struct task_struct *task, pfm_context_t *ctx)
1113 {
1114         unsigned long mask = ctx->ctx_all_pmcs[0];
1115         int i;
1116 
1117         DPRINT(("mask=0x%lx\n", mask));
1118 
1119         for (i=0; mask; i++, mask>>=1) {
1120                 /* masking 0 with ovfl_val yields 0 */
1121                 ctx->th_pmcs[i] = ctx->ctx_pmcs[i];
1122                 DPRINT(("pmc[%d]=0x%lx\n", i, ctx->th_pmcs[i]));
1123         }
1124 }
1125 
1126 
1127 
1128 static inline void
1129 pfm_restore_pmcs(unsigned long *pmcs, unsigned long mask)
1130 {
1131         int i;
1132 
1133         for (i=0; mask; i++, mask>>=1) {
1134                 if ((mask & 0x1) == 0) continue;
1135                 ia64_set_pmc(i, pmcs[i]);
1136         }
1137         ia64_srlz_d();
1138 }
1139 
1140 static inline int
1141 pfm_uuid_cmp(pfm_uuid_t a, pfm_uuid_t b)
1142 {
1143         return memcmp(a, b, sizeof(pfm_uuid_t));
1144 }
1145 
1146 static inline int
1147 pfm_buf_fmt_exit(pfm_buffer_fmt_t *fmt, struct task_struct *task, void *buf, struct pt_regs *regs)
1148 {
1149         int ret = 0;
1150         if (fmt->fmt_exit) ret = (*fmt->fmt_exit)(task, buf, regs);
1151         return ret;
1152 }
1153 
1154 static inline int
1155 pfm_buf_fmt_getsize(pfm_buffer_fmt_t *fmt, struct task_struct *task, unsigned int flags, int cpu, void *arg, unsigned long *size)
1156 {
1157         int ret = 0;
1158         if (fmt->fmt_getsize) ret = (*fmt->fmt_getsize)(task, flags, cpu, arg, size);
1159         return ret;
1160 }
1161 
1162 
1163 static inline int
1164 pfm_buf_fmt_validate(pfm_buffer_fmt_t *fmt, struct task_struct *task, unsigned int flags,
1165                      int cpu, void *arg)
1166 {
1167         int ret = 0;
1168         if (fmt->fmt_validate) ret = (*fmt->fmt_validate)(task, flags, cpu, arg);
1169         return ret;
1170 }
1171 
1172 static inline int
1173 pfm_buf_fmt_init(pfm_buffer_fmt_t *fmt, struct task_struct *task, void *buf, unsigned int flags,
1174                      int cpu, void *arg)
1175 {
1176         int ret = 0;
1177         if (fmt->fmt_init) ret = (*fmt->fmt_init)(task, buf, flags, cpu, arg);
1178         return ret;
1179 }
1180 
1181 static inline int
1182 pfm_buf_fmt_restart(pfm_buffer_fmt_t *fmt, struct task_struct *task, pfm_ovfl_ctrl_t *ctrl, void *buf, struct pt_regs *regs)
1183 {
1184         int ret = 0;
1185         if (fmt->fmt_restart) ret = (*fmt->fmt_restart)(task, ctrl, buf, regs);
1186         return ret;
1187 }
1188 
1189 static inline int
1190 pfm_buf_fmt_restart_active(pfm_buffer_fmt_t *fmt, struct task_struct *task, pfm_ovfl_ctrl_t *ctrl, void *buf, struct pt_regs *regs)
1191 {
1192         int ret = 0;
1193         if (fmt->fmt_restart_active) ret = (*fmt->fmt_restart_active)(task, ctrl, buf, regs);
1194         return ret;
1195 }
1196 
1197 static pfm_buffer_fmt_t *
1198 __pfm_find_buffer_fmt(pfm_uuid_t uuid)
1199 {
1200         struct list_head * pos;
1201         pfm_buffer_fmt_t * entry;
1202 
1203         list_for_each(pos, &pfm_buffer_fmt_list) {
1204                 entry = list_entry(pos, pfm_buffer_fmt_t, fmt_list);
1205                 if (pfm_uuid_cmp(uuid, entry->fmt_uuid) == 0)
1206                         return entry;
1207         }
1208         return NULL;
1209 }
1210  
1211 /*
1212  * find a buffer format based on its uuid
1213  */
1214 static pfm_buffer_fmt_t *
1215 pfm_find_buffer_fmt(pfm_uuid_t uuid)
1216 {
1217         pfm_buffer_fmt_t * fmt;
1218         spin_lock(&pfm_buffer_fmt_lock);
1219         fmt = __pfm_find_buffer_fmt(uuid);
1220         spin_unlock(&pfm_buffer_fmt_lock);
1221         return fmt;
1222 }
1223  
1224 int
1225 pfm_register_buffer_fmt(pfm_buffer_fmt_t *fmt)
1226 {
1227         int ret = 0;
1228 
1229         /* some sanity checks */
1230         if (fmt == NULL || fmt->fmt_name == NULL) return -EINVAL;
1231 
1232         /* we need at least a handler */
1233         if (fmt->fmt_handler == NULL) return -EINVAL;
1234 
1235         /*
1236          * XXX: need check validity of fmt_arg_size
1237          */
1238 
1239         spin_lock(&pfm_buffer_fmt_lock);
1240 
1241         if (__pfm_find_buffer_fmt(fmt->fmt_uuid)) {
1242                 printk(KERN_ERR "perfmon: duplicate sampling format: %s\n", fmt->fmt_name);
1243                 ret = -EBUSY;
1244                 goto out;
1245         } 
1246         list_add(&fmt->fmt_list, &pfm_buffer_fmt_list);
1247         printk(KERN_INFO "perfmon: added sampling format %s\n", fmt->fmt_name);
1248 
1249 out:
1250         spin_unlock(&pfm_buffer_fmt_lock);
1251         return ret;
1252 }
1253 EXPORT_SYMBOL(pfm_register_buffer_fmt);
1254 
1255 int
1256 pfm_unregister_buffer_fmt(pfm_uuid_t uuid)
1257 {
1258         pfm_buffer_fmt_t *fmt;
1259         int ret = 0;
1260 
1261         spin_lock(&pfm_buffer_fmt_lock);
1262 
1263         fmt = __pfm_find_buffer_fmt(uuid);
1264         if (!fmt) {
1265                 printk(KERN_ERR "perfmon: cannot unregister format, not found\n");
1266                 ret = -EINVAL;
1267                 goto out;
1268         }
1269         list_del_init(&fmt->fmt_list);
1270         printk(KERN_INFO "perfmon: removed sampling format: %s\n", fmt->fmt_name);
1271 
1272 out:
1273         spin_unlock(&pfm_buffer_fmt_lock);
1274         return ret;
1275 
1276 }
1277 EXPORT_SYMBOL(pfm_unregister_buffer_fmt);
1278 
1279 static int
1280 pfm_reserve_session(struct task_struct *task, int is_syswide, unsigned int cpu)
1281 {
1282         unsigned long flags;
1283         /*
1284          * validity checks on cpu_mask have been done upstream
1285          */
1286         LOCK_PFS(flags);
1287 
1288         DPRINT(("in sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
1289                 pfm_sessions.pfs_sys_sessions,
1290                 pfm_sessions.pfs_task_sessions,
1291                 pfm_sessions.pfs_sys_use_dbregs,
1292                 is_syswide,
1293                 cpu));
1294 
1295         if (is_syswide) {
1296                 /*
1297                  * cannot mix system wide and per-task sessions
1298                  */
1299                 if (pfm_sessions.pfs_task_sessions > 0UL) {
1300                         DPRINT(("system wide not possible, %u conflicting task_sessions\n",
1301                                 pfm_sessions.pfs_task_sessions));
1302                         goto abort;
1303                 }
1304 
1305                 if (pfm_sessions.pfs_sys_session[cpu]) goto error_conflict;
1306 
1307                 DPRINT(("reserving system wide session on CPU%u currently on CPU%u\n", cpu, smp_processor_id()));
1308 
1309                 pfm_sessions.pfs_sys_session[cpu] = task;
1310 
1311                 pfm_sessions.pfs_sys_sessions++ ;
1312 
1313         } else {
1314                 if (pfm_sessions.pfs_sys_sessions) goto abort;
1315                 pfm_sessions.pfs_task_sessions++;
1316         }
1317 
1318         DPRINT(("out sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
1319                 pfm_sessions.pfs_sys_sessions,
1320                 pfm_sessions.pfs_task_sessions,
1321                 pfm_sessions.pfs_sys_use_dbregs,
1322                 is_syswide,
1323                 cpu));
1324 
1325         /*
1326          * Force idle() into poll mode
1327          */
1328         cpu_idle_poll_ctrl(true);
1329 
1330         UNLOCK_PFS(flags);
1331 
1332         return 0;
1333 
1334 error_conflict:
1335         DPRINT(("system wide not possible, conflicting session [%d] on CPU%d\n",
1336                 task_pid_nr(pfm_sessions.pfs_sys_session[cpu]),
1337                 cpu));
1338 abort:
1339         UNLOCK_PFS(flags);
1340 
1341         return -EBUSY;
1342 
1343 }
1344 
1345 static int
1346 pfm_unreserve_session(pfm_context_t *ctx, int is_syswide, unsigned int cpu)
1347 {
1348         unsigned long flags;
1349         /*
1350          * validity checks on cpu_mask have been done upstream
1351          */
1352         LOCK_PFS(flags);
1353 
1354         DPRINT(("in sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
1355                 pfm_sessions.pfs_sys_sessions,
1356                 pfm_sessions.pfs_task_sessions,
1357                 pfm_sessions.pfs_sys_use_dbregs,
1358                 is_syswide,
1359                 cpu));
1360 
1361 
1362         if (is_syswide) {
1363                 pfm_sessions.pfs_sys_session[cpu] = NULL;
1364                 /*
1365                  * would not work with perfmon+more than one bit in cpu_mask
1366                  */
1367                 if (ctx && ctx->ctx_fl_using_dbreg) {
1368                         if (pfm_sessions.pfs_sys_use_dbregs == 0) {
1369                                 printk(KERN_ERR "perfmon: invalid release for ctx %p sys_use_dbregs=0\n", ctx);
1370                         } else {
1371                                 pfm_sessions.pfs_sys_use_dbregs--;
1372                         }
1373                 }
1374                 pfm_sessions.pfs_sys_sessions--;
1375         } else {
1376                 pfm_sessions.pfs_task_sessions--;
1377         }
1378         DPRINT(("out sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
1379                 pfm_sessions.pfs_sys_sessions,
1380                 pfm_sessions.pfs_task_sessions,
1381                 pfm_sessions.pfs_sys_use_dbregs,
1382                 is_syswide,
1383                 cpu));
1384 
1385         /* Undo forced polling. Last session reenables pal_halt */
1386         cpu_idle_poll_ctrl(false);
1387 
1388         UNLOCK_PFS(flags);
1389 
1390         return 0;
1391 }
1392 
1393 /*
1394  * removes virtual mapping of the sampling buffer.
1395  * IMPORTANT: cannot be called with interrupts disable, e.g. inside
1396  * a PROTECT_CTX() section.
1397  */
1398 static int
1399 pfm_remove_smpl_mapping(void *vaddr, unsigned long size)
1400 {
1401         struct task_struct *task = current;
1402         int r;
1403 
1404         /* sanity checks */
1405         if (task->mm == NULL || size == 0UL || vaddr == NULL) {
1406                 printk(KERN_ERR "perfmon: pfm_remove_smpl_mapping [%d] invalid context mm=%p\n", task_pid_nr(task), task->mm);
1407                 return -EINVAL;
1408         }
1409 
1410         DPRINT(("smpl_vaddr=%p size=%lu\n", vaddr, size));
1411 
1412         /*
1413          * does the actual unmapping
1414          */
1415         r = vm_munmap((unsigned long)vaddr, size);
1416 
1417         if (r !=0) {
1418                 printk(KERN_ERR "perfmon: [%d] unable to unmap sampling buffer @%p size=%lu\n", task_pid_nr(task), vaddr, size);
1419         }
1420 
1421         DPRINT(("do_unmap(%p, %lu)=%d\n", vaddr, size, r));
1422 
1423         return 0;
1424 }
1425 
1426 /*
1427  * free actual physical storage used by sampling buffer
1428  */
1429 #if 0
1430 static int
1431 pfm_free_smpl_buffer(pfm_context_t *ctx)
1432 {
1433         pfm_buffer_fmt_t *fmt;
1434 
1435         if (ctx->ctx_smpl_hdr == NULL) goto invalid_free;
1436 
1437         /*
1438          * we won't use the buffer format anymore
1439          */
1440         fmt = ctx->ctx_buf_fmt;
1441 
1442         DPRINT(("sampling buffer @%p size %lu vaddr=%p\n",
1443                 ctx->ctx_smpl_hdr,
1444                 ctx->ctx_smpl_size,
1445                 ctx->ctx_smpl_vaddr));
1446 
1447         pfm_buf_fmt_exit(fmt, current, NULL, NULL);
1448 
1449         /*
1450          * free the buffer
1451          */
1452         vfree(ctx->ctx_smpl_hdr);
1453 
1454         ctx->ctx_smpl_hdr  = NULL;
1455         ctx->ctx_smpl_size = 0UL;
1456 
1457         return 0;
1458 
1459 invalid_free:
1460         printk(KERN_ERR "perfmon: pfm_free_smpl_buffer [%d] no buffer\n", task_pid_nr(current));
1461         return -EINVAL;
1462 }
1463 #endif
1464 
1465 static inline void
1466 pfm_exit_smpl_buffer(pfm_buffer_fmt_t *fmt)
1467 {
1468         if (fmt == NULL) return;
1469 
1470         pfm_buf_fmt_exit(fmt, current, NULL, NULL);
1471 
1472 }
1473 
1474 /*
1475  * pfmfs should _never_ be mounted by userland - too much of security hassle,
1476  * no real gain from having the whole whorehouse mounted. So we don't need
1477  * any operations on the root directory. However, we need a non-trivial
1478  * d_name - pfm: will go nicely and kill the special-casing in procfs.
1479  */
1480 static struct vfsmount *pfmfs_mnt __read_mostly;
1481 
1482 static int __init
1483 init_pfm_fs(void)
1484 {
1485         int err = register_filesystem(&pfm_fs_type);
1486         if (!err) {
1487                 pfmfs_mnt = kern_mount(&pfm_fs_type);
1488                 err = PTR_ERR(pfmfs_mnt);
1489                 if (IS_ERR(pfmfs_mnt))
1490                         unregister_filesystem(&pfm_fs_type);
1491                 else
1492                         err = 0;
1493         }
1494         return err;
1495 }
1496 
1497 static ssize_t
1498 pfm_read(struct file *filp, char __user *buf, size_t size, loff_t *ppos)
1499 {
1500         pfm_context_t *ctx;
1501         pfm_msg_t *msg;
1502         ssize_t ret;
1503         unsigned long flags;
1504         DECLARE_WAITQUEUE(wait, current);
1505         if (PFM_IS_FILE(filp) == 0) {
1506                 printk(KERN_ERR "perfmon: pfm_poll: bad magic [%d]\n", task_pid_nr(current));
1507                 return -EINVAL;
1508         }
1509 
1510         ctx = filp->private_data;
1511         if (ctx == NULL) {
1512                 printk(KERN_ERR "perfmon: pfm_read: NULL ctx [%d]\n", task_pid_nr(current));
1513                 return -EINVAL;
1514         }
1515 
1516         /*
1517          * check even when there is no message
1518          */
1519         if (size < sizeof(pfm_msg_t)) {
1520                 DPRINT(("message is too small ctx=%p (>=%ld)\n", ctx, sizeof(pfm_msg_t)));
1521                 return -EINVAL;
1522         }
1523 
1524         PROTECT_CTX(ctx, flags);
1525 
1526         /*
1527          * put ourselves on the wait queue
1528          */
1529         add_wait_queue(&ctx->ctx_msgq_wait, &wait);
1530 
1531 
1532         for(;;) {
1533                 /*
1534                  * check wait queue
1535                  */
1536 
1537                 set_current_state(TASK_INTERRUPTIBLE);
1538 
1539                 DPRINT(("head=%d tail=%d\n", ctx->ctx_msgq_head, ctx->ctx_msgq_tail));
1540 
1541                 ret = 0;
1542                 if(PFM_CTXQ_EMPTY(ctx) == 0) break;
1543 
1544                 UNPROTECT_CTX(ctx, flags);
1545 
1546                 /*
1547                  * check non-blocking read
1548                  */
1549                 ret = -EAGAIN;
1550                 if(filp->f_flags & O_NONBLOCK) break;
1551 
1552                 /*
1553                  * check pending signals
1554                  */
1555                 if(signal_pending(current)) {
1556                         ret = -EINTR;
1557                         break;
1558                 }
1559                 /*
1560                  * no message, so wait
1561                  */
1562                 schedule();
1563 
1564                 PROTECT_CTX(ctx, flags);
1565         }
1566         DPRINT(("[%d] back to running ret=%ld\n", task_pid_nr(current), ret));
1567         set_current_state(TASK_RUNNING);
1568         remove_wait_queue(&ctx->ctx_msgq_wait, &wait);
1569 
1570         if (ret < 0) goto abort;
1571 
1572         ret = -EINVAL;
1573         msg = pfm_get_next_msg(ctx);
1574         if (msg == NULL) {
1575                 printk(KERN_ERR "perfmon: pfm_read no msg for ctx=%p [%d]\n", ctx, task_pid_nr(current));
1576                 goto abort_locked;
1577         }
1578 
1579         DPRINT(("fd=%d type=%d\n", msg->pfm_gen_msg.msg_ctx_fd, msg->pfm_gen_msg.msg_type));
1580 
1581         ret = -EFAULT;
1582         if(copy_to_user(buf, msg, sizeof(pfm_msg_t)) == 0) ret = sizeof(pfm_msg_t);
1583 
1584 abort_locked:
1585         UNPROTECT_CTX(ctx, flags);
1586 abort:
1587         return ret;
1588 }
1589 
1590 static ssize_t
1591 pfm_write(struct file *file, const char __user *ubuf,
1592                           size_t size, loff_t *ppos)
1593 {
1594         DPRINT(("pfm_write called\n"));
1595         return -EINVAL;
1596 }
1597 
1598 static __poll_t
1599 pfm_poll(struct file *filp, poll_table * wait)
1600 {
1601         pfm_context_t *ctx;
1602         unsigned long flags;
1603         __poll_t mask = 0;
1604 
1605         if (PFM_IS_FILE(filp) == 0) {
1606                 printk(KERN_ERR "perfmon: pfm_poll: bad magic [%d]\n", task_pid_nr(current));
1607                 return 0;
1608         }
1609 
1610         ctx = filp->private_data;
1611         if (ctx == NULL) {
1612                 printk(KERN_ERR "perfmon: pfm_poll: NULL ctx [%d]\n", task_pid_nr(current));
1613                 return 0;
1614         }
1615 
1616 
1617         DPRINT(("pfm_poll ctx_fd=%d before poll_wait\n", ctx->ctx_fd));
1618 
1619         poll_wait(filp, &ctx->ctx_msgq_wait, wait);
1620 
1621         PROTECT_CTX(ctx, flags);
1622 
1623         if (PFM_CTXQ_EMPTY(ctx) == 0)
1624                 mask =  EPOLLIN | EPOLLRDNORM;
1625 
1626         UNPROTECT_CTX(ctx, flags);
1627 
1628         DPRINT(("pfm_poll ctx_fd=%d mask=0x%x\n", ctx->ctx_fd, mask));
1629 
1630         return mask;
1631 }
1632 
1633 static long
1634 pfm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1635 {
1636         DPRINT(("pfm_ioctl called\n"));
1637         return -EINVAL;
1638 }
1639 
1640 /*
1641  * interrupt cannot be masked when coming here
1642  */
1643 static inline int
1644 pfm_do_fasync(int fd, struct file *filp, pfm_context_t *ctx, int on)
1645 {
1646         int ret;
1647 
1648         ret = fasync_helper (fd, filp, on, &ctx->ctx_async_queue);
1649 
1650         DPRINT(("pfm_fasync called by [%d] on ctx_fd=%d on=%d async_queue=%p ret=%d\n",
1651                 task_pid_nr(current),
1652                 fd,
1653                 on,
1654                 ctx->ctx_async_queue, ret));
1655 
1656         return ret;
1657 }
1658 
1659 static int
1660 pfm_fasync(int fd, struct file *filp, int on)
1661 {
1662         pfm_context_t *ctx;
1663         int ret;
1664 
1665         if (PFM_IS_FILE(filp) == 0) {
1666                 printk(KERN_ERR "perfmon: pfm_fasync bad magic [%d]\n", task_pid_nr(current));
1667                 return -EBADF;
1668         }
1669 
1670         ctx = filp->private_data;
1671         if (ctx == NULL) {
1672                 printk(KERN_ERR "perfmon: pfm_fasync NULL ctx [%d]\n", task_pid_nr(current));
1673                 return -EBADF;
1674         }
1675         /*
1676          * we cannot mask interrupts during this call because this may
1677          * may go to sleep if memory is not readily avalaible.
1678          *
1679          * We are protected from the conetxt disappearing by the get_fd()/put_fd()
1680          * done in caller. Serialization of this function is ensured by caller.
1681          */
1682         ret = pfm_do_fasync(fd, filp, ctx, on);
1683 
1684 
1685         DPRINT(("pfm_fasync called on ctx_fd=%d on=%d async_queue=%p ret=%d\n",
1686                 fd,
1687                 on,
1688                 ctx->ctx_async_queue, ret));
1689 
1690         return ret;
1691 }
1692 
1693 #ifdef CONFIG_SMP
1694 /*
1695  * this function is exclusively called from pfm_close().
1696  * The context is not protected at that time, nor are interrupts
1697  * on the remote CPU. That's necessary to avoid deadlocks.
1698  */
1699 static void
1700 pfm_syswide_force_stop(void *info)
1701 {
1702         pfm_context_t   *ctx = (pfm_context_t *)info;
1703         struct pt_regs *regs = task_pt_regs(current);
1704         struct task_struct *owner;
1705         unsigned long flags;
1706         int ret;
1707 
1708         if (ctx->ctx_cpu != smp_processor_id()) {
1709                 printk(KERN_ERR "perfmon: pfm_syswide_force_stop for CPU%d  but on CPU%d\n",
1710                         ctx->ctx_cpu,
1711                         smp_processor_id());
1712                 return;
1713         }
1714         owner = GET_PMU_OWNER();
1715         if (owner != ctx->ctx_task) {
1716                 printk(KERN_ERR "perfmon: pfm_syswide_force_stop CPU%d unexpected owner [%d] instead of [%d]\n",
1717                         smp_processor_id(),
1718                         task_pid_nr(owner), task_pid_nr(ctx->ctx_task));
1719                 return;
1720         }
1721         if (GET_PMU_CTX() != ctx) {
1722                 printk(KERN_ERR "perfmon: pfm_syswide_force_stop CPU%d unexpected ctx %p instead of %p\n",
1723                         smp_processor_id(),
1724                         GET_PMU_CTX(), ctx);
1725                 return;
1726         }
1727 
1728         DPRINT(("on CPU%d forcing system wide stop for [%d]\n", smp_processor_id(), task_pid_nr(ctx->ctx_task)));
1729         /*
1730          * the context is already protected in pfm_close(), we simply
1731          * need to mask interrupts to avoid a PMU interrupt race on
1732          * this CPU
1733          */
1734         local_irq_save(flags);
1735 
1736         ret = pfm_context_unload(ctx, NULL, 0, regs);
1737         if (ret) {
1738                 DPRINT(("context_unload returned %d\n", ret));
1739         }
1740 
1741         /*
1742          * unmask interrupts, PMU interrupts are now spurious here
1743          */
1744         local_irq_restore(flags);
1745 }
1746 
1747 static void
1748 pfm_syswide_cleanup_other_cpu(pfm_context_t *ctx)
1749 {
1750         int ret;
1751 
1752         DPRINT(("calling CPU%d for cleanup\n", ctx->ctx_cpu));
1753         ret = smp_call_function_single(ctx->ctx_cpu, pfm_syswide_force_stop, ctx, 1);
1754         DPRINT(("called CPU%d for cleanup ret=%d\n", ctx->ctx_cpu, ret));
1755 }
1756 #endif /* CONFIG_SMP */
1757 
1758 /*
1759  * called for each close(). Partially free resources.
1760  * When caller is self-monitoring, the context is unloaded.
1761  */
1762 static int
1763 pfm_flush(struct file *filp, fl_owner_t id)
1764 {
1765         pfm_context_t *ctx;
1766         struct task_struct *task;
1767         struct pt_regs *regs;
1768         unsigned long flags;
1769         unsigned long smpl_buf_size = 0UL;
1770         void *smpl_buf_vaddr = NULL;
1771         int state, is_system;
1772 
1773         if (PFM_IS_FILE(filp) == 0) {
1774                 DPRINT(("bad magic for\n"));
1775                 return -EBADF;
1776         }
1777 
1778         ctx = filp->private_data;
1779         if (ctx == NULL) {
1780                 printk(KERN_ERR "perfmon: pfm_flush: NULL ctx [%d]\n", task_pid_nr(current));
1781                 return -EBADF;
1782         }
1783 
1784         /*
1785          * remove our file from the async queue, if we use this mode.
1786          * This can be done without the context being protected. We come
1787          * here when the context has become unreachable by other tasks.
1788          *
1789          * We may still have active monitoring at this point and we may
1790          * end up in pfm_overflow_handler(). However, fasync_helper()
1791          * operates with interrupts disabled and it cleans up the
1792          * queue. If the PMU handler is called prior to entering
1793          * fasync_helper() then it will send a signal. If it is
1794          * invoked after, it will find an empty queue and no
1795          * signal will be sent. In both case, we are safe
1796          */
1797         PROTECT_CTX(ctx, flags);
1798 
1799         state     = ctx->ctx_state;
1800         is_system = ctx->ctx_fl_system;
1801 
1802         task = PFM_CTX_TASK(ctx);
1803         regs = task_pt_regs(task);
1804 
1805         DPRINT(("ctx_state=%d is_current=%d\n",
1806                 state,
1807                 task == current ? 1 : 0));
1808 
1809         /*
1810          * if state == UNLOADED, then task is NULL
1811          */
1812 
1813         /*
1814          * we must stop and unload because we are losing access to the context.
1815          */
1816         if (task == current) {
1817 #ifdef CONFIG_SMP
1818                 /*
1819                  * the task IS the owner but it migrated to another CPU: that's bad
1820                  * but we must handle this cleanly. Unfortunately, the kernel does
1821                  * not provide a mechanism to block migration (while the context is loaded).
1822                  *
1823                  * We need to release the resource on the ORIGINAL cpu.
1824                  */
1825                 if (is_system && ctx->ctx_cpu != smp_processor_id()) {
1826 
1827                         DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
1828                         /*
1829                          * keep context protected but unmask interrupt for IPI
1830                          */
1831                         local_irq_restore(flags);
1832 
1833                         pfm_syswide_cleanup_other_cpu(ctx);
1834 
1835                         /*
1836                          * restore interrupt masking
1837                          */
1838                         local_irq_save(flags);
1839 
1840                         /*
1841                          * context is unloaded at this point
1842                          */
1843                 } else
1844 #endif /* CONFIG_SMP */
1845                 {
1846 
1847                         DPRINT(("forcing unload\n"));
1848                         /*
1849                         * stop and unload, returning with state UNLOADED
1850                         * and session unreserved.
1851                         */
1852                         pfm_context_unload(ctx, NULL, 0, regs);
1853 
1854                         DPRINT(("ctx_state=%d\n", ctx->ctx_state));
1855                 }
1856         }
1857 
1858         /*
1859          * remove virtual mapping, if any, for the calling task.
1860          * cannot reset ctx field until last user is calling close().
1861          *
1862          * ctx_smpl_vaddr must never be cleared because it is needed
1863          * by every task with access to the context
1864          *
1865          * When called from do_exit(), the mm context is gone already, therefore
1866          * mm is NULL, i.e., the VMA is already gone  and we do not have to
1867          * do anything here
1868          */
1869         if (ctx->ctx_smpl_vaddr && current->mm) {
1870                 smpl_buf_vaddr = ctx->ctx_smpl_vaddr;
1871                 smpl_buf_size  = ctx->ctx_smpl_size;
1872         }
1873 
1874         UNPROTECT_CTX(ctx, flags);
1875 
1876         /*
1877          * if there was a mapping, then we systematically remove it
1878          * at this point. Cannot be done inside critical section
1879          * because some VM function reenables interrupts.
1880          *
1881          */
1882         if (smpl_buf_vaddr) pfm_remove_smpl_mapping(smpl_buf_vaddr, smpl_buf_size);
1883 
1884         return 0;
1885 }
1886 /*
1887  * called either on explicit close() or from exit_files(). 
1888  * Only the LAST user of the file gets to this point, i.e., it is
1889  * called only ONCE.
1890  *
1891  * IMPORTANT: we get called ONLY when the refcnt on the file gets to zero 
1892  * (fput()),i.e, last task to access the file. Nobody else can access the 
1893  * file at this point.
1894  *
1895  * When called from exit_files(), the VMA has been freed because exit_mm()
1896  * is executed before exit_files().
1897  *
1898  * When called from exit_files(), the current task is not yet ZOMBIE but we
1899  * flush the PMU state to the context. 
1900  */
1901 static int
1902 pfm_close(struct inode *inode, struct file *filp)
1903 {
1904         pfm_context_t *ctx;
1905         struct task_struct *task;
1906         struct pt_regs *regs;
1907         DECLARE_WAITQUEUE(wait, current);
1908         unsigned long flags;
1909         unsigned long smpl_buf_size = 0UL;
1910         void *smpl_buf_addr = NULL;
1911         int free_possible = 1;
1912         int state, is_system;
1913 
1914         DPRINT(("pfm_close called private=%p\n", filp->private_data));
1915 
1916         if (PFM_IS_FILE(filp) == 0) {
1917                 DPRINT(("bad magic\n"));
1918                 return -EBADF;
1919         }
1920         
1921         ctx = filp->private_data;
1922         if (ctx == NULL) {
1923                 printk(KERN_ERR "perfmon: pfm_close: NULL ctx [%d]\n", task_pid_nr(current));
1924                 return -EBADF;
1925         }
1926 
1927         PROTECT_CTX(ctx, flags);
1928 
1929         state     = ctx->ctx_state;
1930         is_system = ctx->ctx_fl_system;
1931 
1932         task = PFM_CTX_TASK(ctx);
1933         regs = task_pt_regs(task);
1934 
1935         DPRINT(("ctx_state=%d is_current=%d\n", 
1936                 state,
1937                 task == current ? 1 : 0));
1938 
1939         /*
1940          * if task == current, then pfm_flush() unloaded the context
1941          */
1942         if (state == PFM_CTX_UNLOADED) goto doit;
1943 
1944         /*
1945          * context is loaded/masked and task != current, we need to
1946          * either force an unload or go zombie
1947          */
1948 
1949         /*
1950          * The task is currently blocked or will block after an overflow.
1951          * we must force it to wakeup to get out of the
1952          * MASKED state and transition to the unloaded state by itself.
1953          *
1954          * This situation is only possible for per-task mode
1955          */
1956         if (state == PFM_CTX_MASKED && CTX_OVFL_NOBLOCK(ctx) == 0) {
1957 
1958                 /*
1959                  * set a "partial" zombie state to be checked
1960                  * upon return from down() in pfm_handle_work().
1961                  *
1962                  * We cannot use the ZOMBIE state, because it is checked
1963                  * by pfm_load_regs() which is called upon wakeup from down().
1964                  * In such case, it would free the context and then we would
1965                  * return to pfm_handle_work() which would access the
1966                  * stale context. Instead, we set a flag invisible to pfm_load_regs()
1967                  * but visible to pfm_handle_work().
1968                  *
1969                  * For some window of time, we have a zombie context with
1970                  * ctx_state = MASKED  and not ZOMBIE
1971                  */
1972                 ctx->ctx_fl_going_zombie = 1;
1973 
1974                 /*
1975                  * force task to wake up from MASKED state
1976                  */
1977                 complete(&ctx->ctx_restart_done);
1978 
1979                 DPRINT(("waking up ctx_state=%d\n", state));
1980 
1981                 /*
1982                  * put ourself to sleep waiting for the other
1983                  * task to report completion
1984                  *
1985                  * the context is protected by mutex, therefore there
1986                  * is no risk of being notified of completion before
1987                  * begin actually on the waitq.
1988                  */
1989                 set_current_state(TASK_INTERRUPTIBLE);
1990                 add_wait_queue(&ctx->ctx_zombieq, &wait);
1991 
1992                 UNPROTECT_CTX(ctx, flags);
1993 
1994                 /*
1995                  * XXX: check for signals :
1996                  *      - ok for explicit close
1997                  *      - not ok when coming from exit_files()
1998                  */
1999                 schedule();
2000 
2001 
2002                 PROTECT_CTX(ctx, flags);
2003 
2004 
2005                 remove_wait_queue(&ctx->ctx_zombieq, &wait);
2006                 set_current_state(TASK_RUNNING);
2007 
2008                 /*
2009                  * context is unloaded at this point
2010                  */
2011                 DPRINT(("after zombie wakeup ctx_state=%d for\n", state));
2012         }
2013         else if (task != current) {
2014 #ifdef CONFIG_SMP
2015                 /*
2016                  * switch context to zombie state
2017                  */
2018                 ctx->ctx_state = PFM_CTX_ZOMBIE;
2019 
2020                 DPRINT(("zombie ctx for [%d]\n", task_pid_nr(task)));
2021                 /*
2022                  * cannot free the context on the spot. deferred until
2023                  * the task notices the ZOMBIE state
2024                  */
2025                 free_possible = 0;
2026 #else
2027                 pfm_context_unload(ctx, NULL, 0, regs);
2028 #endif
2029         }
2030 
2031 doit:
2032         /* reload state, may have changed during  opening of critical section */
2033         state = ctx->ctx_state;
2034 
2035         /*
2036          * the context is still attached to a task (possibly current)
2037          * we cannot destroy it right now
2038          */
2039 
2040         /*
2041          * we must free the sampling buffer right here because
2042          * we cannot rely on it being cleaned up later by the
2043          * monitored task. It is not possible to free vmalloc'ed
2044          * memory in pfm_load_regs(). Instead, we remove the buffer
2045          * now. should there be subsequent PMU overflow originally
2046          * meant for sampling, the will be converted to spurious
2047          * and that's fine because the monitoring tools is gone anyway.
2048          */
2049         if (ctx->ctx_smpl_hdr) {
2050                 smpl_buf_addr = ctx->ctx_smpl_hdr;
2051                 smpl_buf_size = ctx->ctx_smpl_size;
2052                 /* no more sampling */
2053                 ctx->ctx_smpl_hdr = NULL;
2054                 ctx->ctx_fl_is_sampling = 0;
2055         }
2056 
2057         DPRINT(("ctx_state=%d free_possible=%d addr=%p size=%lu\n",
2058                 state,
2059                 free_possible,
2060                 smpl_buf_addr,
2061                 smpl_buf_size));
2062 
2063         if (smpl_buf_addr) pfm_exit_smpl_buffer(ctx->ctx_buf_fmt);
2064 
2065         /*
2066          * UNLOADED that the session has already been unreserved.
2067          */
2068         if (state == PFM_CTX_ZOMBIE) {
2069                 pfm_unreserve_session(ctx, ctx->ctx_fl_system , ctx->ctx_cpu);
2070         }
2071 
2072         /*
2073          * disconnect file descriptor from context must be done
2074          * before we unlock.
2075          */
2076         filp->private_data = NULL;
2077 
2078         /*
2079          * if we free on the spot, the context is now completely unreachable
2080          * from the callers side. The monitored task side is also cut, so we
2081          * can freely cut.
2082          *
2083          * If we have a deferred free, only the caller side is disconnected.
2084          */
2085         UNPROTECT_CTX(ctx, flags);
2086 
2087         /*
2088          * All memory free operations (especially for vmalloc'ed memory)
2089          * MUST be done with interrupts ENABLED.
2090          */
2091         vfree(smpl_buf_addr);
2092 
2093         /*
2094          * return the memory used by the context
2095          */
2096         if (free_possible) pfm_context_free(ctx);
2097 
2098         return 0;
2099 }
2100 
2101 static const struct file_operations pfm_file_ops = {
2102         .llseek         = no_llseek,
2103         .read           = pfm_read,
2104         .write          = pfm_write,
2105         .poll           = pfm_poll,
2106         .unlocked_ioctl = pfm_ioctl,
2107         .fasync         = pfm_fasync,
2108         .release        = pfm_close,
2109         .flush          = pfm_flush
2110 };
2111 
2112 static char *pfmfs_dname(struct dentry *dentry, char *buffer, int buflen)
2113 {
2114         return dynamic_dname(dentry, buffer, buflen, "pfm:[%lu]",
2115                              d_inode(dentry)->i_ino);
2116 }
2117 
2118 static const struct dentry_operations pfmfs_dentry_operations = {
2119         .d_delete = always_delete_dentry,
2120         .d_dname = pfmfs_dname,
2121 };
2122 
2123 
2124 static struct file *
2125 pfm_alloc_file(pfm_context_t *ctx)
2126 {
2127         struct file *file;
2128         struct inode *inode;
2129         struct path path;
2130         struct qstr this = { .name = "" };
2131 
2132         /*
2133          * allocate a new inode
2134          */
2135         inode = new_inode(pfmfs_mnt->mnt_sb);
2136         if (!inode)
2137                 return ERR_PTR(-ENOMEM);
2138 
2139         DPRINT(("new inode ino=%ld @%p\n", inode->i_ino, inode));
2140 
2141         inode->i_mode = S_IFCHR|S_IRUGO;
2142         inode->i_uid  = current_fsuid();
2143         inode->i_gid  = current_fsgid();
2144 
2145         /*
2146          * allocate a new dcache entry
2147          */
2148         path.dentry = d_alloc(pfmfs_mnt->mnt_root, &this);
2149         if (!path.dentry) {
2150                 iput(inode);
2151                 return ERR_PTR(-ENOMEM);
2152         }
2153         path.mnt = mntget(pfmfs_mnt);
2154 
2155         d_add(path.dentry, inode);
2156 
2157         file = alloc_file(&path, FMODE_READ, &pfm_file_ops);
2158         if (IS_ERR(file)) {
2159                 path_put(&path);
2160                 return file;
2161         }
2162 
2163         file->f_flags = O_RDONLY;
2164         file->private_data = ctx;
2165 
2166         return file;
2167 }
2168 
2169 static int
2170 pfm_remap_buffer(struct vm_area_struct *vma, unsigned long buf, unsigned long addr, unsigned long size)
2171 {
2172         DPRINT(("CPU%d buf=0x%lx addr=0x%lx size=%ld\n", smp_processor_id(), buf, addr, size));
2173 
2174         while (size > 0) {
2175                 unsigned long pfn = ia64_tpa(buf) >> PAGE_SHIFT;
2176 
2177 
2178                 if (remap_pfn_range(vma, addr, pfn, PAGE_SIZE, PAGE_READONLY))
2179                         return -ENOMEM;
2180 
2181                 addr  += PAGE_SIZE;
2182                 buf   += PAGE_SIZE;
2183                 size  -= PAGE_SIZE;
2184         }
2185         return 0;
2186 }
2187 
2188 /*
2189  * allocate a sampling buffer and remaps it into the user address space of the task
2190  */
2191 static int
2192 pfm_smpl_buffer_alloc(struct task_struct *task, struct file *filp, pfm_context_t *ctx, unsigned long rsize, void **user_vaddr)
2193 {
2194         struct mm_struct *mm = task->mm;
2195         struct vm_area_struct *vma = NULL;
2196         unsigned long size;
2197         void *smpl_buf;
2198 
2199 
2200         /*
2201          * the fixed header + requested size and align to page boundary
2202          */
2203         size = PAGE_ALIGN(rsize);
2204 
2205         DPRINT(("sampling buffer rsize=%lu size=%lu bytes\n", rsize, size));
2206 
2207         /*
2208          * check requested size to avoid Denial-of-service attacks
2209          * XXX: may have to refine this test
2210          * Check against address space limit.
2211          *
2212          * if ((mm->total_vm << PAGE_SHIFT) + len> task->rlim[RLIMIT_AS].rlim_cur)
2213          *      return -ENOMEM;
2214          */
2215         if (size > task_rlimit(task, RLIMIT_MEMLOCK))
2216                 return -ENOMEM;
2217 
2218         /*
2219          * We do the easy to undo allocations first.
2220          */
2221         smpl_buf = vzalloc(size);
2222         if (smpl_buf == NULL) {
2223                 DPRINT(("Can't allocate sampling buffer\n"));
2224                 return -ENOMEM;
2225         }
2226 
2227         DPRINT(("smpl_buf @%p\n", smpl_buf));
2228 
2229         /* allocate vma */
2230         vma = vm_area_alloc(mm);
2231         if (!vma) {
2232                 DPRINT(("Cannot allocate vma\n"));
2233                 goto error_kmem;
2234         }
2235 
2236         /*
2237          * partially initialize the vma for the sampling buffer
2238          */
2239         vma->vm_file         = get_file(filp);
2240         vma->vm_flags        = VM_READ|VM_MAYREAD|VM_DONTEXPAND|VM_DONTDUMP;
2241         vma->vm_page_prot    = PAGE_READONLY; /* XXX may need to change */
2242 
2243         /*
2244          * Now we have everything we need and we can initialize
2245          * and connect all the data structures
2246          */
2247 
2248         ctx->ctx_smpl_hdr   = smpl_buf;
2249         ctx->ctx_smpl_size  = size; /* aligned size */
2250 
2251         /*
2252          * Let's do the difficult operations next.
2253          *
2254          * now we atomically find some area in the address space and
2255          * remap the buffer in it.
2256          */
2257         down_write(&task->mm->mmap_sem);
2258 
2259         /* find some free area in address space, must have mmap sem held */
2260         vma->vm_start = get_unmapped_area(NULL, 0, size, 0, MAP_PRIVATE|MAP_ANONYMOUS);
2261         if (IS_ERR_VALUE(vma->vm_start)) {
2262                 DPRINT(("Cannot find unmapped area for size %ld\n", size));
2263                 up_write(&task->mm->mmap_sem);
2264                 goto error;
2265         }
2266         vma->vm_end = vma->vm_start + size;
2267         vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
2268 
2269         DPRINT(("aligned size=%ld, hdr=%p mapped @0x%lx\n", size, ctx->ctx_smpl_hdr, vma->vm_start));
2270 
2271         /* can only be applied to current task, need to have the mm semaphore held when called */
2272         if (pfm_remap_buffer(vma, (unsigned long)smpl_buf, vma->vm_start, size)) {
2273                 DPRINT(("Can't remap buffer\n"));
2274                 up_write(&task->mm->mmap_sem);
2275                 goto error;
2276         }
2277 
2278         /*
2279          * now insert the vma in the vm list for the process, must be
2280          * done with mmap lock held
2281          */
2282         insert_vm_struct(mm, vma);
2283 
2284         vm_stat_account(vma->vm_mm, vma->vm_flags, vma_pages(vma));
2285         up_write(&task->mm->mmap_sem);
2286 
2287         /*
2288          * keep track of user level virtual address
2289          */
2290         ctx->ctx_smpl_vaddr = (void *)vma->vm_start;
2291         *(unsigned long *)user_vaddr = vma->vm_start;
2292 
2293         return 0;
2294 
2295 error:
2296         vm_area_free(vma);
2297 error_kmem:
2298         vfree(smpl_buf);
2299 
2300         return -ENOMEM;
2301 }
2302 
2303 /*
2304  * XXX: do something better here
2305  */
2306 static int
2307 pfm_bad_permissions(struct task_struct *task)
2308 {
2309         const struct cred *tcred;
2310         kuid_t uid = current_uid();
2311         kgid_t gid = current_gid();
2312         int ret;
2313 
2314         rcu_read_lock();
2315         tcred = __task_cred(task);
2316 
2317         /* inspired by ptrace_attach() */
2318         DPRINT(("cur: uid=%d gid=%d task: euid=%d suid=%d uid=%d egid=%d sgid=%d\n",
2319                 from_kuid(&init_user_ns, uid),
2320                 from_kgid(&init_user_ns, gid),
2321                 from_kuid(&init_user_ns, tcred->euid),
2322                 from_kuid(&init_user_ns, tcred->suid),
2323                 from_kuid(&init_user_ns, tcred->uid),
2324                 from_kgid(&init_user_ns, tcred->egid),
2325                 from_kgid(&init_user_ns, tcred->sgid)));
2326 
2327         ret = ((!uid_eq(uid, tcred->euid))
2328                || (!uid_eq(uid, tcred->suid))
2329                || (!uid_eq(uid, tcred->uid))
2330                || (!gid_eq(gid, tcred->egid))
2331                || (!gid_eq(gid, tcred->sgid))
2332                || (!gid_eq(gid, tcred->gid))) && !capable(CAP_SYS_PTRACE);
2333 
2334         rcu_read_unlock();
2335         return ret;
2336 }
2337 
2338 static int
2339 pfarg_is_sane(struct task_struct *task, pfarg_context_t *pfx)
2340 {
2341         int ctx_flags;
2342 
2343         /* valid signal */
2344 
2345         ctx_flags = pfx->ctx_flags;
2346 
2347         if (ctx_flags & PFM_FL_SYSTEM_WIDE) {
2348 
2349                 /*
2350                  * cannot block in this mode
2351                  */
2352                 if (ctx_flags & PFM_FL_NOTIFY_BLOCK) {
2353                         DPRINT(("cannot use blocking mode when in system wide monitoring\n"));
2354                         return -EINVAL;
2355                 }
2356         } else {
2357         }
2358         /* probably more to add here */
2359 
2360         return 0;
2361 }
2362 
2363 static int
2364 pfm_setup_buffer_fmt(struct task_struct *task, struct file *filp, pfm_context_t *ctx, unsigned int ctx_flags,
2365                      unsigned int cpu, pfarg_context_t *arg)
2366 {
2367         pfm_buffer_fmt_t *fmt = NULL;
2368         unsigned long size = 0UL;
2369         void *uaddr = NULL;
2370         void *fmt_arg = NULL;
2371         int ret = 0;
2372 #define PFM_CTXARG_BUF_ARG(a)   (pfm_buffer_fmt_t *)(a+1)
2373 
2374         /* invoke and lock buffer format, if found */
2375         fmt = pfm_find_buffer_fmt(arg->ctx_smpl_buf_id);
2376         if (fmt == NULL) {
2377                 DPRINT(("[%d] cannot find buffer format\n", task_pid_nr(task)));
2378                 return -EINVAL;
2379         }
2380 
2381         /*
2382          * buffer argument MUST be contiguous to pfarg_context_t
2383          */
2384         if (fmt->fmt_arg_size) fmt_arg = PFM_CTXARG_BUF_ARG(arg);
2385 
2386         ret = pfm_buf_fmt_validate(fmt, task, ctx_flags, cpu, fmt_arg);
2387 
2388         DPRINT(("[%d] after validate(0x%x,%d,%p)=%d\n", task_pid_nr(task), ctx_flags, cpu, fmt_arg, ret));
2389 
2390         if (ret) goto error;
2391 
2392         /* link buffer format and context */
2393         ctx->ctx_buf_fmt = fmt;
2394         ctx->ctx_fl_is_sampling = 1; /* assume record() is defined */
2395 
2396         /*
2397          * check if buffer format wants to use perfmon buffer allocation/mapping service
2398          */
2399         ret = pfm_buf_fmt_getsize(fmt, task, ctx_flags, cpu, fmt_arg, &size);
2400         if (ret) goto error;
2401 
2402         if (size) {
2403                 /*
2404                  * buffer is always remapped into the caller's address space
2405                  */
2406                 ret = pfm_smpl_buffer_alloc(current, filp, ctx, size, &uaddr);
2407                 if (ret) goto error;
2408 
2409                 /* keep track of user address of buffer */
2410                 arg->ctx_smpl_vaddr = uaddr;
2411         }
2412         ret = pfm_buf_fmt_init(fmt, task, ctx->ctx_smpl_hdr, ctx_flags, cpu, fmt_arg);
2413 
2414 error:
2415         return ret;
2416 }
2417 
2418 static void
2419 pfm_reset_pmu_state(pfm_context_t *ctx)
2420 {
2421         int i;
2422 
2423         /*
2424          * install reset values for PMC.
2425          */
2426         for (i=1; PMC_IS_LAST(i) == 0; i++) {
2427                 if (PMC_IS_IMPL(i) == 0) continue;
2428                 ctx->ctx_pmcs[i] = PMC_DFL_VAL(i);
2429                 DPRINT(("pmc[%d]=0x%lx\n", i, ctx->ctx_pmcs[i]));
2430         }
2431         /*
2432          * PMD registers are set to 0UL when the context in memset()
2433          */
2434 
2435         /*
2436          * On context switched restore, we must restore ALL pmc and ALL pmd even
2437          * when they are not actively used by the task. In UP, the incoming process
2438          * may otherwise pick up left over PMC, PMD state from the previous process.
2439          * As opposed to PMD, stale PMC can cause harm to the incoming
2440          * process because they may change what is being measured.
2441          * Therefore, we must systematically reinstall the entire
2442          * PMC state. In SMP, the same thing is possible on the
2443          * same CPU but also on between 2 CPUs.
2444          *
2445          * The problem with PMD is information leaking especially
2446          * to user level when psr.sp=0
2447          *
2448          * There is unfortunately no easy way to avoid this problem
2449          * on either UP or SMP. This definitively slows down the
2450          * pfm_load_regs() function.
2451          */
2452 
2453          /*
2454           * bitmask of all PMCs accessible to this context
2455           *
2456           * PMC0 is treated differently.
2457           */
2458         ctx->ctx_all_pmcs[0] = pmu_conf->impl_pmcs[0] & ~0x1;
2459 
2460         /*
2461          * bitmask of all PMDs that are accessible to this context
2462          */
2463         ctx->ctx_all_pmds[0] = pmu_conf->impl_pmds[0];
2464 
2465         DPRINT(("<%d> all_pmcs=0x%lx all_pmds=0x%lx\n", ctx->ctx_fd, ctx->ctx_all_pmcs[0],ctx->ctx_all_pmds[0]));
2466 
2467         /*
2468          * useful in case of re-enable after disable
2469          */
2470         ctx->ctx_used_ibrs[0] = 0UL;
2471         ctx->ctx_used_dbrs[0] = 0UL;
2472 }
2473 
2474 static int
2475 pfm_ctx_getsize(void *arg, size_t *sz)
2476 {
2477         pfarg_context_t *req = (pfarg_context_t *)arg;
2478         pfm_buffer_fmt_t *fmt;
2479 
2480         *sz = 0;
2481 
2482         if (!pfm_uuid_cmp(req->ctx_smpl_buf_id, pfm_null_uuid)) return 0;
2483 
2484         fmt = pfm_find_buffer_fmt(req->ctx_smpl_buf_id);
2485         if (fmt == NULL) {
2486                 DPRINT(("cannot find buffer format\n"));
2487                 return -EINVAL;
2488         }
2489         /* get just enough to copy in user parameters */
2490         *sz = fmt->fmt_arg_size;
2491         DPRINT(("arg_size=%lu\n", *sz));
2492 
2493         return 0;
2494 }
2495 
2496 
2497 
2498 /*
2499  * cannot attach if :
2500  *      - kernel task
2501  *      - task not owned by caller
2502  *      - task incompatible with context mode
2503  */
2504 static int
2505 pfm_task_incompatible(pfm_context_t *ctx, struct task_struct *task)
2506 {
2507         /*
2508          * no kernel task or task not owner by caller
2509          */
2510         if (task->mm == NULL) {
2511                 DPRINT(("task [%d] has not memory context (kernel thread)\n", task_pid_nr(task)));
2512                 return -EPERM;
2513         }
2514         if (pfm_bad_permissions(task)) {
2515                 DPRINT(("no permission to attach to  [%d]\n", task_pid_nr(task)));
2516                 return -EPERM;
2517         }
2518         /*
2519          * cannot block in self-monitoring mode
2520          */
2521         if (CTX_OVFL_NOBLOCK(ctx) == 0 && task == current) {
2522                 DPRINT(("cannot load a blocking context on self for [%d]\n", task_pid_nr(task)));
2523                 return -EINVAL;
2524         }
2525 
2526         if (task->exit_state == EXIT_ZOMBIE) {
2527                 DPRINT(("cannot attach to  zombie task [%d]\n", task_pid_nr(task)));
2528                 return -EBUSY;
2529         }
2530 
2531         /*
2532          * always ok for self
2533          */
2534         if (task == current) return 0;
2535 
2536         if (!task_is_stopped_or_traced(task)) {
2537                 DPRINT(("cannot attach to non-stopped task [%d] state=%ld\n", task_pid_nr(task), task->state));
2538                 return -EBUSY;
2539         }
2540         /*
2541          * make sure the task is off any CPU
2542          */
2543         wait_task_inactive(task, 0);
2544 
2545         /* more to come... */
2546 
2547         return 0;
2548 }
2549 
2550 static int
2551 pfm_get_task(pfm_context_t *ctx, pid_t pid, struct task_struct **task)
2552 {
2553         struct task_struct *p = current;
2554         int ret;
2555 
2556         /* XXX: need to add more checks here */
2557         if (pid < 2) return -EPERM;
2558 
2559         if (pid != task_pid_vnr(current)) {
2560                 /* make sure task cannot go away while we operate on it */
2561                 p = find_get_task_by_vpid(pid);
2562                 if (!p)
2563                         return -ESRCH;
2564         }
2565 
2566         ret = pfm_task_incompatible(ctx, p);
2567         if (ret == 0) {
2568                 *task = p;
2569         } else if (p != current) {
2570                 pfm_put_task(p);
2571         }
2572         return ret;
2573 }
2574 
2575 
2576 
2577 static int
2578 pfm_context_create(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
2579 {
2580         pfarg_context_t *req = (pfarg_context_t *)arg;
2581         struct file *filp;
2582         struct path path;
2583         int ctx_flags;
2584         int fd;
2585         int ret;
2586 
2587         /* let's check the arguments first */
2588         ret = pfarg_is_sane(current, req);
2589         if (ret < 0)
2590                 return ret;
2591 
2592         ctx_flags = req->ctx_flags;
2593 
2594         ret = -ENOMEM;
2595 
2596         fd = get_unused_fd_flags(0);
2597         if (fd < 0)
2598                 return fd;
2599 
2600         ctx = pfm_context_alloc(ctx_flags);
2601         if (!ctx)
2602                 goto error;
2603 
2604         filp = pfm_alloc_file(ctx);
2605         if (IS_ERR(filp)) {
2606                 ret = PTR_ERR(filp);
2607                 goto error_file;
2608         }
2609 
2610         req->ctx_fd = ctx->ctx_fd = fd;
2611 
2612         /*
2613          * does the user want to sample?
2614          */
2615         if (pfm_uuid_cmp(req->ctx_smpl_buf_id, pfm_null_uuid)) {
2616                 ret = pfm_setup_buffer_fmt(current, filp, ctx, ctx_flags, 0, req);
2617                 if (ret)
2618                         goto buffer_error;
2619         }
2620 
2621         DPRINT(("ctx=%p flags=0x%x system=%d notify_block=%d excl_idle=%d no_msg=%d ctx_fd=%d\n",
2622                 ctx,
2623                 ctx_flags,
2624                 ctx->ctx_fl_system,
2625                 ctx->ctx_fl_block,
2626                 ctx->ctx_fl_excl_idle,
2627                 ctx->ctx_fl_no_msg,
2628                 ctx->ctx_fd));
2629 
2630         /*
2631          * initialize soft PMU state
2632          */
2633         pfm_reset_pmu_state(ctx);
2634 
2635         fd_install(fd, filp);
2636 
2637         return 0;
2638 
2639 buffer_error:
2640         path = filp->f_path;
2641         put_filp(filp);
2642         path_put(&path);
2643 
2644         if (ctx->ctx_buf_fmt) {
2645                 pfm_buf_fmt_exit(ctx->ctx_buf_fmt, current, NULL, regs);
2646         }
2647 error_file:
2648         pfm_context_free(ctx);
2649 
2650 error:
2651         put_unused_fd(fd);
2652         return ret;
2653 }
2654 
2655 static inline unsigned long
2656 pfm_new_counter_value (pfm_counter_t *reg, int is_long_reset)
2657 {
2658         unsigned long val = is_long_reset ? reg->long_reset : reg->short_reset;
2659         unsigned long new_seed, old_seed = reg->seed, mask = reg->mask;
2660         extern unsigned long carta_random32 (unsigned long seed);
2661 
2662         if (reg->flags & PFM_REGFL_RANDOM) {
2663                 new_seed = carta_random32(old_seed);
2664                 val -= (old_seed & mask);       /* counter values are negative numbers! */
2665                 if ((mask >> 32) != 0)
2666                         /* construct a full 64-bit random value: */
2667                         new_seed |= carta_random32(old_seed >> 32) << 32;
2668                 reg->seed = new_seed;
2669         }
2670         reg->lval = val;
2671         return val;
2672 }
2673 
2674 static void
2675 pfm_reset_regs_masked(pfm_context_t *ctx, unsigned long *ovfl_regs, int is_long_reset)
2676 {
2677         unsigned long mask = ovfl_regs[0];
2678         unsigned long reset_others = 0UL;
2679         unsigned long val;
2680         int i;
2681 
2682         /*
2683          * now restore reset value on sampling overflowed counters
2684          */
2685         mask >>= PMU_FIRST_COUNTER;
2686         for(i = PMU_FIRST_COUNTER; mask; i++, mask >>= 1) {
2687 
2688                 if ((mask & 0x1UL) == 0UL) continue;
2689 
2690                 ctx->ctx_pmds[i].val = val = pfm_new_counter_value(ctx->ctx_pmds+ i, is_long_reset);
2691                 reset_others        |= ctx->ctx_pmds[i].reset_pmds[0];
2692 
2693                 DPRINT_ovfl((" %s reset ctx_pmds[%d]=%lx\n", is_long_reset ? "long" : "short", i, val));
2694         }
2695 
2696         /*
2697          * Now take care of resetting the other registers
2698          */
2699         for(i = 0; reset_others; i++, reset_others >>= 1) {
2700 
2701                 if ((reset_others & 0x1) == 0) continue;
2702 
2703                 ctx->ctx_pmds[i].val = val = pfm_new_counter_value(ctx->ctx_pmds + i, is_long_reset);
2704 
2705                 DPRINT_ovfl(("%s reset_others pmd[%d]=%lx\n",
2706                           is_long_reset ? "long" : "short", i, val));
2707         }
2708 }
2709 
2710 static void
2711 pfm_reset_regs(pfm_context_t *ctx, unsigned long *ovfl_regs, int is_long_reset)
2712 {
2713         unsigned long mask = ovfl_regs[0];
2714         unsigned long reset_others = 0UL;
2715         unsigned long val;
2716         int i;
2717 
2718         DPRINT_ovfl(("ovfl_regs=0x%lx is_long_reset=%d\n", ovfl_regs[0], is_long_reset));
2719 
2720         if (ctx->ctx_state == PFM_CTX_MASKED) {
2721                 pfm_reset_regs_masked(ctx, ovfl_regs, is_long_reset);
2722                 return;
2723         }
2724 
2725         /*
2726          * now restore reset value on sampling overflowed counters
2727          */
2728         mask >>= PMU_FIRST_COUNTER;
2729         for(i = PMU_FIRST_COUNTER; mask; i++, mask >>= 1) {
2730 
2731                 if ((mask & 0x1UL) == 0UL) continue;
2732 
2733                 val           = pfm_new_counter_value(ctx->ctx_pmds+ i, is_long_reset);
2734                 reset_others |= ctx->ctx_pmds[i].reset_pmds[0];
2735 
2736                 DPRINT_ovfl((" %s reset ctx_pmds[%d]=%lx\n", is_long_reset ? "long" : "short", i, val));
2737 
2738                 pfm_write_soft_counter(ctx, i, val);
2739         }
2740 
2741         /*
2742          * Now take care of resetting the other registers
2743          */
2744         for(i = 0; reset_others; i++, reset_others >>= 1) {
2745 
2746                 if ((reset_others & 0x1) == 0) continue;
2747 
2748                 val = pfm_new_counter_value(ctx->ctx_pmds + i, is_long_reset);
2749 
2750                 if (PMD_IS_COUNTING(i)) {
2751                         pfm_write_soft_counter(ctx, i, val);
2752                 } else {
2753                         ia64_set_pmd(i, val);
2754                 }
2755                 DPRINT_ovfl(("%s reset_others pmd[%d]=%lx\n",
2756                           is_long_reset ? "long" : "short", i, val));
2757         }
2758         ia64_srlz_d();
2759 }
2760 
2761 static int
2762 pfm_write_pmcs(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
2763 {
2764         struct task_struct *task;
2765         pfarg_reg_t *req = (pfarg_reg_t *)arg;
2766         unsigned long value, pmc_pm;
2767         unsigned long smpl_pmds, reset_pmds, impl_pmds;
2768         unsigned int cnum, reg_flags, flags, pmc_type;
2769         int i, can_access_pmu = 0, is_loaded, is_system, expert_mode;
2770         int is_monitor, is_counting, state;
2771         int ret = -EINVAL;
2772         pfm_reg_check_t wr_func;
2773 #define PFM_CHECK_PMC_PM(x, y, z) ((x)->ctx_fl_system ^ PMC_PM(y, z))
2774 
2775         state     = ctx->ctx_state;
2776         is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
2777         is_system = ctx->ctx_fl_system;
2778         task      = ctx->ctx_task;
2779         impl_pmds = pmu_conf->impl_pmds[0];
2780 
2781         if (state == PFM_CTX_ZOMBIE) return -EINVAL;
2782 
2783         if (is_loaded) {
2784                 /*
2785                  * In system wide and when the context is loaded, access can only happen
2786                  * when the caller is running on the CPU being monitored by the session.
2787                  * It does not have to be the owner (ctx_task) of the context per se.
2788                  */
2789                 if (is_system && ctx->ctx_cpu != smp_processor_id()) {
2790                         DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
2791                         return -EBUSY;
2792                 }
2793                 can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
2794         }
2795         expert_mode = pfm_sysctl.expert_mode; 
2796 
2797         for (i = 0; i < count; i++, req++) {
2798 
2799                 cnum       = req->reg_num;
2800                 reg_flags  = req->reg_flags;
2801                 value      = req->reg_value;
2802                 smpl_pmds  = req->reg_smpl_pmds[0];
2803                 reset_pmds = req->reg_reset_pmds[0];
2804                 flags      = 0;
2805 
2806 
2807                 if (cnum >= PMU_MAX_PMCS) {
2808                         DPRINT(("pmc%u is invalid\n", cnum));
2809                         goto error;
2810                 }
2811 
2812                 pmc_type   = pmu_conf->pmc_desc[cnum].type;
2813                 pmc_pm     = (value >> pmu_conf->pmc_desc[cnum].pm_pos) & 0x1;
2814                 is_counting = (pmc_type & PFM_REG_COUNTING) == PFM_REG_COUNTING ? 1 : 0;
2815                 is_monitor  = (pmc_type & PFM_REG_MONITOR) == PFM_REG_MONITOR ? 1 : 0;
2816 
2817                 /*
2818                  * we reject all non implemented PMC as well
2819                  * as attempts to modify PMC[0-3] which are used
2820                  * as status registers by the PMU
2821                  */
2822                 if ((pmc_type & PFM_REG_IMPL) == 0 || (pmc_type & PFM_REG_CONTROL) == PFM_REG_CONTROL) {
2823                         DPRINT(("pmc%u is unimplemented or no-access pmc_type=%x\n", cnum, pmc_type));
2824                         goto error;
2825                 }
2826                 wr_func = pmu_conf->pmc_desc[cnum].write_check;
2827                 /*
2828                  * If the PMC is a monitor, then if the value is not the default:
2829                  *      - system-wide session: PMCx.pm=1 (privileged monitor)
2830                  *      - per-task           : PMCx.pm=0 (user monitor)
2831                  */
2832                 if (is_monitor && value != PMC_DFL_VAL(cnum) && is_system ^ pmc_pm) {
2833                         DPRINT(("pmc%u pmc_pm=%lu is_system=%d\n",
2834                                 cnum,
2835                                 pmc_pm,
2836                                 is_system));
2837                         goto error;
2838                 }
2839 
2840                 if (is_counting) {
2841                         /*
2842                          * enforce generation of overflow interrupt. Necessary on all
2843                          * CPUs.
2844                          */
2845                         value |= 1 << PMU_PMC_OI;
2846 
2847                         if (reg_flags & PFM_REGFL_OVFL_NOTIFY) {
2848                                 flags |= PFM_REGFL_OVFL_NOTIFY;
2849                         }
2850 
2851                         if (reg_flags & PFM_REGFL_RANDOM) flags |= PFM_REGFL_RANDOM;
2852 
2853                         /* verify validity of smpl_pmds */
2854                         if ((smpl_pmds & impl_pmds) != smpl_pmds) {
2855                                 DPRINT(("invalid smpl_pmds 0x%lx for pmc%u\n", smpl_pmds, cnum));
2856                                 goto error;
2857                         }
2858 
2859                         /* verify validity of reset_pmds */
2860                         if ((reset_pmds & impl_pmds) != reset_pmds) {
2861                                 DPRINT(("invalid reset_pmds 0x%lx for pmc%u\n", reset_pmds, cnum));
2862                                 goto error;
2863                         }
2864                 } else {
2865                         if (reg_flags & (PFM_REGFL_OVFL_NOTIFY|PFM_REGFL_RANDOM)) {
2866                                 DPRINT(("cannot set ovfl_notify or random on pmc%u\n", cnum));
2867                                 goto error;
2868                         }
2869                         /* eventid on non-counting monitors are ignored */
2870                 }
2871 
2872                 /*
2873                  * execute write checker, if any
2874                  */
2875                 if (likely(expert_mode == 0 && wr_func)) {
2876                         ret = (*wr_func)(task, ctx, cnum, &value, regs);
2877                         if (ret) goto error;
2878                         ret = -EINVAL;
2879                 }
2880 
2881                 /*
2882                  * no error on this register
2883                  */
2884                 PFM_REG_RETFLAG_SET(req->reg_flags, 0);
2885 
2886                 /*
2887                  * Now we commit the changes to the software state
2888                  */
2889 
2890                 /*
2891                  * update overflow information
2892                  */
2893                 if (is_counting) {
2894                         /*
2895                          * full flag update each time a register is programmed
2896                          */
2897                         ctx->ctx_pmds[cnum].flags = flags;
2898 
2899                         ctx->ctx_pmds[cnum].reset_pmds[0] = reset_pmds;
2900                         ctx->ctx_pmds[cnum].smpl_pmds[0]  = smpl_pmds;
2901                         ctx->ctx_pmds[cnum].eventid       = req->reg_smpl_eventid;
2902 
2903                         /*
2904                          * Mark all PMDS to be accessed as used.
2905                          *
2906                          * We do not keep track of PMC because we have to
2907                          * systematically restore ALL of them.
2908                          *
2909                          * We do not update the used_monitors mask, because
2910                          * if we have not programmed them, then will be in
2911                          * a quiescent state, therefore we will not need to
2912                          * mask/restore then when context is MASKED.
2913                          */
2914                         CTX_USED_PMD(ctx, reset_pmds);
2915                         CTX_USED_PMD(ctx, smpl_pmds);
2916                         /*
2917                          * make sure we do not try to reset on
2918                          * restart because we have established new values
2919                          */
2920                         if (state == PFM_CTX_MASKED) ctx->ctx_ovfl_regs[0] &= ~1UL << cnum;
2921                 }
2922                 /*
2923                  * Needed in case the user does not initialize the equivalent
2924                  * PMD. Clearing is done indirectly via pfm_reset_pmu_state() so there is no
2925                  * possible leak here.
2926                  */
2927                 CTX_USED_PMD(ctx, pmu_conf->pmc_desc[cnum].dep_pmd[0]);
2928 
2929                 /*
2930                  * keep track of the monitor PMC that we are using.
2931                  * we save the value of the pmc in ctx_pmcs[] and if
2932                  * the monitoring is not stopped for the context we also
2933                  * place it in the saved state area so that it will be
2934                  * picked up later by the context switch code.
2935                  *
2936                  * The value in ctx_pmcs[] can only be changed in pfm_write_pmcs().
2937                  *
2938                  * The value in th_pmcs[] may be modified on overflow, i.e.,  when
2939                  * monitoring needs to be stopped.
2940                  */
2941                 if (is_monitor) CTX_USED_MONITOR(ctx, 1UL << cnum);
2942 
2943                 /*
2944                  * update context state
2945                  */
2946                 ctx->ctx_pmcs[cnum] = value;
2947 
2948                 if (is_loaded) {
2949                         /*
2950                          * write thread state
2951                          */
2952                         if (is_system == 0) ctx->th_pmcs[cnum] = value;
2953 
2954                         /*
2955                          * write hardware register if we can
2956                          */
2957                         if (can_access_pmu) {
2958                                 ia64_set_pmc(cnum, value);
2959                         }
2960 #ifdef CONFIG_SMP
2961                         else {
2962                                 /*
2963                                  * per-task SMP only here
2964                                  *
2965                                  * we are guaranteed that the task is not running on the other CPU,
2966                                  * we indicate that this PMD will need to be reloaded if the task
2967                                  * is rescheduled on the CPU it ran last on.
2968                                  */
2969                                 ctx->ctx_reload_pmcs[0] |= 1UL << cnum;
2970                         }
2971 #endif
2972                 }
2973 
2974                 DPRINT(("pmc[%u]=0x%lx ld=%d apmu=%d flags=0x%x all_pmcs=0x%lx used_pmds=0x%lx eventid=%ld smpl_pmds=0x%lx reset_pmds=0x%lx reloads_pmcs=0x%lx used_monitors=0x%lx ovfl_regs=0x%lx\n",
2975                           cnum,
2976                           value,
2977                           is_loaded,
2978                           can_access_pmu,
2979                           flags,
2980                           ctx->ctx_all_pmcs[0],
2981                           ctx->ctx_used_pmds[0],
2982                           ctx->ctx_pmds[cnum].eventid,
2983                           smpl_pmds,
2984                           reset_pmds,
2985                           ctx->ctx_reload_pmcs[0],
2986                           ctx->ctx_used_monitors[0],
2987                           ctx->ctx_ovfl_regs[0]));
2988         }
2989 
2990         /*
2991          * make sure the changes are visible
2992          */
2993         if (can_access_pmu) ia64_srlz_d();
2994 
2995         return 0;
2996 error:
2997         PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
2998         return ret;
2999 }
3000 
3001 static int
3002 pfm_write_pmds(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3003 {
3004         struct task_struct *task;
3005         pfarg_reg_t *req = (pfarg_reg_t *)arg;
3006         unsigned long value, hw_value, ovfl_mask;
3007         unsigned int cnum;
3008         int i, can_access_pmu = 0, state;
3009         int is_counting, is_loaded, is_system, expert_mode;
3010         int ret = -EINVAL;
3011         pfm_reg_check_t wr_func;
3012 
3013 
3014         state     = ctx->ctx_state;
3015         is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
3016         is_system = ctx->ctx_fl_system;
3017         ovfl_mask = pmu_conf->ovfl_val;
3018         task      = ctx->ctx_task;
3019 
3020         if (unlikely(state == PFM_CTX_ZOMBIE)) return -EINVAL;
3021 
3022         /*
3023          * on both UP and SMP, we can only write to the PMC when the task is
3024          * the owner of the local PMU.
3025          */
3026         if (likely(is_loaded)) {
3027                 /*
3028                  * In system wide and when the context is loaded, access can only happen
3029                  * when the caller is running on the CPU being monitored by the session.
3030                  * It does not have to be the owner (ctx_task) of the context per se.
3031                  */
3032                 if (unlikely(is_system && ctx->ctx_cpu != smp_processor_id())) {
3033                         DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
3034                         return -EBUSY;
3035                 }
3036                 can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
3037         }
3038         expert_mode = pfm_sysctl.expert_mode; 
3039 
3040         for (i = 0; i < count; i++, req++) {
3041 
3042                 cnum  = req->reg_num;
3043                 value = req->reg_value;
3044 
3045                 if (!PMD_IS_IMPL(cnum)) {
3046                         DPRINT(("pmd[%u] is unimplemented or invalid\n", cnum));
3047                         goto abort_mission;
3048                 }
3049                 is_counting = PMD_IS_COUNTING(cnum);
3050                 wr_func     = pmu_conf->pmd_desc[cnum].write_check;
3051 
3052                 /*
3053                  * execute write checker, if any
3054                  */
3055                 if (unlikely(expert_mode == 0 && wr_func)) {
3056                         unsigned long v = value;
3057 
3058                         ret = (*wr_func)(task, ctx, cnum, &v, regs);
3059                         if (ret) goto abort_mission;
3060 
3061                         value = v;
3062                         ret   = -EINVAL;
3063                 }
3064 
3065                 /*
3066                  * no error on this register
3067                  */
3068                 PFM_REG_RETFLAG_SET(req->reg_flags, 0);
3069 
3070                 /*
3071                  * now commit changes to software state
3072                  */
3073                 hw_value = value;
3074 
3075                 /*
3076                  * update virtualized (64bits) counter
3077                  */
3078                 if (is_counting) {
3079                         /*
3080                          * write context state
3081                          */
3082                         ctx->ctx_pmds[cnum].lval = value;
3083 
3084                         /*
3085                          * when context is load we use the split value
3086                          */
3087                         if (is_loaded) {
3088                                 hw_value = value &  ovfl_mask;
3089                                 value    = value & ~ovfl_mask;
3090                         }
3091                 }
3092                 /*
3093                  * update reset values (not just for counters)
3094                  */
3095                 ctx->ctx_pmds[cnum].long_reset  = req->reg_long_reset;
3096                 ctx->ctx_pmds[cnum].short_reset = req->reg_short_reset;
3097 
3098                 /*
3099                  * update randomization parameters (not just for counters)
3100                  */
3101                 ctx->ctx_pmds[cnum].seed = req->reg_random_seed;
3102                 ctx->ctx_pmds[cnum].mask = req->reg_random_mask;
3103 
3104                 /*
3105                  * update context value
3106                  */
3107                 ctx->ctx_pmds[cnum].val  = value;
3108 
3109                 /*
3110                  * Keep track of what we use
3111                  *
3112                  * We do not keep track of PMC because we have to
3113                  * systematically restore ALL of them.
3114                  */
3115                 CTX_USED_PMD(ctx, PMD_PMD_DEP(cnum));
3116 
3117                 /*
3118                  * mark this PMD register used as well
3119                  */
3120                 CTX_USED_PMD(ctx, RDEP(cnum));
3121 
3122                 /*
3123                  * make sure we do not try to reset on
3124                  * restart because we have established new values
3125                  */
3126                 if (is_counting && state == PFM_CTX_MASKED) {
3127                         ctx->ctx_ovfl_regs[0] &= ~1UL << cnum;
3128                 }
3129 
3130                 if (is_loaded) {
3131                         /*
3132                          * write thread state
3133                          */
3134                         if (is_system == 0) ctx->th_pmds[cnum] = hw_value;
3135 
3136                         /*
3137                          * write hardware register if we can
3138                          */
3139                         if (can_access_pmu) {
3140                                 ia64_set_pmd(cnum, hw_value);
3141                         } else {
3142 #ifdef CONFIG_SMP
3143                                 /*
3144                                  * we are guaranteed that the task is not running on the other CPU,
3145                                  * we indicate that this PMD will need to be reloaded if the task
3146                                  * is rescheduled on the CPU it ran last on.
3147                                  */
3148                                 ctx->ctx_reload_pmds[0] |= 1UL << cnum;
3149 #endif
3150                         }
3151                 }
3152 
3153                 DPRINT(("pmd[%u]=0x%lx ld=%d apmu=%d, hw_value=0x%lx ctx_pmd=0x%lx  short_reset=0x%lx "
3154                           "long_reset=0x%lx notify=%c seed=0x%lx mask=0x%lx used_pmds=0x%lx reset_pmds=0x%lx reload_pmds=0x%lx all_pmds=0x%lx ovfl_regs=0x%lx\n",
3155                         cnum,
3156                         value,
3157                         is_loaded,
3158                         can_access_pmu,
3159                         hw_value,
3160                         ctx->ctx_pmds[cnum].val,
3161                         ctx->ctx_pmds[cnum].short_reset,
3162                         ctx->ctx_pmds[cnum].long_reset,
3163                         PMC_OVFL_NOTIFY(ctx, cnum) ? 'Y':'N',
3164                         ctx->ctx_pmds[cnum].seed,
3165                         ctx->ctx_pmds[cnum].mask,
3166                         ctx->ctx_used_pmds[0],
3167                         ctx->ctx_pmds[cnum].reset_pmds[0],
3168                         ctx->ctx_reload_pmds[0],
3169                         ctx->ctx_all_pmds[0],
3170                         ctx->ctx_ovfl_regs[0]));
3171         }
3172 
3173         /*
3174          * make changes visible
3175          */
3176         if (can_access_pmu) ia64_srlz_d();
3177 
3178         return 0;
3179 
3180 abort_mission:
3181         /*
3182          * for now, we have only one possibility for error
3183          */
3184         PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
3185         return ret;
3186 }
3187 
3188 /*
3189  * By the way of PROTECT_CONTEXT(), interrupts are masked while we are in this function.
3190  * Therefore we know, we do not have to worry about the PMU overflow interrupt. If an
3191  * interrupt is delivered during the call, it will be kept pending until we leave, making
3192  * it appears as if it had been generated at the UNPROTECT_CONTEXT(). At least we are
3193  * guaranteed to return consistent data to the user, it may simply be old. It is not
3194  * trivial to treat the overflow while inside the call because you may end up in
3195  * some module sampling buffer code causing deadlocks.
3196  */
3197 static int
3198 pfm_read_pmds(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3199 {
3200         struct task_struct *task;
3201         unsigned long val = 0UL, lval, ovfl_mask, sval;
3202         pfarg_reg_t *req = (pfarg_reg_t *)arg;
3203         unsigned int cnum, reg_flags = 0;
3204         int i, can_access_pmu = 0, state;
3205         int is_loaded, is_system, is_counting, expert_mode;
3206         int ret = -EINVAL;
3207         pfm_reg_check_t rd_func;
3208 
3209         /*
3210          * access is possible when loaded only for
3211          * self-monitoring tasks or in UP mode
3212          */
3213 
3214         state     = ctx->ctx_state;
3215         is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
3216         is_system = ctx->ctx_fl_system;
3217         ovfl_mask = pmu_conf->ovfl_val;
3218         task      = ctx->ctx_task;
3219 
3220         if (state == PFM_CTX_ZOMBIE) return -EINVAL;
3221 
3222         if (likely(is_loaded)) {
3223                 /*
3224                  * In system wide and when the context is loaded, access can only happen
3225                  * when the caller is running on the CPU being monitored by the session.
3226                  * It does not have to be the owner (ctx_task) of the context per se.
3227                  */
3228                 if (unlikely(is_system && ctx->ctx_cpu != smp_processor_id())) {
3229                         DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
3230                         return -EBUSY;
3231                 }
3232                 /*
3233                  * this can be true when not self-monitoring only in UP
3234                  */
3235                 can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
3236 
3237                 if (can_access_pmu) ia64_srlz_d();
3238         }
3239         expert_mode = pfm_sysctl.expert_mode; 
3240 
3241         DPRINT(("ld=%d apmu=%d ctx_state=%d\n",
3242                 is_loaded,
3243                 can_access_pmu,
3244                 state));
3245 
3246         /*
3247          * on both UP and SMP, we can only read the PMD from the hardware register when
3248          * the task is the owner of the local PMU.
3249          */
3250 
3251         for (i = 0; i < count; i++, req++) {
3252 
3253                 cnum        = req->reg_num;
3254                 reg_flags   = req->reg_flags;
3255 
3256                 if (unlikely(!PMD_IS_IMPL(cnum))) goto error;
3257                 /*
3258                  * we can only read the register that we use. That includes
3259                  * the one we explicitly initialize AND the one we want included
3260                  * in the sampling buffer (smpl_regs).
3261                  *
3262                  * Having this restriction allows optimization in the ctxsw routine
3263                  * without compromising security (leaks)
3264                  */
3265                 if (unlikely(!CTX_IS_USED_PMD(ctx, cnum))) goto error;
3266 
3267                 sval        = ctx->ctx_pmds[cnum].val;
3268                 lval        = ctx->ctx_pmds[cnum].lval;
3269                 is_counting = PMD_IS_COUNTING(cnum);
3270 
3271                 /*
3272                  * If the task is not the current one, then we check if the
3273                  * PMU state is still in the local live register due to lazy ctxsw.
3274                  * If true, then we read directly from the registers.
3275                  */
3276                 if (can_access_pmu){
3277                         val = ia64_get_pmd(cnum);
3278                 } else {
3279                         /*
3280                          * context has been saved
3281                          * if context is zombie, then task does not exist anymore.
3282                          * In this case, we use the full value saved in the context (pfm_flush_regs()).
3283                          */
3284                         val = is_loaded ? ctx->th_pmds[cnum] : 0UL;
3285                 }
3286                 rd_func = pmu_conf->pmd_desc[cnum].read_check;
3287 
3288                 if (is_counting) {
3289                         /*
3290                          * XXX: need to check for overflow when loaded
3291                          */
3292                         val &= ovfl_mask;
3293                         val += sval;
3294                 }
3295 
3296                 /*
3297                  * execute read checker, if any
3298                  */
3299                 if (unlikely(expert_mode == 0 && rd_func)) {
3300                         unsigned long v = val;
3301                         ret = (*rd_func)(ctx->ctx_task, ctx, cnum, &v, regs);
3302                         if (ret) goto error;
3303                         val = v;
3304                         ret = -EINVAL;
3305                 }
3306 
3307                 PFM_REG_RETFLAG_SET(reg_flags, 0);
3308 
3309                 DPRINT(("pmd[%u]=0x%lx\n", cnum, val));
3310 
3311                 /*
3312                  * update register return value, abort all if problem during copy.
3313                  * we only modify the reg_flags field. no check mode is fine because
3314                  * access has been verified upfront in sys_perfmonctl().
3315                  */
3316                 req->reg_value            = val;
3317                 req->reg_flags            = reg_flags;
3318                 req->reg_last_reset_val   = lval;
3319         }
3320 
3321         return 0;
3322 
3323 error:
3324         PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
3325         return ret;
3326 }
3327 
3328 int
3329 pfm_mod_write_pmcs(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
3330 {
3331         pfm_context_t *ctx;
3332 
3333         if (req == NULL) return -EINVAL;
3334 
3335         ctx = GET_PMU_CTX();
3336 
3337         if (ctx == NULL) return -EINVAL;
3338 
3339         /*
3340          * for now limit to current task, which is enough when calling
3341          * from overflow handler
3342          */
3343         if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
3344 
3345         return pfm_write_pmcs(ctx, req, nreq, regs);
3346 }
3347 EXPORT_SYMBOL(pfm_mod_write_pmcs);
3348 
3349 int
3350 pfm_mod_read_pmds(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
3351 {
3352         pfm_context_t *ctx;
3353 
3354         if (req == NULL) return -EINVAL;
3355 
3356         ctx = GET_PMU_CTX();
3357 
3358         if (ctx == NULL) return -EINVAL;
3359 
3360         /*
3361          * for now limit to current task, which is enough when calling
3362          * from overflow handler
3363          */
3364         if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
3365 
3366         return pfm_read_pmds(ctx, req, nreq, regs);
3367 }
3368 EXPORT_SYMBOL(pfm_mod_read_pmds);
3369 
3370 /*
3371  * Only call this function when a process it trying to
3372  * write the debug registers (reading is always allowed)
3373  */
3374 int
3375 pfm_use_debug_registers(struct task_struct *task)
3376 {
3377         pfm_context_t *ctx = task->thread.pfm_context;
3378         unsigned long flags;
3379         int ret = 0;
3380 
3381         if (pmu_conf->use_rr_dbregs == 0) return 0;
3382 
3383         DPRINT(("called for [%d]\n", task_pid_nr(task)));
3384 
3385         /*
3386          * do it only once
3387          */
3388         if (task->thread.flags & IA64_THREAD_DBG_VALID) return 0;
3389 
3390         /*
3391          * Even on SMP, we do not need to use an atomic here because
3392          * the only way in is via ptrace() and this is possible only when the
3393          * process is stopped. Even in the case where the ctxsw out is not totally
3394          * completed by the time we come here, there is no way the 'stopped' process
3395          * could be in the middle of fiddling with the pfm_write_ibr_dbr() routine.
3396          * So this is always safe.
3397          */
3398         if (ctx && ctx->ctx_fl_using_dbreg == 1) return -1;
3399 
3400         LOCK_PFS(flags);
3401 
3402         /*
3403          * We cannot allow setting breakpoints when system wide monitoring
3404          * sessions are using the debug registers.
3405          */
3406         if (pfm_sessions.pfs_sys_use_dbregs> 0)
3407                 ret = -1;
3408         else
3409                 pfm_sessions.pfs_ptrace_use_dbregs++;
3410 
3411         DPRINT(("ptrace_use_dbregs=%u  sys_use_dbregs=%u by [%d] ret = %d\n",
3412                   pfm_sessions.pfs_ptrace_use_dbregs,
3413                   pfm_sessions.pfs_sys_use_dbregs,
3414                   task_pid_nr(task), ret));
3415 
3416         UNLOCK_PFS(flags);
3417 
3418         return ret;
3419 }
3420 
3421 /*
3422  * This function is called for every task that exits with the
3423  * IA64_THREAD_DBG_VALID set. This indicates a task which was
3424  * able to use the debug registers for debugging purposes via
3425  * ptrace(). Therefore we know it was not using them for
3426  * performance monitoring, so we only decrement the number
3427  * of "ptraced" debug register users to keep the count up to date
3428  */
3429 int
3430 pfm_release_debug_registers(struct task_struct *task)
3431 {
3432         unsigned long flags;
3433         int ret;
3434 
3435         if (pmu_conf->use_rr_dbregs == 0) return 0;
3436 
3437         LOCK_PFS(flags);
3438         if (pfm_sessions.pfs_ptrace_use_dbregs == 0) {
3439                 printk(KERN_ERR "perfmon: invalid release for [%d] ptrace_use_dbregs=0\n", task_pid_nr(task));
3440                 ret = -1;
3441         }  else {
3442                 pfm_sessions.pfs_ptrace_use_dbregs--;
3443                 ret = 0;
3444         }
3445         UNLOCK_PFS(flags);
3446 
3447         return ret;
3448 }
3449 
3450 static int
3451 pfm_restart(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3452 {
3453         struct task_struct *task;
3454         pfm_buffer_fmt_t *fmt;
3455         pfm_ovfl_ctrl_t rst_ctrl;
3456         int state, is_system;
3457         int ret = 0;
3458 
3459         state     = ctx->ctx_state;
3460         fmt       = ctx->ctx_buf_fmt;
3461         is_system = ctx->ctx_fl_system;
3462         task      = PFM_CTX_TASK(ctx);
3463 
3464         switch(state) {
3465                 case PFM_CTX_MASKED:
3466                         break;
3467                 case PFM_CTX_LOADED: 
3468                         if (CTX_HAS_SMPL(ctx) && fmt->fmt_restart_active) break;
3469                         /* fall through */
3470                 case PFM_CTX_UNLOADED:
3471                 case PFM_CTX_ZOMBIE:
3472                         DPRINT(("invalid state=%d\n", state));
3473                         return -EBUSY;
3474                 default:
3475                         DPRINT(("state=%d, cannot operate (no active_restart handler)\n", state));
3476                         return -EINVAL;
3477         }
3478 
3479         /*
3480          * In system wide and when the context is loaded, access can only happen
3481          * when the caller is running on the CPU being monitored by the session.
3482          * It does not have to be the owner (ctx_task) of the context per se.
3483          */
3484         if (is_system && ctx->ctx_cpu != smp_processor_id()) {
3485                 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
3486                 return -EBUSY;
3487         }
3488 
3489         /* sanity check */
3490         if (unlikely(task == NULL)) {
3491                 printk(KERN_ERR "perfmon: [%d] pfm_restart no task\n", task_pid_nr(current));
3492                 return -EINVAL;
3493         }
3494 
3495         if (task == current || is_system) {
3496 
3497                 fmt = ctx->ctx_buf_fmt;
3498 
3499                 DPRINT(("restarting self %d ovfl=0x%lx\n",
3500                         task_pid_nr(task),
3501                         ctx->ctx_ovfl_regs[0]));
3502 
3503                 if (CTX_HAS_SMPL(ctx)) {
3504 
3505                         prefetch(ctx->ctx_smpl_hdr);
3506 
3507                         rst_ctrl.bits.mask_monitoring = 0;
3508                         rst_ctrl.bits.reset_ovfl_pmds = 0;
3509 
3510                         if (state == PFM_CTX_LOADED)
3511                                 ret = pfm_buf_fmt_restart_active(fmt, task, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
3512                         else
3513                                 ret = pfm_buf_fmt_restart(fmt, task, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
3514                 } else {
3515                         rst_ctrl.bits.mask_monitoring = 0;
3516                         rst_ctrl.bits.reset_ovfl_pmds = 1;
3517                 }
3518 
3519                 if (ret == 0) {
3520                         if (rst_ctrl.bits.reset_ovfl_pmds)
3521                                 pfm_reset_regs(ctx, ctx->ctx_ovfl_regs, PFM_PMD_LONG_RESET);
3522 
3523                         if (rst_ctrl.bits.mask_monitoring == 0) {
3524                                 DPRINT(("resuming monitoring for [%d]\n", task_pid_nr(task)));
3525 
3526                                 if (state == PFM_CTX_MASKED) pfm_restore_monitoring(task);
3527                         } else {
3528                                 DPRINT(("keeping monitoring stopped for [%d]\n", task_pid_nr(task)));
3529 
3530                                 // cannot use pfm_stop_monitoring(task, regs);
3531                         }
3532                 }
3533                 /*
3534                  * clear overflowed PMD mask to remove any stale information
3535                  */
3536                 ctx->ctx_ovfl_regs[0] = 0UL;
3537 
3538                 /*
3539                  * back to LOADED state
3540                  */
3541                 ctx->ctx_state = PFM_CTX_LOADED;
3542 
3543                 /*
3544                  * XXX: not really useful for self monitoring
3545                  */
3546                 ctx->ctx_fl_can_restart = 0;
3547 
3548                 return 0;
3549         }
3550 
3551         /* 
3552          * restart another task
3553          */
3554 
3555         /*
3556          * When PFM_CTX_MASKED, we cannot issue a restart before the previous 
3557          * one is seen by the task.
3558          */
3559         if (state == PFM_CTX_MASKED) {
3560                 if (ctx->ctx_fl_can_restart == 0) return -EINVAL;
3561                 /*
3562                  * will prevent subsequent restart before this one is
3563                  * seen by other task
3564                  */
3565                 ctx->ctx_fl_can_restart = 0;
3566         }
3567 
3568         /*
3569          * if blocking, then post the semaphore is PFM_CTX_MASKED, i.e.
3570          * the task is blocked or on its way to block. That's the normal
3571          * restart path. If the monitoring is not masked, then the task
3572          * can be actively monitoring and we cannot directly intervene.
3573          * Therefore we use the trap mechanism to catch the task and
3574          * force it to reset the buffer/reset PMDs.
3575          *
3576          * if non-blocking, then we ensure that the task will go into
3577          * pfm_handle_work() before returning to user mode.
3578          *
3579          * We cannot explicitly reset another task, it MUST always
3580          * be done by the task itself. This works for system wide because
3581          * the tool that is controlling the session is logically doing 
3582          * "self-monitoring".
3583          */
3584         if (CTX_OVFL_NOBLOCK(ctx) == 0 && state == PFM_CTX_MASKED) {
3585                 DPRINT(("unblocking [%d]\n", task_pid_nr(task)));
3586                 complete(&ctx->ctx_restart_done);
3587         } else {
3588                 DPRINT(("[%d] armed exit trap\n", task_pid_nr(task)));
3589 
3590                 ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_RESET;
3591 
3592                 PFM_SET_WORK_PENDING(task, 1);
3593 
3594                 set_notify_resume(task);
3595 
3596                 /*
3597                  * XXX: send reschedule if task runs on another CPU
3598                  */
3599         }
3600         return 0;
3601 }
3602 
3603 static int
3604 pfm_debug(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3605 {
3606         unsigned int m = *(unsigned int *)arg;
3607 
3608         pfm_sysctl.debug = m == 0 ? 0 : 1;
3609 
3610         printk(KERN_INFO "perfmon debugging %s (timing reset)\n", pfm_sysctl.debug ? "on" : "off");
3611 
3612         if (m == 0) {
3613                 memset(pfm_stats, 0, sizeof(pfm_stats));
3614                 for(m=0; m < NR_CPUS; m++) pfm_stats[m].pfm_ovfl_intr_cycles_min = ~0UL;
3615         }
3616         return 0;
3617 }
3618 
3619 /*
3620  * arg can be NULL and count can be zero for this function
3621  */
3622 static int
3623 pfm_write_ibr_dbr(int mode, pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3624 {
3625         struct thread_struct *thread = NULL;
3626         struct task_struct *task;
3627         pfarg_dbreg_t *req = (pfarg_dbreg_t *)arg;
3628         unsigned long flags;
3629         dbreg_t dbreg;
3630         unsigned int rnum;
3631         int first_time;
3632         int ret = 0, state;
3633         int i, can_access_pmu = 0;
3634         int is_system, is_loaded;
3635 
3636         if (pmu_conf->use_rr_dbregs == 0) return -EINVAL;
3637 
3638         state     = ctx->ctx_state;
3639         is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
3640         is_system = ctx->ctx_fl_system;
3641         task      = ctx->ctx_task;
3642 
3643         if (state == PFM_CTX_ZOMBIE) return -EINVAL;
3644 
3645         /*
3646          * on both UP and SMP, we can only write to the PMC when the task is
3647          * the owner of the local PMU.
3648          */
3649         if (is_loaded) {
3650                 thread = &task->thread;
3651                 /*
3652                  * In system wide and when the context is loaded, access can only happen
3653                  * when the caller is running on the CPU being monitored by the session.
3654                  * It does not have to be the owner (ctx_task) of the context per se.
3655                  */
3656                 if (unlikely(is_system && ctx->ctx_cpu != smp_processor_id())) {
3657                         DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
3658                         return -EBUSY;
3659                 }
3660                 can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
3661         }
3662 
3663         /*
3664          * we do not need to check for ipsr.db because we do clear ibr.x, dbr.r, and dbr.w
3665          * ensuring that no real breakpoint can be installed via this call.
3666          *
3667          * IMPORTANT: regs can be NULL in this function
3668          */
3669 
3670         first_time = ctx->ctx_fl_using_dbreg == 0;
3671 
3672         /*
3673          * don't bother if we are loaded and task is being debugged
3674          */
3675         if (is_loaded && (thread->flags & IA64_THREAD_DBG_VALID) != 0) {
3676                 DPRINT(("debug registers already in use for [%d]\n", task_pid_nr(task)));
3677                 return -EBUSY;
3678         }
3679 
3680         /*
3681          * check for debug registers in system wide mode
3682          *
3683          * If though a check is done in pfm_context_load(),
3684          * we must repeat it here, in case the registers are
3685          * written after the context is loaded
3686          */
3687         if (is_loaded) {
3688                 LOCK_PFS(flags);
3689 
3690                 if (first_time && is_system) {
3691                         if (pfm_sessions.pfs_ptrace_use_dbregs)
3692                                 ret = -EBUSY;
3693                         else
3694                                 pfm_sessions.pfs_sys_use_dbregs++;
3695                 }
3696                 UNLOCK_PFS(flags);
3697         }
3698 
3699         if (ret != 0) return ret;
3700 
3701         /*
3702          * mark ourself as user of the debug registers for
3703          * perfmon purposes.
3704          */
3705         ctx->ctx_fl_using_dbreg = 1;
3706 
3707         /*
3708          * clear hardware registers to make sure we don't
3709          * pick up stale state.
3710          *
3711          * for a system wide session, we do not use
3712          * thread.dbr, thread.ibr because this process
3713          * never leaves the current CPU and the state
3714          * is shared by all processes running on it
3715          */
3716         if (first_time && can_access_pmu) {
3717                 DPRINT(("[%d] clearing ibrs, dbrs\n", task_pid_nr(task)));
3718                 for (i=0; i < pmu_conf->num_ibrs; i++) {
3719                         ia64_set_ibr(i, 0UL);
3720                         ia64_dv_serialize_instruction();
3721                 }
3722                 ia64_srlz_i();
3723                 for (i=0; i < pmu_conf->num_dbrs; i++) {
3724                         ia64_set_dbr(i, 0UL);
3725                         ia64_dv_serialize_data();
3726                 }
3727                 ia64_srlz_d();
3728         }
3729 
3730         /*
3731          * Now install the values into the registers
3732          */
3733         for (i = 0; i < count; i++, req++) {
3734 
3735                 rnum      = req->dbreg_num;
3736                 dbreg.val = req->dbreg_value;
3737 
3738                 ret = -EINVAL;
3739 
3740                 if ((mode == PFM_CODE_RR && rnum >= PFM_NUM_IBRS) || ((mode == PFM_DATA_RR) && rnum >= PFM_NUM_DBRS)) {
3741                         DPRINT(("invalid register %u val=0x%lx mode=%d i=%d count=%d\n",
3742                                   rnum, dbreg.val, mode, i, count));
3743 
3744                         goto abort_mission;
3745                 }
3746 
3747                 /*
3748                  * make sure we do not install enabled breakpoint
3749                  */
3750                 if (rnum & 0x1) {
3751                         if (mode == PFM_CODE_RR)
3752                                 dbreg.ibr.ibr_x = 0;
3753                         else
3754                                 dbreg.dbr.dbr_r = dbreg.dbr.dbr_w = 0;
3755                 }
3756 
3757                 PFM_REG_RETFLAG_SET(req->dbreg_flags, 0);
3758 
3759                 /*
3760                  * Debug registers, just like PMC, can only be modified
3761                  * by a kernel call. Moreover, perfmon() access to those
3762                  * registers are centralized in this routine. The hardware
3763                  * does not modify the value of these registers, therefore,
3764                  * if we save them as they are written, we can avoid having
3765                  * to save them on context switch out. This is made possible
3766                  * by the fact that when perfmon uses debug registers, ptrace()
3767                  * won't be able to modify them concurrently.
3768                  */
3769                 if (mode == PFM_CODE_RR) {
3770                         CTX_USED_IBR(ctx, rnum);
3771 
3772                         if (can_access_pmu) {
3773                                 ia64_set_ibr(rnum, dbreg.val);
3774                                 ia64_dv_serialize_instruction();
3775                         }
3776 
3777                         ctx->ctx_ibrs[rnum] = dbreg.val;
3778 
3779                         DPRINT(("write ibr%u=0x%lx used_ibrs=0x%x ld=%d apmu=%d\n",
3780                                 rnum, dbreg.val, ctx->ctx_used_ibrs[0], is_loaded, can_access_pmu));
3781                 } else {
3782                         CTX_USED_DBR(ctx, rnum);
3783 
3784                         if (can_access_pmu) {
3785                                 ia64_set_dbr(rnum, dbreg.val);
3786                                 ia64_dv_serialize_data();
3787                         }
3788                         ctx->ctx_dbrs[rnum] = dbreg.val;
3789 
3790                         DPRINT(("write dbr%u=0x%lx used_dbrs=0x%x ld=%d apmu=%d\n",
3791                                 rnum, dbreg.val, ctx->ctx_used_dbrs[0], is_loaded, can_access_pmu));
3792                 }
3793         }
3794 
3795         return 0;
3796 
3797 abort_mission:
3798         /*
3799          * in case it was our first attempt, we undo the global modifications
3800          */
3801         if (first_time) {
3802                 LOCK_PFS(flags);
3803                 if (ctx->ctx_fl_system) {
3804                         pfm_sessions.pfs_sys_use_dbregs--;
3805                 }
3806                 UNLOCK_PFS(flags);
3807                 ctx->ctx_fl_using_dbreg = 0;
3808         }
3809         /*
3810          * install error return flag
3811          */
3812         PFM_REG_RETFLAG_SET(req->dbreg_flags, PFM_REG_RETFL_EINVAL);
3813 
3814         return ret;
3815 }
3816 
3817 static int
3818 pfm_write_ibrs(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3819 {
3820         return pfm_write_ibr_dbr(PFM_CODE_RR, ctx, arg, count, regs);
3821 }
3822 
3823 static int
3824 pfm_write_dbrs(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3825 {
3826         return pfm_write_ibr_dbr(PFM_DATA_RR, ctx, arg, count, regs);
3827 }
3828 
3829 int
3830 pfm_mod_write_ibrs(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
3831 {
3832         pfm_context_t *ctx;
3833 
3834         if (req == NULL) return -EINVAL;
3835 
3836         ctx = GET_PMU_CTX();
3837 
3838         if (ctx == NULL) return -EINVAL;
3839 
3840         /*
3841          * for now limit to current task, which is enough when calling
3842          * from overflow handler
3843          */
3844         if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
3845 
3846         return pfm_write_ibrs(ctx, req, nreq, regs);
3847 }
3848 EXPORT_SYMBOL(pfm_mod_write_ibrs);
3849 
3850 int
3851 pfm_mod_write_dbrs(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
3852 {
3853         pfm_context_t *ctx;
3854 
3855         if (req == NULL) return -EINVAL;
3856 
3857         ctx = GET_PMU_CTX();
3858 
3859         if (ctx == NULL) return -EINVAL;
3860 
3861         /*
3862          * for now limit to current task, which is enough when calling
3863          * from overflow handler
3864          */
3865         if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
3866 
3867         return pfm_write_dbrs(ctx, req, nreq, regs);
3868 }
3869 EXPORT_SYMBOL(pfm_mod_write_dbrs);
3870 
3871 
3872 static int
3873 pfm_get_features(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3874 {
3875         pfarg_features_t *req = (pfarg_features_t *)arg;
3876 
3877         req->ft_version = PFM_VERSION;
3878         return 0;
3879 }
3880 
3881 static int
3882 pfm_stop(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3883 {
3884         struct pt_regs *tregs;
3885         struct task_struct *task = PFM_CTX_TASK(ctx);
3886         int state, is_system;
3887 
3888         state     = ctx->ctx_state;
3889         is_system = ctx->ctx_fl_system;
3890 
3891         /*
3892          * context must be attached to issue the stop command (includes LOADED,MASKED,ZOMBIE)
3893          */
3894         if (state == PFM_CTX_UNLOADED) return -EINVAL;
3895 
3896         /*
3897          * In system wide and when the context is loaded, access can only happen
3898          * when the caller is running on the CPU being monitored by the session.
3899          * It does not have to be the owner (ctx_task) of the context per se.
3900          */
3901         if (is_system && ctx->ctx_cpu != smp_processor_id()) {
3902                 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
3903                 return -EBUSY;
3904         }
3905         DPRINT(("task [%d] ctx_state=%d is_system=%d\n",
3906                 task_pid_nr(PFM_CTX_TASK(ctx)),
3907                 state,
3908                 is_system));
3909         /*
3910          * in system mode, we need to update the PMU directly
3911          * and the user level state of the caller, which may not
3912          * necessarily be the creator of the context.
3913          */
3914         if (is_system) {
3915                 /*
3916                  * Update local PMU first
3917                  *
3918                  * disable dcr pp
3919                  */
3920                 ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) & ~IA64_DCR_PP);
3921                 ia64_srlz_i();
3922 
3923                 /*
3924                  * update local cpuinfo
3925                  */
3926                 PFM_CPUINFO_CLEAR(PFM_CPUINFO_DCR_PP);
3927 
3928                 /*
3929                  * stop monitoring, does srlz.i
3930                  */
3931                 pfm_clear_psr_pp();
3932 
3933                 /*
3934                  * stop monitoring in the caller
3935                  */
3936                 ia64_psr(regs)->pp = 0;
3937 
3938                 return 0;
3939         }
3940         /*
3941          * per-task mode
3942          */
3943 
3944         if (task == current) {
3945                 /* stop monitoring  at kernel level */
3946                 pfm_clear_psr_up();
3947 
3948                 /*
3949                  * stop monitoring at the user level
3950                  */
3951                 ia64_psr(regs)->up = 0;
3952         } else {
3953                 tregs = task_pt_regs(task);
3954 
3955                 /*
3956                  * stop monitoring at the user level
3957                  */
3958                 ia64_psr(tregs)->up = 0;
3959 
3960                 /*
3961                  * monitoring disabled in kernel at next reschedule
3962                  */
3963                 ctx->ctx_saved_psr_up = 0;
3964                 DPRINT(("task=[%d]\n", task_pid_nr(task)));
3965         }
3966         return 0;
3967 }
3968 
3969 
3970 static int
3971 pfm_start(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
3972 {
3973         struct pt_regs *tregs;
3974         int state, is_system;
3975 
3976         state     = ctx->ctx_state;
3977         is_system = ctx->ctx_fl_system;
3978 
3979         if (state != PFM_CTX_LOADED) return -EINVAL;
3980 
3981         /*
3982          * In system wide and when the context is loaded, access can only happen
3983          * when the caller is running on the CPU being monitored by the session.
3984          * It does not have to be the owner (ctx_task) of the context per se.
3985          */
3986         if (is_system && ctx->ctx_cpu != smp_processor_id()) {
3987                 DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
3988                 return -EBUSY;
3989         }
3990 
3991         /*
3992          * in system mode, we need to update the PMU directly
3993          * and the user level state of the caller, which may not
3994          * necessarily be the creator of the context.
3995          */
3996         if (is_system) {
3997 
3998                 /*
3999                  * set user level psr.pp for the caller
4000                  */
4001                 ia64_psr(regs)->pp = 1;
4002 
4003                 /*
4004                  * now update the local PMU and cpuinfo
4005                  */
4006                 PFM_CPUINFO_SET(PFM_CPUINFO_DCR_PP);
4007 
4008                 /*
4009                  * start monitoring at kernel level
4010                  */
4011                 pfm_set_psr_pp();
4012 
4013                 /* enable dcr pp */
4014                 ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) | IA64_DCR_PP);
4015                 ia64_srlz_i();
4016 
4017                 return 0;
4018         }
4019 
4020         /*
4021          * per-process mode
4022          */
4023 
4024         if (ctx->ctx_task == current) {
4025 
4026                 /* start monitoring at kernel level */
4027                 pfm_set_psr_up();
4028 
4029                 /*
4030                  * activate monitoring at user level
4031                  */
4032                 ia64_psr(regs)->up = 1;
4033 
4034         } else {
4035                 tregs = task_pt_regs(ctx->ctx_task);
4036 
4037                 /*
4038                  * start monitoring at the kernel level the next
4039                  * time the task is scheduled
4040                  */
4041                 ctx->ctx_saved_psr_up = IA64_PSR_UP;
4042 
4043                 /*
4044                  * activate monitoring at user level
4045                  */
4046                 ia64_psr(tregs)->up = 1;
4047         }
4048         return 0;
4049 }
4050 
4051 static int
4052 pfm_get_pmc_reset(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
4053 {
4054         pfarg_reg_t *req = (pfarg_reg_t *)arg;
4055         unsigned int cnum;
4056         int i;
4057         int ret = -EINVAL;
4058 
4059         for (i = 0; i < count; i++, req++) {
4060 
4061                 cnum = req->reg_num;
4062 
4063                 if (!PMC_IS_IMPL(cnum)) goto abort_mission;
4064 
4065                 req->reg_value = PMC_DFL_VAL(cnum);
4066 
4067                 PFM_REG_RETFLAG_SET(req->reg_flags, 0);
4068 
4069                 DPRINT(("pmc_reset_val pmc[%u]=0x%lx\n", cnum, req->reg_value));
4070         }
4071         return 0;
4072 
4073 abort_mission:
4074         PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
4075         return ret;
4076 }
4077 
4078 static int
4079 pfm_check_task_exist(pfm_context_t *ctx)
4080 {
4081         struct task_struct *g, *t;
4082         int ret = -ESRCH;
4083 
4084         read_lock(&tasklist_lock);
4085 
4086         do_each_thread (g, t) {
4087                 if (t->thread.pfm_context == ctx) {
4088                         ret = 0;
4089                         goto out;
4090                 }
4091         } while_each_thread (g, t);
4092 out:
4093         read_unlock(&tasklist_lock);
4094 
4095         DPRINT(("pfm_check_task_exist: ret=%d ctx=%p\n", ret, ctx));
4096 
4097         return ret;
4098 }
4099 
4100 static int
4101 pfm_context_load(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
4102 {
4103         struct task_struct *task;
4104         struct thread_struct *thread;
4105         struct pfm_context_t *old;
4106         unsigned long flags;
4107 #ifndef CONFIG_SMP
4108         struct task_struct *owner_task = NULL;
4109 #endif
4110         pfarg_load_t *req = (pfarg_load_t *)arg;
4111         unsigned long *pmcs_source, *pmds_source;
4112         int the_cpu;
4113         int ret = 0;
4114         int state, is_system, set_dbregs = 0;
4115 
4116         state     = ctx->ctx_state;
4117         is_system = ctx->ctx_fl_system;
4118         /*
4119          * can only load from unloaded or terminated state
4120          */
4121         if (state != PFM_CTX_UNLOADED) {
4122                 DPRINT(("cannot load to [%d], invalid ctx_state=%d\n",
4123                         req->load_pid,
4124                         ctx->ctx_state));
4125                 return -EBUSY;
4126         }
4127 
4128         DPRINT(("load_pid [%d] using_dbreg=%d\n", req->load_pid, ctx->ctx_fl_using_dbreg));
4129 
4130         if (CTX_OVFL_NOBLOCK(ctx) == 0 && req->load_pid == current->pid) {
4131                 DPRINT(("cannot use blocking mode on self\n"));
4132                 return -EINVAL;
4133         }
4134 
4135         ret = pfm_get_task(ctx, req->load_pid, &task);
4136         if (ret) {
4137                 DPRINT(("load_pid [%d] get_task=%d\n", req->load_pid, ret));
4138                 return ret;
4139         }
4140 
4141         ret = -EINVAL;
4142 
4143         /*
4144          * system wide is self monitoring only
4145          */
4146         if (is_system && task != current) {
4147                 DPRINT(("system wide is self monitoring only load_pid=%d\n",
4148                         req->load_pid));
4149                 goto error;
4150         }
4151 
4152         thread = &task->thread;
4153 
4154         ret = 0;
4155         /*
4156          * cannot load a context which is using range restrictions,
4157          * into a task that is being debugged.
4158          */
4159         if (ctx->ctx_fl_using_dbreg) {
4160                 if (thread->flags & IA64_THREAD_DBG_VALID) {
4161                         ret = -EBUSY;
4162                         DPRINT(("load_pid [%d] task is debugged, cannot load range restrictions\n", req->load_pid));
4163                         goto error;
4164                 }
4165                 LOCK_PFS(flags);
4166 
4167                 if (is_system) {
4168                         if (pfm_sessions.pfs_ptrace_use_dbregs) {
4169                                 DPRINT(("cannot load [%d] dbregs in use\n",
4170                                                         task_pid_nr(task)));
4171                                 ret = -EBUSY;
4172                         } else {
4173                                 pfm_sessions.pfs_sys_use_dbregs++;
4174                                 DPRINT(("load [%d] increased sys_use_dbreg=%u\n", task_pid_nr(task), pfm_sessions.pfs_sys_use_dbregs));
4175                                 set_dbregs = 1;
4176                         }
4177                 }
4178 
4179                 UNLOCK_PFS(flags);
4180 
4181                 if (ret) goto error;
4182         }
4183 
4184         /*
4185          * SMP system-wide monitoring implies self-monitoring.
4186          *
4187          * The programming model expects the task to
4188          * be pinned on a CPU throughout the session.
4189          * Here we take note of the current CPU at the
4190          * time the context is loaded. No call from
4191          * another CPU will be allowed.
4192          *
4193          * The pinning via shed_setaffinity()
4194          * must be done by the calling task prior
4195          * to this call.
4196          *
4197          * systemwide: keep track of CPU this session is supposed to run on
4198          */
4199         the_cpu = ctx->ctx_cpu = smp_processor_id();
4200 
4201         ret = -EBUSY;
4202         /*
4203          * now reserve the session
4204          */
4205         ret = pfm_reserve_session(current, is_system, the_cpu);
4206         if (ret) goto error;
4207 
4208         /*
4209          * task is necessarily stopped at this point.
4210          *
4211          * If the previous context was zombie, then it got removed in
4212          * pfm_save_regs(). Therefore we should not see it here.
4213          * If we see a context, then this is an active context
4214          *
4215          * XXX: needs to be atomic
4216          */
4217         DPRINT(("before cmpxchg() old_ctx=%p new_ctx=%p\n",
4218                 thread->pfm_context, ctx));
4219 
4220         ret = -EBUSY;
4221         old = ia64_cmpxchg(acq, &thread->pfm_context, NULL, ctx, sizeof(pfm_context_t *));
4222         if (old != NULL) {
4223                 DPRINT(("load_pid [%d] already has a context\n", req->load_pid));
4224                 goto error_unres;
4225         }
4226 
4227         pfm_reset_msgq(ctx);
4228 
4229         ctx->ctx_state = PFM_CTX_LOADED;
4230 
4231         /*
4232          * link context to task
4233          */
4234         ctx->ctx_task = task;
4235 
4236         if (is_system) {
4237                 /*
4238                  * we load as stopped
4239                  */
4240                 PFM_CPUINFO_SET(PFM_CPUINFO_SYST_WIDE);
4241                 PFM_CPUINFO_CLEAR(PFM_CPUINFO_DCR_PP);
4242 
4243                 if (ctx->ctx_fl_excl_idle) PFM_CPUINFO_SET(PFM_CPUINFO_EXCL_IDLE);
4244         } else {
4245                 thread->flags |= IA64_THREAD_PM_VALID;
4246         }
4247 
4248         /*
4249          * propagate into thread-state
4250          */
4251         pfm_copy_pmds(task, ctx);
4252         pfm_copy_pmcs(task, ctx);
4253 
4254         pmcs_source = ctx->th_pmcs;
4255         pmds_source = ctx->th_pmds;
4256 
4257         /*
4258          * always the case for system-wide
4259          */
4260         if (task == current) {
4261 
4262                 if (is_system == 0) {
4263 
4264                         /* allow user level control */
4265                         ia64_psr(regs)->sp = 0;
4266                         DPRINT(("clearing psr.sp for [%d]\n", task_pid_nr(task)));
4267 
4268                         SET_LAST_CPU(ctx, smp_processor_id());
4269                         INC_ACTIVATION();
4270                         SET_ACTIVATION(ctx);
4271 #ifndef CONFIG_SMP
4272                         /*
4273                          * push the other task out, if any
4274                          */
4275                         owner_task = GET_PMU_OWNER();
4276                         if (owner_task) pfm_lazy_save_regs(owner_task);
4277 #endif
4278                 }
4279                 /*
4280                  * load all PMD from ctx to PMU (as opposed to thread state)
4281                  * restore all PMC from ctx to PMU
4282                  */
4283                 pfm_restore_pmds(pmds_source, ctx->ctx_all_pmds[0]);
4284                 pfm_restore_pmcs(pmcs_source, ctx->ctx_all_pmcs[0]);
4285 
4286                 ctx->ctx_reload_pmcs[0] = 0UL;
4287                 ctx->ctx_reload_pmds[0] = 0UL;
4288 
4289                 /*
4290                  * guaranteed safe by earlier check against DBG_VALID
4291                  */
4292                 if (ctx->ctx_fl_using_dbreg) {
4293                         pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
4294                         pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
4295                 }
4296                 /*
4297                  * set new ownership
4298                  */
4299                 SET_PMU_OWNER(task, ctx);
4300 
4301                 DPRINT(("context loaded on PMU for [%d]\n", task_pid_nr(task)));
4302         } else {
4303                 /*
4304                  * when not current, task MUST be stopped, so this is safe
4305                  */
4306                 regs = task_pt_regs(task);
4307 
4308                 /* force a full reload */
4309                 ctx->ctx_last_activation = PFM_INVALID_ACTIVATION;
4310                 SET_LAST_CPU(ctx, -1);
4311 
4312                 /* initial saved psr (stopped) */
4313                 ctx->ctx_saved_psr_up = 0UL;
4314                 ia64_psr(regs)->up = ia64_psr(regs)->pp = 0;
4315         }
4316 
4317         ret = 0;
4318 
4319 error_unres:
4320         if (ret) pfm_unreserve_session(ctx, ctx->ctx_fl_system, the_cpu);
4321 error:
4322         /*
4323          * we must undo the dbregs setting (for system-wide)
4324          */
4325         if (ret && set_dbregs) {
4326                 LOCK_PFS(flags);
4327                 pfm_sessions.pfs_sys_use_dbregs--;
4328                 UNLOCK_PFS(flags);
4329         }
4330         /*
4331          * release task, there is now a link with the context
4332          */
4333         if (is_system == 0 && task != current) {
4334                 pfm_put_task(task);
4335 
4336                 if (ret == 0) {
4337                         ret = pfm_check_task_exist(ctx);
4338                         if (ret) {
4339                                 ctx->ctx_state = PFM_CTX_UNLOADED;
4340                                 ctx->ctx_task  = NULL;
4341                         }
4342                 }
4343         }
4344         return ret;
4345 }
4346 
4347 /*
4348  * in this function, we do not need to increase the use count
4349  * for the task via get_task_struct(), because we hold the
4350  * context lock. If the task were to disappear while having
4351  * a context attached, it would go through pfm_exit_thread()
4352  * which also grabs the context lock  and would therefore be blocked
4353  * until we are here.
4354  */
4355 static void pfm_flush_pmds(struct task_struct *, pfm_context_t *ctx);
4356 
4357 static int
4358 pfm_context_unload(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
4359 {
4360         struct task_struct *task = PFM_CTX_TASK(ctx);
4361         struct pt_regs *tregs;
4362         int prev_state, is_system;
4363         int ret;
4364 
4365         DPRINT(("ctx_state=%d task [%d]\n", ctx->ctx_state, task ? task_pid_nr(task) : -1));
4366 
4367         prev_state = ctx->ctx_state;
4368         is_system  = ctx->ctx_fl_system;
4369 
4370         /*
4371          * unload only when necessary
4372          */
4373         if (prev_state == PFM_CTX_UNLOADED) {
4374                 DPRINT(("ctx_state=%d, nothing to do\n", prev_state));
4375                 return 0;
4376         }
4377 
4378         /*
4379          * clear psr and dcr bits
4380          */
4381         ret = pfm_stop(ctx, NULL, 0, regs);
4382         if (ret) return ret;
4383 
4384         ctx->ctx_state = PFM_CTX_UNLOADED;
4385 
4386         /*
4387          * in system mode, we need to update the PMU directly
4388          * and the user level state of the caller, which may not
4389          * necessarily be the creator of the context.
4390          */
4391         if (is_system) {
4392 
4393                 /*
4394                  * Update cpuinfo
4395                  *
4396                  * local PMU is taken care of in pfm_stop()
4397                  */
4398                 PFM_CPUINFO_CLEAR(PFM_CPUINFO_SYST_WIDE);
4399                 PFM_CPUINFO_CLEAR(PFM_CPUINFO_EXCL_IDLE);
4400 
4401                 /*
4402                  * save PMDs in context
4403                  * release ownership
4404                  */
4405                 pfm_flush_pmds(current, ctx);
4406 
4407                 /*
4408                  * at this point we are done with the PMU
4409                  * so we can unreserve the resource.
4410                  */
4411                 if (prev_state != PFM_CTX_ZOMBIE) 
4412                         pfm_unreserve_session(ctx, 1 , ctx->ctx_cpu);
4413 
4414                 /*
4415                  * disconnect context from task
4416                  */
4417                 task->thread.pfm_context = NULL;
4418                 /*
4419                  * disconnect task from context
4420                  */
4421                 ctx->ctx_task = NULL;
4422 
4423                 /*
4424                  * There is nothing more to cleanup here.
4425                  */
4426                 return 0;
4427         }
4428 
4429         /*
4430          * per-task mode
4431          */
4432         tregs = task == current ? regs : task_pt_regs(task);
4433 
4434         if (task == current) {
4435                 /*
4436                  * cancel user level control
4437                  */
4438                 ia64_psr(regs)->sp = 1;
4439 
4440                 DPRINT(("setting psr.sp for [%d]\n", task_pid_nr(task)));
4441         }
4442         /*
4443          * save PMDs to context
4444          * release ownership
4445          */
4446         pfm_flush_pmds(task, ctx);
4447 
4448         /*
4449          * at this point we are done with the PMU
4450          * so we can unreserve the resource.
4451          *
4452          * when state was ZOMBIE, we have already unreserved.
4453          */
4454         if (prev_state != PFM_CTX_ZOMBIE) 
4455                 pfm_unreserve_session(ctx, 0 , ctx->ctx_cpu);
4456 
4457         /*
4458          * reset activation counter and psr
4459          */
4460         ctx->ctx_last_activation = PFM_INVALID_ACTIVATION;
4461         SET_LAST_CPU(ctx, -1);
4462 
4463         /*
4464          * PMU state will not be restored
4465          */
4466         task->thread.flags &= ~IA64_THREAD_PM_VALID;
4467 
4468         /*
4469          * break links between context and task
4470          */
4471         task->thread.pfm_context  = NULL;
4472         ctx->ctx_task             = NULL;
4473 
4474         PFM_SET_WORK_PENDING(task, 0);
4475 
4476         ctx->ctx_fl_trap_reason  = PFM_TRAP_REASON_NONE;
4477         ctx->ctx_fl_can_restart  = 0;
4478         ctx->ctx_fl_going_zombie = 0;
4479 
4480         DPRINT(("disconnected [%d] from context\n", task_pid_nr(task)));
4481 
4482         return 0;
4483 }
4484 
4485 
4486 /*
4487  * called only from exit_thread()
4488  * we come here only if the task has a context attached (loaded or masked)
4489  */
4490 void
4491 pfm_exit_thread(struct task_struct *task)
4492 {
4493         pfm_context_t *ctx;
4494         unsigned long flags;
4495         struct pt_regs *regs = task_pt_regs(task);
4496         int ret, state;
4497         int free_ok = 0;
4498 
4499         ctx = PFM_GET_CTX(task);
4500 
4501         PROTECT_CTX(ctx, flags);
4502 
4503         DPRINT(("state=%d task [%d]\n", ctx->ctx_state, task_pid_nr(task)));
4504 
4505         state = ctx->ctx_state;
4506         switch(state) {
4507                 case PFM_CTX_UNLOADED:
4508                         /*
4509                          * only comes to this function if pfm_context is not NULL, i.e., cannot
4510                          * be in unloaded state
4511                          */
4512                         printk(KERN_ERR "perfmon: pfm_exit_thread [%d] ctx unloaded\n", task_pid_nr(task));
4513                         break;
4514                 case PFM_CTX_LOADED:
4515                 case PFM_CTX_MASKED:
4516                         ret = pfm_context_unload(ctx, NULL, 0, regs);
4517                         if (ret) {
4518                                 printk(KERN_ERR "perfmon: pfm_exit_thread [%d] state=%d unload failed %d\n", task_pid_nr(task), state, ret);
4519                         }
4520                         DPRINT(("ctx unloaded for current state was %d\n", state));
4521 
4522                         pfm_end_notify_user(ctx);
4523                         break;
4524                 case PFM_CTX_ZOMBIE:
4525                         ret = pfm_context_unload(ctx, NULL, 0, regs);
4526                         if (ret) {
4527                                 printk(KERN_ERR "perfmon: pfm_exit_thread [%d] state=%d unload failed %d\n", task_pid_nr(task), state, ret);
4528                         }
4529                         free_ok = 1;
4530                         break;
4531                 default:
4532                         printk(KERN_ERR "perfmon: pfm_exit_thread [%d] unexpected state=%d\n", task_pid_nr(task), state);
4533                         break;
4534         }
4535         UNPROTECT_CTX(ctx, flags);
4536 
4537         { u64 psr = pfm_get_psr();
4538           BUG_ON(psr & (IA64_PSR_UP|IA64_PSR_PP));
4539           BUG_ON(GET_PMU_OWNER());
4540           BUG_ON(ia64_psr(regs)->up);
4541           BUG_ON(ia64_psr(regs)->pp);
4542         }
4543 
4544         /*
4545          * All memory free operations (especially for vmalloc'ed memory)
4546          * MUST be done with interrupts ENABLED.
4547          */
4548         if (free_ok) pfm_context_free(ctx);
4549 }
4550 
4551 /*
4552  * functions MUST be listed in the increasing order of their index (see permfon.h)
4553  */
4554 #define PFM_CMD(name, flags, arg_count, arg_type, getsz) { name, #name, flags, arg_count, sizeof(arg_type), getsz }
4555 #define PFM_CMD_S(name, flags) { name, #name, flags, 0, 0, NULL }
4556 #define PFM_CMD_PCLRWS  (PFM_CMD_FD|PFM_CMD_ARG_RW|PFM_CMD_STOP)
4557 #define PFM_CMD_PCLRW   (PFM_CMD_FD|PFM_CMD_ARG_RW)
4558 #define PFM_CMD_NONE    { NULL, "no-cmd", 0, 0, 0, NULL}
4559 
4560 static pfm_cmd_desc_t pfm_cmd_tab[]={
4561 /* 0  */PFM_CMD_NONE,
4562 /* 1  */PFM_CMD(pfm_write_pmcs, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
4563 /* 2  */PFM_CMD(pfm_write_pmds, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
4564 /* 3  */PFM_CMD(pfm_read_pmds, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
4565 /* 4  */PFM_CMD_S(pfm_stop, PFM_CMD_PCLRWS),
4566 /* 5  */PFM_CMD_S(pfm_start, PFM_CMD_PCLRWS),
4567 /* 6  */PFM_CMD_NONE,
4568 /* 7  */PFM_CMD_NONE,
4569 /* 8  */PFM_CMD(pfm_context_create, PFM_CMD_ARG_RW, 1, pfarg_context_t, pfm_ctx_getsize),
4570 /* 9  */PFM_CMD_NONE,
4571 /* 10 */PFM_CMD_S(pfm_restart, PFM_CMD_PCLRW),
4572 /* 11 */PFM_CMD_NONE,
4573 /* 12 */PFM_CMD(pfm_get_features, PFM_CMD_ARG_RW, 1, pfarg_features_t, NULL),
4574 /* 13 */PFM_CMD(pfm_debug, 0, 1, unsigned int, NULL),
4575 /* 14 */PFM_CMD_NONE,
4576 /* 15 */PFM_CMD(pfm_get_pmc_reset, PFM_CMD_ARG_RW, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
4577 /* 16 */PFM_CMD(pfm_context_load, PFM_CMD_PCLRWS, 1, pfarg_load_t, NULL),
4578 /* 17 */PFM_CMD_S(pfm_context_unload, PFM_CMD_PCLRWS),
4579 /* 18 */PFM_CMD_NONE,
4580 /* 19 */PFM_CMD_NONE,
4581 /* 20 */PFM_CMD_NONE,
4582 /* 21 */PFM_CMD_NONE,
4583 /* 22 */PFM_CMD_NONE,
4584 /* 23 */PFM_CMD_NONE,
4585 /* 24 */PFM_CMD_NONE,
4586 /* 25 */PFM_CMD_NONE,
4587 /* 26 */PFM_CMD_NONE,
4588 /* 27 */PFM_CMD_NONE,
4589 /* 28 */PFM_CMD_NONE,
4590 /* 29 */PFM_CMD_NONE,
4591 /* 30 */PFM_CMD_NONE,
4592 /* 31 */PFM_CMD_NONE,
4593 /* 32 */PFM_CMD(pfm_write_ibrs, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_dbreg_t, NULL),
4594 /* 33 */PFM_CMD(pfm_write_dbrs, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_dbreg_t, NULL)
4595 };
4596 #define PFM_CMD_COUNT   (sizeof(pfm_cmd_tab)/sizeof(pfm_cmd_desc_t))
4597 
4598 static int
4599 pfm_check_task_state(pfm_context_t *ctx, int cmd, unsigned long flags)
4600 {
4601         struct task_struct *task;
4602         int state, old_state;
4603 
4604 recheck:
4605         state = ctx->ctx_state;
4606         task  = ctx->ctx_task;
4607 
4608         if (task == NULL) {
4609                 DPRINT(("context %d no task, state=%d\n", ctx->ctx_fd, state));
4610                 return 0;
4611         }
4612 
4613         DPRINT(("context %d state=%d [%d] task_state=%ld must_stop=%d\n",
4614                 ctx->ctx_fd,
4615                 state,
4616                 task_pid_nr(task),
4617                 task->state, PFM_CMD_STOPPED(cmd)));
4618 
4619         /*
4620          * self-monitoring always ok.
4621          *
4622          * for system-wide the caller can either be the creator of the
4623          * context (to one to which the context is attached to) OR
4624          * a task running on the same CPU as the session.
4625          */
4626         if (task == current || ctx->ctx_fl_system) return 0;
4627 
4628         /*
4629          * we are monitoring another thread
4630          */
4631         switch(state) {
4632                 case PFM_CTX_UNLOADED:
4633                         /*
4634                          * if context is UNLOADED we are safe to go
4635                          */
4636                         return 0;
4637                 case PFM_CTX_ZOMBIE:
4638                         /*
4639                          * no command can operate on a zombie context
4640                          */
4641                         DPRINT(("cmd %d state zombie cannot operate on context\n", cmd));
4642                         return -EINVAL;
4643                 case PFM_CTX_MASKED:
4644                         /*
4645                          * PMU state has been saved to software even though
4646                          * the thread may still be running.
4647                          */
4648                         if (cmd != PFM_UNLOAD_CONTEXT) return 0;
4649         }
4650 
4651         /*
4652          * context is LOADED or MASKED. Some commands may need to have 
4653          * the task stopped.
4654          *
4655          * We could lift this restriction for UP but it would mean that
4656          * the user has no guarantee the task would not run between
4657          * two successive calls to perfmonctl(). That's probably OK.
4658          * If this user wants to ensure the task does not run, then
4659          * the task must be stopped.
4660          */
4661         if (PFM_CMD_STOPPED(cmd)) {
4662                 if (!task_is_stopped_or_traced(task)) {
4663                         DPRINT(("[%d] task not in stopped state\n", task_pid_nr(task)));
4664                         return -EBUSY;
4665                 }
4666                 /*
4667                  * task is now stopped, wait for ctxsw out
4668                  *
4669                  * This is an interesting point in the code.
4670                  * We need to unprotect the context because
4671                  * the pfm_save_regs() routines needs to grab
4672                  * the same lock. There are danger in doing
4673                  * this because it leaves a window open for
4674                  * another task to get access to the context
4675                  * and possibly change its state. The one thing
4676                  * that is not possible is for the context to disappear
4677                  * because we are protected by the VFS layer, i.e.,
4678                  * get_fd()/put_fd().
4679                  */
4680                 old_state = state;
4681 
4682                 UNPROTECT_CTX(ctx, flags);
4683 
4684                 wait_task_inactive(task, 0);
4685 
4686                 PROTECT_CTX(ctx, flags);
4687 
4688                 /*
4689                  * we must recheck to verify if state has changed
4690                  */
4691                 if (ctx->ctx_state != old_state) {
4692                         DPRINT(("old_state=%d new_state=%d\n", old_state, ctx->ctx_state));
4693                         goto recheck;
4694                 }
4695         }
4696         return 0;
4697 }
4698 
4699 /*
4700  * system-call entry point (must return long)
4701  */
4702 asmlinkage long
4703 sys_perfmonctl (int fd, int cmd, void __user *arg, int count)
4704 {
4705         struct fd f = {NULL, 0};
4706         pfm_context_t *ctx = NULL;
4707         unsigned long flags = 0UL;
4708         void *args_k = NULL;
4709         long ret; /* will expand int return types */
4710         size_t base_sz, sz, xtra_sz = 0;
4711         int narg, completed_args = 0, call_made = 0, cmd_flags;
4712         int (*func)(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
4713         int (*getsize)(void *arg, size_t *sz);
4714 #define PFM_MAX_ARGSIZE 4096
4715 
4716         /*
4717          * reject any call if perfmon was disabled at initialization
4718          */
4719         if (unlikely(pmu_conf == NULL)) return -ENOSYS;
4720 
4721         if (unlikely(cmd < 0 || cmd >= PFM_CMD_COUNT)) {
4722                 DPRINT(("invalid cmd=%d\n", cmd));
4723                 return -EINVAL;
4724         }
4725 
4726         func      = pfm_cmd_tab[cmd].cmd_func;
4727         narg      = pfm_cmd_tab[cmd].cmd_narg;
4728         base_sz   = pfm_cmd_tab[cmd].cmd_argsize;
4729         getsize   = pfm_cmd_tab[cmd].cmd_getsize;
4730         cmd_flags = pfm_cmd_tab[cmd].cmd_flags;
4731 
4732         if (unlikely(func == NULL)) {
4733                 DPRINT(("invalid cmd=%d\n", cmd));
4734                 return -EINVAL;
4735         }
4736 
4737         DPRINT(("cmd=%s idx=%d narg=0x%x argsz=%lu count=%d\n",
4738                 PFM_CMD_NAME(cmd),
4739                 cmd,
4740                 narg,
4741                 base_sz,
4742                 count));
4743 
4744         /*
4745          * check if number of arguments matches what the command expects
4746          */
4747         if (unlikely((narg == PFM_CMD_ARG_MANY && count <= 0) || (narg > 0 && narg != count)))
4748                 return -EINVAL;
4749 
4750 restart_args:
4751         sz = xtra_sz + base_sz*count;
4752         /*
4753          * limit abuse to min page size
4754          */
4755         if (unlikely(sz > PFM_MAX_ARGSIZE)) {
4756                 printk(KERN_ERR "perfmon: [%d] argument too big %lu\n", task_pid_nr(current), sz);
4757                 return -E2BIG;
4758         }
4759 
4760         /*
4761          * allocate default-sized argument buffer
4762          */
4763         if (likely(count && args_k == NULL)) {
4764                 args_k = kmalloc(PFM_MAX_ARGSIZE, GFP_KERNEL);
4765                 if (args_k == NULL) return -ENOMEM;
4766         }
4767 
4768         ret = -EFAULT;
4769 
4770         /*
4771          * copy arguments
4772          *
4773          * assume sz = 0 for command without parameters
4774          */
4775         if (sz && copy_from_user(args_k, arg, sz)) {
4776                 DPRINT(("cannot copy_from_user %lu bytes @%p\n", sz, arg));
4777                 goto error_args;
4778         }
4779 
4780         /*
4781          * check if command supports extra parameters
4782          */
4783         if (completed_args == 0 && getsize) {
4784                 /*
4785                  * get extra parameters size (based on main argument)
4786                  */
4787                 ret = (*getsize)(args_k, &xtra_sz);
4788                 if (ret) goto error_args;
4789 
4790                 completed_args = 1;
4791 
4792                 DPRINT(("restart_args sz=%lu xtra_sz=%lu\n", sz, xtra_sz));
4793 
4794                 /* retry if necessary */
4795                 if (likely(xtra_sz)) goto restart_args;
4796         }
4797 
4798         if (unlikely((cmd_flags & PFM_CMD_FD) == 0)) goto skip_fd;
4799 
4800         ret = -EBADF;
4801 
4802         f = fdget(fd);
4803         if (unlikely(f.file == NULL)) {
4804                 DPRINT(("invalid fd %d\n", fd));
4805                 goto error_args;
4806         }
4807         if (unlikely(PFM_IS_FILE(f.file) == 0)) {
4808                 DPRINT(("fd %d not related to perfmon\n", fd));
4809                 goto error_args;
4810         }
4811 
4812         ctx = f.file->private_data;
4813         if (unlikely(ctx == NULL)) {
4814                 DPRINT(("no context for fd %d\n", fd));
4815                 goto error_args;
4816         }
4817         prefetch(&ctx->ctx_state);
4818 
4819         PROTECT_CTX(ctx, flags);
4820 
4821         /*
4822          * check task is stopped
4823          */
4824         ret = pfm_check_task_state(ctx, cmd, flags);
4825         if (unlikely(ret)) goto abort_locked;
4826 
4827 skip_fd:
4828         ret = (*func)(ctx, args_k, count, task_pt_regs(current));
4829 
4830         call_made = 1;
4831 
4832 abort_locked:
4833         if (likely(ctx)) {
4834                 DPRINT(("context unlocked\n"));
4835                 UNPROTECT_CTX(ctx, flags);
4836         }
4837 
4838         /* copy argument back to user, if needed */
4839         if (call_made && PFM_CMD_RW_ARG(cmd) && copy_to_user(arg, args_k, base_sz*count)) ret = -EFAULT;
4840 
4841 error_args:
4842         if (f.file)
4843                 fdput(f);
4844 
4845         kfree(args_k);
4846 
4847         DPRINT(("cmd=%s ret=%ld\n", PFM_CMD_NAME(cmd), ret));
4848 
4849         return ret;
4850 }
4851 
4852 static void
4853 pfm_resume_after_ovfl(pfm_context_t *ctx, unsigned long ovfl_regs, struct pt_regs *regs)
4854 {
4855         pfm_buffer_fmt_t *fmt = ctx->ctx_buf_fmt;
4856         pfm_ovfl_ctrl_t rst_ctrl;
4857         int state;
4858         int ret = 0;
4859 
4860         state = ctx->ctx_state;
4861         /*
4862          * Unlock sampling buffer and reset index atomically
4863          * XXX: not really needed when blocking
4864          */
4865         if (CTX_HAS_SMPL(ctx)) {
4866 
4867                 rst_ctrl.bits.mask_monitoring = 0;
4868                 rst_ctrl.bits.reset_ovfl_pmds = 0;
4869 
4870                 if (state == PFM_CTX_LOADED)
4871                         ret = pfm_buf_fmt_restart_active(fmt, current, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
4872                 else
4873                         ret = pfm_buf_fmt_restart(fmt, current, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
4874         } else {
4875                 rst_ctrl.bits.mask_monitoring = 0;
4876                 rst_ctrl.bits.reset_ovfl_pmds = 1;
4877         }
4878 
4879         if (ret == 0) {
4880                 if (rst_ctrl.bits.reset_ovfl_pmds) {
4881                         pfm_reset_regs(ctx, &ovfl_regs, PFM_PMD_LONG_RESET);
4882                 }
4883                 if (rst_ctrl.bits.mask_monitoring == 0) {
4884                         DPRINT(("resuming monitoring\n"));
4885                         if (ctx->ctx_state == PFM_CTX_MASKED) pfm_restore_monitoring(current);
4886                 } else {
4887                         DPRINT(("stopping monitoring\n"));
4888                         //pfm_stop_monitoring(current, regs);
4889                 }
4890                 ctx->ctx_state = PFM_CTX_LOADED;
4891         }
4892 }
4893 
4894 /*
4895  * context MUST BE LOCKED when calling
4896  * can only be called for current
4897  */
4898 static void
4899 pfm_context_force_terminate(pfm_context_t *ctx, struct pt_regs *regs)
4900 {
4901         int ret;
4902 
4903         DPRINT(("entering for [%d]\n", task_pid_nr(current)));
4904 
4905         ret = pfm_context_unload(ctx, NULL, 0, regs);
4906         if (ret) {
4907                 printk(KERN_ERR "pfm_context_force_terminate: [%d] unloaded failed with %d\n", task_pid_nr(current), ret);
4908         }
4909 
4910         /*
4911          * and wakeup controlling task, indicating we are now disconnected
4912          */
4913         wake_up_interruptible(&ctx->ctx_zombieq);
4914 
4915         /*
4916          * given that context is still locked, the controlling
4917          * task will only get access when we return from
4918          * pfm_handle_work().
4919          */
4920 }
4921 
4922 static int pfm_ovfl_notify_user(pfm_context_t *ctx, unsigned long ovfl_pmds);
4923 
4924  /*
4925   * pfm_handle_work() can be called with interrupts enabled
4926   * (TIF_NEED_RESCHED) or disabled. The down_interruptible
4927   * call may sleep, therefore we must re-enable interrupts
4928   * to avoid deadlocks. It is safe to do so because this function
4929   * is called ONLY when returning to user level (pUStk=1), in which case
4930   * there is no risk of kernel stack overflow due to deep
4931   * interrupt nesting.
4932   */
4933 void
4934 pfm_handle_work(void)
4935 {
4936         pfm_context_t *ctx;
4937         struct pt_regs *regs;
4938         unsigned long flags, dummy_flags;
4939         unsigned long ovfl_regs;
4940         unsigned int reason;
4941         int ret;
4942 
4943         ctx = PFM_GET_CTX(current);
4944         if (ctx == NULL) {
4945                 printk(KERN_ERR "perfmon: [%d] has no PFM context\n",
4946                         task_pid_nr(current));
4947                 return;
4948         }
4949 
4950         PROTECT_CTX(ctx, flags);
4951 
4952         PFM_SET_WORK_PENDING(current, 0);
4953 
4954         regs = task_pt_regs(current);
4955 
4956         /*
4957          * extract reason for being here and clear
4958          */
4959         reason = ctx->ctx_fl_trap_reason;
4960         ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_NONE;
4961         ovfl_regs = ctx->ctx_ovfl_regs[0];
4962 
4963         DPRINT(("reason=%d state=%d\n", reason, ctx->ctx_state));
4964 
4965         /*
4966          * must be done before we check for simple-reset mode
4967          */
4968         if (ctx->ctx_fl_going_zombie || ctx->ctx_state == PFM_CTX_ZOMBIE)
4969                 goto do_zombie;
4970 
4971         //if (CTX_OVFL_NOBLOCK(ctx)) goto skip_blocking;
4972         if (reason == PFM_TRAP_REASON_RESET)
4973                 goto skip_blocking;
4974 
4975         /*
4976          * restore interrupt mask to what it was on entry.
4977          * Could be enabled/diasbled.
4978          */
4979         UNPROTECT_CTX(ctx, flags);
4980 
4981         /*
4982          * force interrupt enable because of down_interruptible()
4983          */
4984         local_irq_enable();
4985 
4986         DPRINT(("before block sleeping\n"));
4987 
4988         /*
4989          * may go through without blocking on SMP systems
4990          * if restart has been received already by the time we call down()
4991          */
4992         ret = wait_for_completion_interruptible(&ctx->ctx_restart_done);
4993 
4994         DPRINT(("after block sleeping ret=%d\n", ret));
4995 
4996         /*
4997          * lock context and mask interrupts again
4998          * We save flags into a dummy because we may have
4999          * altered interrupts mask compared to entry in this
5000          * function.
5001          */
5002         PROTECT_CTX(ctx, dummy_flags);
5003 
5004         /*
5005          * we need to read the ovfl_regs only after wake-up
5006          * because we may have had pfm_write_pmds() in between
5007          * and that can changed PMD values and therefore 
5008          * ovfl_regs is reset for these new PMD values.
5009          */
5010         ovfl_regs = ctx->ctx_ovfl_regs[0];
5011 
5012         if (ctx->ctx_fl_going_zombie) {
5013 do_zombie:
5014                 DPRINT(("context is zombie, bailing out\n"));
5015                 pfm_context_force_terminate(ctx, regs);
5016                 goto nothing_to_do;
5017         }
5018         /*
5019          * in case of interruption of down() we don't restart anything
5020          */
5021         if (ret < 0)
5022                 goto nothing_to_do;
5023 
5024 skip_blocking:
5025         pfm_resume_after_ovfl(ctx, ovfl_regs, regs);
5026         ctx->ctx_ovfl_regs[0] = 0UL;
5027 
5028 nothing_to_do:
5029         /*
5030          * restore flags as they were upon entry
5031          */
5032         UNPROTECT_CTX(ctx, flags);
5033 }
5034 
5035 static int
5036 pfm_notify_user(pfm_context_t *ctx, pfm_msg_t *msg)
5037 {
5038         if (ctx->ctx_state == PFM_CTX_ZOMBIE) {
5039                 DPRINT(("ignoring overflow notification, owner is zombie\n"));
5040                 return 0;
5041         }
5042 
5043         DPRINT(("waking up somebody\n"));
5044 
5045         if (msg) wake_up_interruptible(&ctx->ctx_msgq_wait);
5046 
5047         /*
5048          * safe, we are not in intr handler, nor in ctxsw when
5049          * we come here
5050          */
5051         kill_fasync (&ctx->ctx_async_queue, SIGIO, POLL_IN);
5052 
5053         return 0;
5054 }
5055 
5056 static int
5057 pfm_ovfl_notify_user(pfm_context_t *ctx, unsigned long ovfl_pmds)
5058 {
5059         pfm_msg_t *msg = NULL;
5060 
5061         if (ctx->ctx_fl_no_msg == 0) {
5062                 msg = pfm_get_new_msg(ctx);
5063                 if (msg == NULL) {
5064                         printk(KERN_ERR "perfmon: pfm_ovfl_notify_user no more notification msgs\n");
5065                         return -1;
5066                 }
5067 
5068                 msg->pfm_ovfl_msg.msg_type         = PFM_MSG_OVFL;
5069                 msg->pfm_ovfl_msg.msg_ctx_fd       = ctx->ctx_fd;
5070                 msg->pfm_ovfl_msg.msg_active_set   = 0;
5071                 msg->pfm_ovfl_msg.msg_ovfl_pmds[0] = ovfl_pmds;
5072                 msg->pfm_ovfl_msg.msg_ovfl_pmds[1] = 0UL;
5073                 msg->pfm_ovfl_msg.msg_ovfl_pmds[2] = 0UL;
5074                 msg->pfm_ovfl_msg.msg_ovfl_pmds[3] = 0UL;
5075                 msg->pfm_ovfl_msg.msg_tstamp       = 0UL;
5076         }
5077 
5078         DPRINT(("ovfl msg: msg=%p no_msg=%d fd=%d ovfl_pmds=0x%lx\n",
5079                 msg,
5080                 ctx->ctx_fl_no_msg,
5081                 ctx->ctx_fd,
5082                 ovfl_pmds));
5083 
5084         return pfm_notify_user(ctx, msg);
5085 }
5086 
5087 static int
5088 pfm_end_notify_user(pfm_context_t *ctx)
5089 {
5090         pfm_msg_t *msg;
5091 
5092         msg = pfm_get_new_msg(ctx);
5093         if (msg == NULL) {
5094                 printk(KERN_ERR "perfmon: pfm_end_notify_user no more notification msgs\n");
5095                 return -1;
5096         }
5097         /* no leak */
5098         memset(msg, 0, sizeof(*msg));
5099 
5100         msg->pfm_end_msg.msg_type    = PFM_MSG_END;
5101         msg->pfm_end_msg.msg_ctx_fd  = ctx->ctx_fd;
5102         msg->pfm_ovfl_msg.msg_tstamp = 0UL;
5103 
5104         DPRINT(("end msg: msg=%p no_msg=%d ctx_fd=%d\n",
5105                 msg,
5106                 ctx->ctx_fl_no_msg,
5107                 ctx->ctx_fd));
5108 
5109         return pfm_notify_user(ctx, msg);
5110 }
5111 
5112 /*
5113  * main overflow processing routine.
5114  * it can be called from the interrupt path or explicitly during the context switch code
5115  */
5116 static void pfm_overflow_handler(struct task_struct *task, pfm_context_t *ctx,
5117                                 unsigned long pmc0, struct pt_regs *regs)
5118 {
5119         pfm_ovfl_arg_t *ovfl_arg;
5120         unsigned long mask;
5121         unsigned long old_val, ovfl_val, new_val;
5122         unsigned long ovfl_notify = 0UL, ovfl_pmds = 0UL, smpl_pmds = 0UL, reset_pmds;
5123         unsigned long tstamp;
5124         pfm_ovfl_ctrl_t ovfl_ctrl;
5125         unsigned int i, has_smpl;
5126         int must_notify = 0;
5127 
5128         if (unlikely(ctx->ctx_state == PFM_CTX_ZOMBIE)) goto stop_monitoring;
5129 
5130         /*
5131          * sanity test. Should never happen
5132          */
5133         if (unlikely((pmc0 & 0x1) == 0)) goto sanity_check;
5134 
5135         tstamp   = ia64_get_itc();
5136         mask     = pmc0 >> PMU_FIRST_COUNTER;
5137         ovfl_val = pmu_conf->ovfl_val;
5138         has_smpl = CTX_HAS_SMPL(ctx);
5139 
5140         DPRINT_ovfl(("pmc0=0x%lx pid=%d iip=0x%lx, %s "
5141                      "used_pmds=0x%lx\n",
5142                         pmc0,
5143                         task ? task_pid_nr(task): -1,
5144                         (regs ? regs->cr_iip : 0),
5145                         CTX_OVFL_NOBLOCK(ctx) ? "nonblocking" : "blocking",
5146                         ctx->ctx_used_pmds[0]));
5147 
5148 
5149         /*
5150          * first we update the virtual counters
5151          * assume there was a prior ia64_srlz_d() issued
5152          */
5153         for (i = PMU_FIRST_COUNTER; mask ; i++, mask >>= 1) {
5154 
5155                 /* skip pmd which did not overflow */
5156                 if ((mask & 0x1) == 0) continue;
5157 
5158                 /*
5159                  * Note that the pmd is not necessarily 0 at this point as qualified events
5160                  * may have happened before the PMU was frozen. The residual count is not
5161                  * taken into consideration here but will be with any read of the pmd via
5162                  * pfm_read_pmds().
5163                  */
5164                 old_val              = new_val = ctx->ctx_pmds[i].val;
5165                 new_val             += 1 + ovfl_val;
5166                 ctx->ctx_pmds[i].val = new_val;
5167 
5168                 /*
5169                  * check for overflow condition
5170                  */
5171                 if (likely(old_val > new_val)) {
5172                         ovfl_pmds |= 1UL << i;
5173                         if (PMC_OVFL_NOTIFY(ctx, i)) ovfl_notify |= 1UL << i;
5174                 }
5175 
5176                 DPRINT_ovfl(("ctx_pmd[%d].val=0x%lx old_val=0x%lx pmd=0x%lx ovfl_pmds=0x%lx ovfl_notify=0x%lx\n",
5177                         i,
5178                         new_val,
5179                         old_val,
5180                         ia64_get_pmd(i) & ovfl_val,
5181                         ovfl_pmds,
5182                         ovfl_notify));
5183         }
5184 
5185         /*
5186          * there was no 64-bit overflow, nothing else to do
5187          */
5188         if (ovfl_pmds == 0UL) return;
5189 
5190         /* 
5191          * reset all control bits
5192          */
5193         ovfl_ctrl.val = 0;
5194         reset_pmds    = 0UL;
5195 
5196         /*
5197          * if a sampling format module exists, then we "cache" the overflow by 
5198          * calling the module's handler() routine.
5199          */
5200         if (has_smpl) {
5201                 unsigned long start_cycles, end_cycles;
5202                 unsigned long pmd_mask;
5203                 int j, k, ret = 0;
5204                 int this_cpu = smp_processor_id();
5205 
5206                 pmd_mask = ovfl_pmds >> PMU_FIRST_COUNTER;
5207                 ovfl_arg = &ctx->ctx_ovfl_arg;
5208 
5209                 prefetch(ctx->ctx_smpl_hdr);
5210 
5211                 for(i=PMU_FIRST_COUNTER; pmd_mask && ret == 0; i++, pmd_mask >>=1) {
5212 
5213                         mask = 1UL << i;
5214 
5215                         if ((pmd_mask & 0x1) == 0) continue;
5216 
5217                         ovfl_arg->ovfl_pmd      = (unsigned char )i;
5218                         ovfl_arg->ovfl_notify   = ovfl_notify & mask ? 1 : 0;
5219                         ovfl_arg->active_set    = 0;
5220                         ovfl_arg->ovfl_ctrl.val = 0; /* module must fill in all fields */
5221                         ovfl_arg->smpl_pmds[0]  = smpl_pmds = ctx->ctx_pmds[i].smpl_pmds[0];
5222 
5223                         ovfl_arg->pmd_value      = ctx->ctx_pmds[i].val;
5224                         ovfl_arg->pmd_last_reset = ctx->ctx_pmds[i].lval;
5225                         ovfl_arg->pmd_eventid    = ctx->ctx_pmds[i].eventid;
5226 
5227                         /*
5228                          * copy values of pmds of interest. Sampling format may copy them
5229                          * into sampling buffer.
5230                          */
5231                         if (smpl_pmds) {
5232                                 for(j=0, k=0; smpl_pmds; j++, smpl_pmds >>=1) {
5233                                         if ((smpl_pmds & 0x1) == 0) continue;
5234                                         ovfl_arg->smpl_pmds_values[k++] = PMD_IS_COUNTING(j) ?  pfm_read_soft_counter(ctx, j) : ia64_get_pmd(j);
5235                                         DPRINT_ovfl(("smpl_pmd[%d]=pmd%u=0x%lx\n", k-1, j, ovfl_arg->smpl_pmds_values[k-1]));
5236                                 }
5237                         }
5238 
5239                         pfm_stats[this_cpu].pfm_smpl_handler_calls++;
5240 
5241                         start_cycles = ia64_get_itc();
5242 
5243                         /*
5244                          * call custom buffer format record (handler) routine
5245                          */
5246                         ret = (*ctx->ctx_buf_fmt->fmt_handler)(task, ctx->ctx_smpl_hdr, ovfl_arg, regs, tstamp);
5247 
5248                         end_cycles = ia64_get_itc();
5249 
5250                         /*
5251                          * For those controls, we take the union because they have
5252                          * an all or nothing behavior.
5253                          */
5254                         ovfl_ctrl.bits.notify_user     |= ovfl_arg->ovfl_ctrl.bits.notify_user;
5255                         ovfl_ctrl.bits.block_task      |= ovfl_arg->ovfl_ctrl.bits.block_task;
5256                         ovfl_ctrl.bits.mask_monitoring |= ovfl_arg->ovfl_ctrl.bits.mask_monitoring;
5257                         /*
5258                          * build the bitmask of pmds to reset now
5259                          */
5260                         if (ovfl_arg->ovfl_ctrl.bits.reset_ovfl_pmds) reset_pmds |= mask;
5261 
5262                         pfm_stats[this_cpu].pfm_smpl_handler_cycles += end_cycles - start_cycles;
5263                 }
5264                 /*
5265                  * when the module cannot handle the rest of the overflows, we abort right here
5266                  */
5267                 if (ret && pmd_mask) {
5268                         DPRINT(("handler aborts leftover ovfl_pmds=0x%lx\n",
5269                                 pmd_mask<<PMU_FIRST_COUNTER));
5270                 }
5271                 /*
5272                  * remove the pmds we reset now from the set of pmds to reset in pfm_restart()
5273                  */
5274                 ovfl_pmds &= ~reset_pmds;
5275         } else {
5276                 /*
5277                  * when no sampling module is used, then the default
5278                  * is to notify on overflow if requested by user
5279                  */
5280                 ovfl_ctrl.bits.notify_user     = ovfl_notify ? 1 : 0;
5281                 ovfl_ctrl.bits.block_task      = ovfl_notify ? 1 : 0;
5282                 ovfl_ctrl.bits.mask_monitoring = ovfl_notify ? 1 : 0; /* XXX: change for saturation */
5283                 ovfl_ctrl.bits.reset_ovfl_pmds = ovfl_notify ? 0 : 1;
5284                 /*
5285                  * if needed, we reset all overflowed pmds
5286                  */
5287                 if (ovfl_notify == 0) reset_pmds = ovfl_pmds;
5288         }
5289 
5290         DPRINT_ovfl(("ovfl_pmds=0x%lx reset_pmds=0x%lx\n", ovfl_pmds, reset_pmds));
5291 
5292         /*
5293          * reset the requested PMD registers using the short reset values
5294          */
5295         if (reset_pmds) {
5296                 unsigned long bm = reset_pmds;
5297                 pfm_reset_regs(ctx, &bm, PFM_PMD_SHORT_RESET);
5298         }
5299 
5300         if (ovfl_notify && ovfl_ctrl.bits.notify_user) {
5301                 /*
5302                  * keep track of what to reset when unblocking
5303                  */
5304                 ctx->ctx_ovfl_regs[0] = ovfl_pmds;
5305 
5306                 /*
5307                  * check for blocking context 
5308                  */
5309                 if (CTX_OVFL_NOBLOCK(ctx) == 0 && ovfl_ctrl.bits.block_task) {
5310 
5311                         ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_BLOCK;
5312 
5313                         /*
5314                          * set the perfmon specific checking pending work for the task
5315                          */
5316                         PFM_SET_WORK_PENDING(task, 1);
5317 
5318                         /*
5319                          * when coming from ctxsw, current still points to the
5320                          * previous task, therefore we must work with task and not current.
5321                          */
5322                         set_notify_resume(task);
5323                 }
5324                 /*
5325                  * defer until state is changed (shorten spin window). the context is locked
5326                  * anyway, so the signal receiver would come spin for nothing.
5327                  */
5328                 must_notify = 1;
5329         }
5330 
5331         DPRINT_ovfl(("owner [%d] pending=%ld reason=%u ovfl_pmds=0x%lx ovfl_notify=0x%lx masked=%d\n",
5332                         GET_PMU_OWNER() ? task_pid_nr(GET_PMU_OWNER()) : -1,
5333                         PFM_GET_WORK_PENDING(task),
5334                         ctx->ctx_fl_trap_reason,
5335                         ovfl_pmds,
5336                         ovfl_notify,
5337                         ovfl_ctrl.bits.mask_monitoring ? 1 : 0));
5338         /*
5339          * in case monitoring must be stopped, we toggle the psr bits
5340          */
5341         if (ovfl_ctrl.bits.mask_monitoring) {
5342                 pfm_mask_monitoring(task);
5343                 ctx->ctx_state = PFM_CTX_MASKED;
5344                 ctx->ctx_fl_can_restart = 1;
5345         }
5346 
5347         /*
5348          * send notification now
5349          */
5350         if (must_notify) pfm_ovfl_notify_user(ctx, ovfl_notify);
5351 
5352         return;
5353 
5354 sanity_check:
5355         printk(KERN_ERR "perfmon: CPU%d overflow handler [%d] pmc0=0x%lx\n",
5356                         smp_processor_id(),
5357                         task ? task_pid_nr(task) : -1,
5358                         pmc0);
5359         return;
5360 
5361 stop_monitoring:
5362         /*
5363          * in SMP, zombie context is never restored but reclaimed in pfm_load_regs().
5364          * Moreover, zombies are also reclaimed in pfm_save_regs(). Therefore we can
5365          * come here as zombie only if the task is the current task. In which case, we
5366          * can access the PMU  hardware directly.
5367          *
5368          * Note that zombies do have PM_VALID set. So here we do the minimal.
5369          *
5370          * In case the context was zombified it could not be reclaimed at the time
5371          * the monitoring program exited. At this point, the PMU reservation has been
5372          * returned, the sampiing buffer has been freed. We must convert this call
5373          * into a spurious interrupt. However, we must also avoid infinite overflows
5374          * by stopping monitoring for this task. We can only come here for a per-task
5375          * context. All we need to do is to stop monitoring using the psr bits which
5376          * are always task private. By re-enabling secure montioring, we ensure that
5377          * the monitored task will not be able to re-activate monitoring.
5378          * The task will eventually be context switched out, at which point the context
5379          * will be reclaimed (that includes releasing ownership of the PMU).
5380          *
5381          * So there might be a window of time where the number of per-task session is zero
5382          * yet one PMU might have a owner and get at most one overflow interrupt for a zombie
5383          * context. This is safe because if a per-task session comes in, it will push this one
5384          * out and by the virtue on pfm_save_regs(), this one will disappear. If a system wide
5385          * session is force on that CPU, given that we use task pinning, pfm_save_regs() will
5386          * also push our zombie context out.
5387          *
5388          * Overall pretty hairy stuff....
5389          */
5390         DPRINT(("ctx is zombie for [%d], converted to spurious\n", task ? task_pid_nr(task): -1));
5391         pfm_clear_psr_up();
5392         ia64_psr(regs)->up = 0;
5393         ia64_psr(regs)->sp = 1;
5394         return;
5395 }
5396 
5397 static int
5398 pfm_do_interrupt_handler(void *arg, struct pt_regs *regs)
5399 {
5400         struct task_struct *task;
5401         pfm_context_t *ctx;
5402         unsigned long flags;
5403         u64 pmc0;
5404         int this_cpu = smp_processor_id();
5405         int retval = 0;
5406 
5407         pfm_stats[this_cpu].pfm_ovfl_intr_count++;
5408 
5409         /*
5410          * srlz.d done before arriving here
5411          */
5412         pmc0 = ia64_get_pmc(0);
5413 
5414         task = GET_PMU_OWNER();
5415         ctx  = GET_PMU_CTX();
5416 
5417         /*
5418          * if we have some pending bits set
5419          * assumes : if any PMC0.bit[63-1] is set, then PMC0.fr = 1
5420          */
5421         if (PMC0_HAS_OVFL(pmc0) && task) {
5422                 /*
5423                  * we assume that pmc0.fr is always set here
5424                  */
5425 
5426                 /* sanity check */
5427                 if (!ctx) goto report_spurious1;
5428 
5429                 if (ctx->ctx_fl_system == 0 && (task->thread.flags & IA64_THREAD_PM_VALID) == 0) 
5430                         goto report_spurious2;
5431 
5432                 PROTECT_CTX_NOPRINT(ctx, flags);
5433 
5434                 pfm_overflow_handler(task, ctx, pmc0, regs);
5435 
5436                 UNPROTECT_CTX_NOPRINT(ctx, flags);
5437 
5438         } else {
5439                 pfm_stats[this_cpu].pfm_spurious_ovfl_intr_count++;
5440                 retval = -1;
5441         }
5442         /*
5443          * keep it unfrozen at all times
5444          */
5445         pfm_unfreeze_pmu();
5446 
5447         return retval;
5448 
5449 report_spurious1:
5450         printk(KERN_INFO "perfmon: spurious overflow interrupt on CPU%d: process %d has no PFM context\n",
5451                 this_cpu, task_pid_nr(task));
5452         pfm_unfreeze_pmu();
5453         return -1;
5454 report_spurious2:
5455         printk(KERN_INFO "perfmon: spurious overflow interrupt on CPU%d: process %d, invalid flag\n", 
5456                 this_cpu, 
5457                 task_pid_nr(task));
5458         pfm_unfreeze_pmu();
5459         return -1;
5460 }
5461 
5462 static irqreturn_t
5463 pfm_interrupt_handler(int irq, void *arg)
5464 {
5465         unsigned long start_cycles, total_cycles;
5466         unsigned long min, max;
5467         int this_cpu;
5468         int ret;
5469         struct pt_regs *regs = get_irq_regs();
5470 
5471         this_cpu = get_cpu();
5472         if (likely(!pfm_alt_intr_handler)) {
5473                 min = pfm_stats[this_cpu].pfm_ovfl_intr_cycles_min;
5474                 max = pfm_stats[this_cpu].pfm_ovfl_intr_cycles_max;
5475 
5476                 start_cycles = ia64_get_itc();
5477 
5478                 ret = pfm_do_interrupt_handler(arg, regs);
5479 
5480                 total_cycles = ia64_get_itc();
5481 
5482                 /*
5483                  * don't measure spurious interrupts
5484                  */
5485                 if (likely(ret == 0)) {
5486                         total_cycles -= start_cycles;
5487 
5488                         if (total_cycles < min) pfm_stats[this_cpu].pfm_ovfl_intr_cycles_min = total_cycles;
5489                         if (total_cycles > max) pfm_stats[this_cpu].pfm_ovfl_intr_cycles_max = total_cycles;
5490 
5491                         pfm_stats[this_cpu].pfm_ovfl_intr_cycles += total_cycles;
5492                 }
5493         }
5494         else {
5495                 (*pfm_alt_intr_handler->handler)(irq, arg, regs);
5496         }
5497 
5498         put_cpu();
5499         return IRQ_HANDLED;
5500 }
5501 
5502 /*
5503  * /proc/perfmon interface, for debug only
5504  */
5505 
5506 #define PFM_PROC_SHOW_HEADER    ((void *)(long)nr_cpu_ids+1)
5507 
5508 static void *
5509 pfm_proc_start(struct seq_file *m, loff_t *pos)
5510 {
5511         if (*pos == 0) {
5512                 return PFM_PROC_SHOW_HEADER;
5513         }
5514 
5515         while (*pos <= nr_cpu_ids) {
5516                 if (cpu_online(*pos - 1)) {
5517                         return (void *)*pos;
5518                 }
5519                 ++*pos;
5520         }
5521         return NULL;
5522 }
5523 
5524 static void *
5525 pfm_proc_next(struct seq_file *m, void *v, loff_t *pos)
5526 {
5527         ++*pos;
5528         return pfm_proc_start(m, pos);
5529 }
5530 
5531 static void
5532 pfm_proc_stop(struct seq_file *m, void *v)
5533 {
5534 }
5535 
5536 static void
5537 pfm_proc_show_header(struct seq_file *m)
5538 {
5539         struct list_head * pos;
5540         pfm_buffer_fmt_t * entry;
5541         unsigned long flags;
5542 
5543         seq_printf(m,
5544                 "perfmon version           : %u.%u\n"
5545                 "model                     : %s\n"
5546                 "fastctxsw                 : %s\n"
5547                 "expert mode               : %s\n"
5548                 "ovfl_mask                 : 0x%lx\n"
5549                 "PMU flags                 : 0x%x\n",
5550                 PFM_VERSION_MAJ, PFM_VERSION_MIN,
5551                 pmu_conf->pmu_name,
5552                 pfm_sysctl.fastctxsw > 0 ? "Yes": "No",
5553                 pfm_sysctl.expert_mode > 0 ? "Yes": "No",
5554                 pmu_conf->ovfl_val,
5555                 pmu_conf->flags);
5556 
5557         LOCK_PFS(flags);
5558 
5559         seq_printf(m,
5560                 "proc_sessions             : %u\n"
5561                 "sys_sessions              : %u\n"
5562                 "sys_use_dbregs            : %u\n"
5563                 "ptrace_use_dbregs         : %u\n",
5564                 pfm_sessions.pfs_task_sessions,
5565                 pfm_sessions.pfs_sys_sessions,
5566                 pfm_sessions.pfs_sys_use_dbregs,
5567                 pfm_sessions.pfs_ptrace_use_dbregs);
5568 
5569         UNLOCK_PFS(flags);
5570 
5571         spin_lock(&pfm_buffer_fmt_lock);
5572 
5573         list_for_each(pos, &pfm_buffer_fmt_list) {
5574                 entry = list_entry(pos, pfm_buffer_fmt_t, fmt_list);
5575                 seq_printf(m, "format                    : %16phD %s\n",
5576                            entry->fmt_uuid, entry->fmt_name);
5577         }
5578         spin_unlock(&pfm_buffer_fmt_lock);
5579 
5580 }
5581 
5582 static int
5583 pfm_proc_show(struct seq_file *m, void *v)
5584 {
5585         unsigned long psr;
5586         unsigned int i;
5587         int cpu;
5588 
5589         if (v == PFM_PROC_SHOW_HEADER) {
5590                 pfm_proc_show_header(m);
5591                 return 0;
5592         }
5593 
5594         /* show info for CPU (v - 1) */
5595 
5596         cpu = (long)v - 1;
5597         seq_printf(m,
5598                 "CPU%-2d overflow intrs      : %lu\n"
5599                 "CPU%-2d overflow cycles     : %lu\n"
5600                 "CPU%-2d overflow min        : %lu\n"
5601                 "CPU%-2d overflow max        : %lu\n"
5602                 "CPU%-2d smpl handler calls  : %lu\n"
5603                 "CPU%-2d smpl handler cycles : %lu\n"
5604                 "CPU%-2d spurious intrs      : %lu\n"
5605                 "CPU%-2d replay   intrs      : %lu\n"
5606                 "CPU%-2d syst_wide           : %d\n"
5607                 "CPU%-2d dcr_pp              : %d\n"
5608                 "CPU%-2d exclude idle        : %d\n"
5609                 "CPU%-2d owner               : %d\n"
5610                 "CPU%-2d context             : %p\n"
5611                 "CPU%-2d activations         : %lu\n",
5612                 cpu, pfm_stats[cpu].pfm_ovfl_intr_count,
5613                 cpu, pfm_stats[cpu].pfm_ovfl_intr_cycles,
5614                 cpu, pfm_stats[cpu].pfm_ovfl_intr_cycles_min,
5615                 cpu, pfm_stats[cpu].pfm_ovfl_intr_cycles_max,
5616                 cpu, pfm_stats[cpu].pfm_smpl_handler_calls,
5617                 cpu, pfm_stats[cpu].pfm_smpl_handler_cycles,
5618                 cpu, pfm_stats[cpu].pfm_spurious_ovfl_intr_count,
5619                 cpu, pfm_stats[cpu].pfm_replay_ovfl_intr_count,
5620                 cpu, pfm_get_cpu_data(pfm_syst_info, cpu) & PFM_CPUINFO_SYST_WIDE ? 1 : 0,
5621                 cpu, pfm_get_cpu_data(pfm_syst_info, cpu) & PFM_CPUINFO_DCR_PP ? 1 : 0,
5622                 cpu, pfm_get_cpu_data(pfm_syst_info, cpu) & PFM_CPUINFO_EXCL_IDLE ? 1 : 0,
5623                 cpu, pfm_get_cpu_data(pmu_owner, cpu) ? pfm_get_cpu_data(pmu_owner, cpu)->pid: -1,
5624                 cpu, pfm_get_cpu_data(pmu_ctx, cpu),
5625                 cpu, pfm_get_cpu_data(pmu_activation_number, cpu));
5626 
5627         if (num_online_cpus() == 1 && pfm_sysctl.debug > 0) {
5628 
5629                 psr = pfm_get_psr();
5630 
5631                 ia64_srlz_d();
5632 
5633                 seq_printf(m, 
5634                         "CPU%-2d psr                 : 0x%lx\n"
5635                         "CPU%-2d pmc0                : 0x%lx\n", 
5636                         cpu, psr,
5637                         cpu, ia64_get_pmc(0));
5638 
5639                 for (i=0; PMC_IS_LAST(i) == 0;  i++) {
5640                         if (PMC_IS_COUNTING(i) == 0) continue;
5641                         seq_printf(m, 
5642                                 "CPU%-2d pmc%u                : 0x%lx\n"
5643                                 "CPU%-2d pmd%u                : 0x%lx\n", 
5644                                 cpu, i, ia64_get_pmc(i),
5645                                 cpu, i, ia64_get_pmd(i));
5646                 }
5647         }
5648         return 0;
5649 }
5650 
5651 const struct seq_operations pfm_seq_ops = {
5652         .start =        pfm_proc_start,
5653         .next =         pfm_proc_next,
5654         .stop =         pfm_proc_stop,
5655         .show =         pfm_proc_show
5656 };
5657 
5658 /*
5659  * we come here as soon as local_cpu_data->pfm_syst_wide is set. this happens
5660  * during pfm_enable() hence before pfm_start(). We cannot assume monitoring
5661  * is active or inactive based on mode. We must rely on the value in
5662  * local_cpu_data->pfm_syst_info
5663  */
5664 void
5665 pfm_syst_wide_update_task(struct task_struct *task, unsigned long info, int is_ctxswin)
5666 {
5667         struct pt_regs *regs;
5668         unsigned long dcr;
5669         unsigned long dcr_pp;
5670 
5671         dcr_pp = info & PFM_CPUINFO_DCR_PP ? 1 : 0;
5672 
5673         /*
5674          * pid 0 is guaranteed to be the idle task. There is one such task with pid 0
5675          * on every CPU, so we can rely on the pid to identify the idle task.
5676          */
5677         if ((info & PFM_CPUINFO_EXCL_IDLE) == 0 || task->pid) {
5678                 regs = task_pt_regs(task);
5679                 ia64_psr(regs)->pp = is_ctxswin ? dcr_pp : 0;
5680                 return;
5681         }
5682         /*
5683          * if monitoring has started
5684          */
5685         if (dcr_pp) {
5686                 dcr = ia64_getreg(_IA64_REG_CR_DCR);
5687                 /*
5688                  * context switching in?
5689                  */
5690                 if (is_ctxswin) {
5691                         /* mask monitoring for the idle task */
5692                         ia64_setreg(_IA64_REG_CR_DCR, dcr & ~IA64_DCR_PP);
5693                         pfm_clear_psr_pp();
5694                         ia64_srlz_i();
5695                         return;
5696                 }
5697                 /*
5698                  * context switching out
5699                  * restore monitoring for next task
5700                  *
5701                  * Due to inlining this odd if-then-else construction generates
5702                  * better code.
5703                  */
5704                 ia64_setreg(_IA64_REG_CR_DCR, dcr |IA64_DCR_PP);
5705                 pfm_set_psr_pp();
5706                 ia64_srlz_i();
5707         }
5708 }
5709 
5710 #ifdef CONFIG_SMP
5711 
5712 static void
5713 pfm_force_cleanup(pfm_context_t *ctx, struct pt_regs *regs)
5714 {
5715         struct task_struct *task = ctx->ctx_task;
5716 
5717         ia64_psr(regs)->up = 0;
5718         ia64_psr(regs)->sp = 1;
5719 
5720         if (GET_PMU_OWNER() == task) {
5721                 DPRINT(("cleared ownership for [%d]\n",
5722                                         task_pid_nr(ctx->ctx_task)));
5723                 SET_PMU_OWNER(NULL, NULL);
5724         }
5725 
5726         /*
5727          * disconnect the task from the context and vice-versa
5728          */
5729         PFM_SET_WORK_PENDING(task, 0);
5730 
5731         task->thread.pfm_context  = NULL;
5732         task->thread.flags       &= ~IA64_THREAD_PM_VALID;
5733 
5734         DPRINT(("force cleanup for [%d]\n",  task_pid_nr(task)));
5735 }
5736 
5737 
5738 /*
5739  * in 2.6, interrupts are masked when we come here and the runqueue lock is held
5740  */
5741 void
5742 pfm_save_regs(struct task_struct *task)
5743 {
5744         pfm_context_t *ctx;
5745         unsigned long flags;
5746         u64 psr;
5747 
5748 
5749         ctx = PFM_GET_CTX(task);
5750         if (ctx == NULL) return;
5751 
5752         /*
5753          * we always come here with interrupts ALREADY disabled by
5754          * the scheduler. So we simply need to protect against concurrent
5755          * access, not CPU concurrency.
5756          */
5757         flags = pfm_protect_ctx_ctxsw(ctx);
5758 
5759         if (ctx->ctx_state == PFM_CTX_ZOMBIE) {
5760                 struct pt_regs *regs = task_pt_regs(task);
5761 
5762                 pfm_clear_psr_up();
5763 
5764                 pfm_force_cleanup(ctx, regs);
5765 
5766                 BUG_ON(ctx->ctx_smpl_hdr);
5767 
5768                 pfm_unprotect_ctx_ctxsw(ctx, flags);
5769 
5770                 pfm_context_free(ctx);
5771                 return;
5772         }
5773 
5774         /*
5775          * save current PSR: needed because we modify it
5776          */
5777         ia64_srlz_d();
5778         psr = pfm_get_psr();
5779 
5780         BUG_ON(psr & (IA64_PSR_I));
5781 
5782         /*
5783          * stop monitoring:
5784          * This is the last instruction which may generate an overflow
5785          *
5786          * We do not need to set psr.sp because, it is irrelevant in kernel.
5787          * It will be restored from ipsr when going back to user level
5788          */
5789         pfm_clear_psr_up();
5790 
5791         /*
5792          * keep a copy of psr.up (for reload)
5793          */
5794         ctx->ctx_saved_psr_up = psr & IA64_PSR_UP;
5795 
5796         /*
5797          * release ownership of this PMU.
5798          * PM interrupts are masked, so nothing
5799          * can happen.
5800          */
5801         SET_PMU_OWNER(NULL, NULL);
5802 
5803         /*
5804          * we systematically save the PMD as we have no
5805          * guarantee we will be schedule at that same
5806          * CPU again.
5807          */
5808         pfm_save_pmds(ctx->th_pmds, ctx->ctx_used_pmds[0]);
5809 
5810         /*
5811          * save pmc0 ia64_srlz_d() done in pfm_save_pmds()
5812          * we will need it on the restore path to check
5813          * for pending overflow.
5814          */
5815         ctx->th_pmcs[0] = ia64_get_pmc(0);
5816 
5817         /*
5818          * unfreeze PMU if had pending overflows
5819          */
5820         if (ctx->th_pmcs[0] & ~0x1UL) pfm_unfreeze_pmu();
5821 
5822         /*
5823          * finally, allow context access.
5824          * interrupts will still be masked after this call.
5825          */
5826         pfm_unprotect_ctx_ctxsw(ctx, flags);
5827 }
5828 
5829 #else /* !CONFIG_SMP */
5830 void
5831 pfm_save_regs(struct task_struct *task)
5832 {
5833         pfm_context_t *ctx;
5834         u64 psr;
5835 
5836         ctx = PFM_GET_CTX(task);
5837         if (ctx == NULL) return;
5838 
5839         /*
5840          * save current PSR: needed because we modify it
5841          */
5842         psr = pfm_get_psr();
5843 
5844         BUG_ON(psr & (IA64_PSR_I));
5845 
5846         /*
5847          * stop monitoring:
5848          * This is the last instruction which may generate an overflow
5849          *
5850          * We do not need to set psr.sp because, it is irrelevant in kernel.
5851          * It will be restored from ipsr when going back to user level
5852          */
5853         pfm_clear_psr_up();
5854 
5855         /*
5856          * keep a copy of psr.up (for reload)
5857          */
5858         ctx->ctx_saved_psr_up = psr & IA64_PSR_UP;
5859 }
5860 
5861 static void
5862 pfm_lazy_save_regs (struct task_struct *task)
5863 {
5864         pfm_context_t *ctx;
5865         unsigned long flags;
5866 
5867         { u64 psr  = pfm_get_psr();
5868           BUG_ON(psr & IA64_PSR_UP);
5869         }
5870 
5871         ctx = PFM_GET_CTX(task);
5872 
5873         /*
5874          * we need to mask PMU overflow here to
5875          * make sure that we maintain pmc0 until
5876          * we save it. overflow interrupts are
5877          * treated as spurious if there is no
5878          * owner.
5879          *
5880          * XXX: I don't think this is necessary
5881          */
5882         PROTECT_CTX(ctx,flags);
5883 
5884         /*
5885          * release ownership of this PMU.
5886          * must be done before we save the registers.
5887          *
5888          * after this call any PMU interrupt is treated
5889          * as spurious.
5890          */
5891         SET_PMU_OWNER(NULL, NULL);
5892 
5893         /*
5894          * save all the pmds we use
5895          */
5896         pfm_save_pmds(ctx->th_pmds, ctx->ctx_used_pmds[0]);
5897 
5898         /*
5899          * save pmc0 ia64_srlz_d() done in pfm_save_pmds()
5900          * it is needed to check for pended overflow
5901          * on the restore path
5902          */
5903         ctx->th_pmcs[0] = ia64_get_pmc(0);
5904 
5905         /*
5906          * unfreeze PMU if had pending overflows
5907          */
5908         if (ctx->th_pmcs[0] & ~0x1UL) pfm_unfreeze_pmu();
5909 
5910         /*
5911          * now get can unmask PMU interrupts, they will
5912          * be treated as purely spurious and we will not
5913          * lose any information
5914          */
5915         UNPROTECT_CTX(ctx,flags);
5916 }
5917 #endif /* CONFIG_SMP */
5918 
5919 #ifdef CONFIG_SMP
5920 /*
5921  * in 2.6, interrupts are masked when we come here and the runqueue lock is held
5922  */
5923 void
5924 pfm_load_regs (struct task_struct *task)
5925 {
5926         pfm_context_t *ctx;
5927         unsigned long pmc_mask = 0UL, pmd_mask = 0UL;
5928         unsigned long flags;
5929         u64 psr, psr_up;
5930         int need_irq_resend;
5931 
5932         ctx = PFM_GET_CTX(task);
5933         if (unlikely(ctx == NULL)) return;
5934 
5935         BUG_ON(GET_PMU_OWNER());
5936 
5937         /*
5938          * possible on unload
5939          */
5940         if (unlikely((task->thread.flags & IA64_THREAD_PM_VALID) == 0)) return;
5941 
5942         /*
5943          * we always come here with interrupts ALREADY disabled by
5944          * the scheduler. So we simply need to protect against concurrent
5945          * access, not CPU concurrency.
5946          */
5947         flags = pfm_protect_ctx_ctxsw(ctx);
5948         psr   = pfm_get_psr();
5949 
5950         need_irq_resend = pmu_conf->flags & PFM_PMU_IRQ_RESEND;
5951 
5952         BUG_ON(psr & (IA64_PSR_UP|IA64_PSR_PP));
5953         BUG_ON(psr & IA64_PSR_I);
5954 
5955         if (unlikely(ctx->ctx_state == PFM_CTX_ZOMBIE)) {
5956                 struct pt_regs *regs = task_pt_regs(task);
5957 
5958                 BUG_ON(ctx->ctx_smpl_hdr);
5959 
5960                 pfm_force_cleanup(ctx, regs);
5961 
5962                 pfm_unprotect_ctx_ctxsw(ctx, flags);
5963 
5964                 /*
5965                  * this one (kmalloc'ed) is fine with interrupts disabled
5966                  */
5967                 pfm_context_free(ctx);
5968 
5969                 return;
5970         }
5971 
5972         /*
5973          * we restore ALL the debug registers to avoid picking up
5974          * stale state.
5975          */
5976         if (ctx->ctx_fl_using_dbreg) {
5977                 pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
5978                 pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
5979         }
5980         /*
5981          * retrieve saved psr.up
5982          */
5983         psr_up = ctx->ctx_saved_psr_up;
5984 
5985         /*
5986          * if we were the last user of the PMU on that CPU,
5987          * then nothing to do except restore psr
5988          */
5989         if (GET_LAST_CPU(ctx) == smp_processor_id() && ctx->ctx_last_activation == GET_ACTIVATION()) {
5990 
5991                 /*
5992                  * retrieve partial reload masks (due to user modifications)
5993                  */
5994                 pmc_mask = ctx->ctx_reload_pmcs[0];
5995                 pmd_mask = ctx->ctx_reload_pmds[0];
5996 
5997         } else {
5998                 /*
5999                  * To avoid leaking information to the user level when psr.sp=0,
6000                  * we must reload ALL implemented pmds (even the ones we don't use).
6001                  * In the kernel we only allow PFM_READ_PMDS on registers which
6002                  * we initialized or requested (sampling) so there is no risk there.
6003                  */
6004                 pmd_mask = pfm_sysctl.fastctxsw ?  ctx->ctx_used_pmds[0] : ctx->ctx_all_pmds[0];
6005 
6006                 /*
6007                  * ALL accessible PMCs are systematically reloaded, unused registers
6008                  * get their default (from pfm_reset_pmu_state()) values to avoid picking
6009                  * up stale configuration.
6010                  *
6011                  * PMC0 is never in the mask. It is always restored separately.
6012                  */
6013                 pmc_mask = ctx->ctx_all_pmcs[0];
6014         }
6015         /*
6016          * when context is MASKED, we will restore PMC with plm=0
6017          * and PMD with stale information, but that's ok, nothing
6018          * will be captured.
6019          *
6020          * XXX: optimize here
6021          */
6022         if (pmd_mask) pfm_restore_pmds(ctx->th_pmds, pmd_mask);
6023         if (pmc_mask) pfm_restore_pmcs(ctx->th_pmcs, pmc_mask);
6024 
6025         /*
6026          * check for pending overflow at the time the state
6027          * was saved.
6028          */
6029         if (unlikely(PMC0_HAS_OVFL(ctx->th_pmcs[0]))) {
6030                 /*
6031                  * reload pmc0 with the overflow information
6032                  * On McKinley PMU, this will trigger a PMU interrupt
6033                  */
6034                 ia64_set_pmc(0, ctx->th_pmcs[0]);
6035                 ia64_srlz_d();
6036                 ctx->th_pmcs[0] = 0UL;
6037 
6038                 /*
6039                  * will replay the PMU interrupt
6040                  */
6041                 if (need_irq_resend) ia64_resend_irq(IA64_PERFMON_VECTOR);
6042 
6043                 pfm_stats[smp_processor_id()].pfm_replay_ovfl_intr_count++;
6044         }
6045 
6046         /*
6047          * we just did a reload, so we reset the partial reload fields
6048          */
6049         ctx->ctx_reload_pmcs[0] = 0UL;
6050         ctx->ctx_reload_pmds[0] = 0UL;
6051 
6052         SET_LAST_CPU(ctx, smp_processor_id());
6053 
6054         /*
6055          * dump activation value for this PMU
6056          */
6057         INC_ACTIVATION();
6058         /*
6059          * record current activation for this context
6060          */
6061         SET_ACTIVATION(ctx);
6062 
6063         /*
6064          * establish new ownership. 
6065          */
6066         SET_PMU_OWNER(task, ctx);
6067 
6068         /*
6069          * restore the psr.up bit. measurement
6070          * is active again.
6071          * no PMU interrupt can happen at this point
6072          * because we still have interrupts disabled.
6073          */
6074         if (likely(psr_up)) pfm_set_psr_up();
6075 
6076         /*
6077          * allow concurrent access to context
6078          */
6079         pfm_unprotect_ctx_ctxsw(ctx, flags);
6080 }
6081 #else /*  !CONFIG_SMP */
6082 /*
6083  * reload PMU state for UP kernels
6084  * in 2.5 we come here with interrupts disabled
6085  */
6086 void
6087 pfm_load_regs (struct task_struct *task)
6088 {
6089         pfm_context_t *ctx;
6090         struct task_struct *owner;
6091         unsigned long pmd_mask, pmc_mask;
6092         u64 psr, psr_up;
6093         int need_irq_resend;
6094 
6095         owner = GET_PMU_OWNER();
6096         ctx   = PFM_GET_CTX(task);
6097         psr   = pfm_get_psr();
6098 
6099         BUG_ON(psr & (IA64_PSR_UP|IA64_PSR_PP));
6100         BUG_ON(psr & IA64_PSR_I);
6101 
6102         /*
6103          * we restore ALL the debug registers to avoid picking up
6104          * stale state.
6105          *
6106          * This must be done even when the task is still the owner
6107          * as the registers may have been modified via ptrace()
6108          * (not perfmon) by the previous task.
6109          */
6110         if (ctx->ctx_fl_using_dbreg) {
6111                 pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
6112                 pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
6113         }
6114 
6115         /*
6116          * retrieved saved psr.up
6117          */
6118         psr_up = ctx->ctx_saved_psr_up;
6119         need_irq_resend = pmu_conf->flags & PFM_PMU_IRQ_RESEND;
6120 
6121         /*
6122          * short path, our state is still there, just
6123          * need to restore psr and we go
6124          *
6125          * we do not touch either PMC nor PMD. the psr is not touched
6126          * by the overflow_handler. So we are safe w.r.t. to interrupt
6127          * concurrency even without interrupt masking.
6128          */
6129         if (likely(owner == task)) {
6130                 if (likely(psr_up)) pfm_set_psr_up();
6131                 return;
6132         }
6133 
6134         /*
6135          * someone else is still using the PMU, first push it out and
6136          * then we'll be able to install our stuff !
6137          *
6138          * Upon return, there will be no owner for the current PMU
6139          */
6140         if (owner) pfm_lazy_save_regs(owner);
6141 
6142         /*
6143          * To avoid leaking information to the user level when psr.sp=0,
6144          * we must reload ALL implemented pmds (even the ones we don't use).
6145          * In the kernel we only allow PFM_READ_PMDS on registers which
6146          * we initialized or requested (sampling) so there is no risk there.
6147          */
6148         pmd_mask = pfm_sysctl.fastctxsw ?  ctx->ctx_used_pmds[0] : ctx->ctx_all_pmds[0];
6149 
6150         /*
6151          * ALL accessible PMCs are systematically reloaded, unused registers
6152          * get their default (from pfm_reset_pmu_state()) values to avoid picking
6153          * up stale configuration.
6154          *
6155          * PMC0 is never in the mask. It is always restored separately
6156          */
6157         pmc_mask = ctx->ctx_all_pmcs[0];
6158 
6159         pfm_restore_pmds(ctx->th_pmds, pmd_mask);
6160         pfm_restore_pmcs(ctx->th_pmcs, pmc_mask);
6161 
6162         /*
6163          * check for pending overflow at the time the state
6164          * was saved.
6165          */
6166         if (unlikely(PMC0_HAS_OVFL(ctx->th_pmcs[0]))) {
6167                 /*
6168                  * reload pmc0 with the overflow information
6169                  * On McKinley PMU, this will trigger a PMU interrupt
6170                  */
6171                 ia64_set_pmc(0, ctx->th_pmcs[0]);
6172                 ia64_srlz_d();
6173 
6174                 ctx->th_pmcs[0] = 0UL;
6175 
6176                 /*
6177                  * will replay the PMU interrupt
6178                  */
6179                 if (need_irq_resend) ia64_resend_irq(IA64_PERFMON_VECTOR);
6180 
6181                 pfm_stats[smp_processor_id()].pfm_replay_ovfl_intr_count++;
6182         }
6183 
6184         /*
6185          * establish new ownership. 
6186          */
6187         SET_PMU_OWNER(task, ctx);
6188 
6189         /*
6190          * restore the psr.up bit. measurement
6191          * is active again.
6192          * no PMU interrupt can happen at this point
6193          * because we still have interrupts disabled.
6194          */
6195         if (likely(psr_up)) pfm_set_psr_up();
6196 }
6197 #endif /* CONFIG_SMP */
6198 
6199 /*
6200  * this function assumes monitoring is stopped
6201  */
6202 static void
6203 pfm_flush_pmds(struct task_struct *task, pfm_context_t *ctx)
6204 {
6205         u64 pmc0;
6206         unsigned long mask2, val, pmd_val, ovfl_val;
6207         int i, can_access_pmu = 0;
6208         int is_self;
6209 
6210         /*
6211          * is the caller the task being monitored (or which initiated the
6212          * session for system wide measurements)
6213          */
6214         is_self = ctx->ctx_task == task ? 1 : 0;
6215 
6216         /*
6217          * can access PMU is task is the owner of the PMU state on the current CPU
6218          * or if we are running on the CPU bound to the context in system-wide mode
6219          * (that is not necessarily the task the context is attached to in this mode).
6220          * In system-wide we always have can_access_pmu true because a task running on an
6221          * invalid processor is flagged earlier in the call stack (see pfm_stop).
6222          */
6223         can_access_pmu = (GET_PMU_OWNER() == task) || (ctx->ctx_fl_system && ctx->ctx_cpu == smp_processor_id());
6224         if (can_access_pmu) {
6225                 /*
6226                  * Mark the PMU as not owned
6227                  * This will cause the interrupt handler to do nothing in case an overflow
6228                  * interrupt was in-flight
6229                  * This also guarantees that pmc0 will contain the final state
6230                  * It virtually gives us full control on overflow processing from that point
6231                  * on.
6232                  */
6233                 SET_PMU_OWNER(NULL, NULL);
6234                 DPRINT(("releasing ownership\n"));
6235 
6236                 /*
6237                  * read current overflow status:
6238                  *
6239                  * we are guaranteed to read the final stable state
6240                  */
6241                 ia64_srlz_d();
6242                 pmc0 = ia64_get_pmc(0); /* slow */
6243 
6244                 /*
6245                  * reset freeze bit, overflow status information destroyed
6246                  */
6247                 pfm_unfreeze_pmu();
6248         } else {
6249                 pmc0 = ctx->th_pmcs[0];
6250                 /*
6251                  * clear whatever overflow status bits there were
6252                  */
6253                 ctx->th_pmcs[0] = 0;
6254         }
6255         ovfl_val = pmu_conf->ovfl_val;
6256         /*
6257          * we save all the used pmds
6258          * we take care of overflows for counting PMDs
6259          *
6260          * XXX: sampling situation is not taken into account here
6261          */
6262         mask2 = ctx->ctx_used_pmds[0];
6263 
6264         DPRINT(("is_self=%d ovfl_val=0x%lx mask2=0x%lx\n", is_self, ovfl_val, mask2));
6265 
6266         for (i = 0; mask2; i++, mask2>>=1) {
6267 
6268                 /* skip non used pmds */
6269                 if ((mask2 & 0x1) == 0) continue;
6270 
6271                 /*
6272                  * can access PMU always true in system wide mode
6273                  */
6274                 val = pmd_val = can_access_pmu ? ia64_get_pmd(i) : ctx->th_pmds[i];
6275 
6276                 if (PMD_IS_COUNTING(i)) {
6277                         DPRINT(("[%d] pmd[%d] ctx_pmd=0x%lx hw_pmd=0x%lx\n",
6278                                 task_pid_nr(task),
6279                                 i,
6280                                 ctx->ctx_pmds[i].val,
6281                                 val & ovfl_val));
6282 
6283                         /*
6284                          * we rebuild the full 64 bit value of the counter
6285                          */
6286                         val = ctx->ctx_pmds[i].val + (val & ovfl_val);
6287 
6288                         /*
6289                          * now everything is in ctx_pmds[] and we need
6290                          * to clear the saved context from save_regs() such that
6291                          * pfm_read_pmds() gets the correct value
6292                          */
6293                         pmd_val = 0UL;
6294 
6295                         /*
6296                          * take care of overflow inline
6297                          */
6298                         if (pmc0 & (1UL << i)) {
6299                                 val += 1 + ovfl_val;
6300                                 DPRINT(("[%d] pmd[%d] overflowed\n", task_pid_nr(task), i));
6301                         }
6302                 }
6303 
6304                 DPRINT(("[%d] ctx_pmd[%d]=0x%lx  pmd_val=0x%lx\n", task_pid_nr(task), i, val, pmd_val));
6305 
6306                 if (is_self) ctx->th_pmds[i] = pmd_val;
6307 
6308                 ctx->ctx_pmds[i].val = val;
6309         }
6310 }
6311 
6312 static struct irqaction perfmon_irqaction = {
6313         .handler = pfm_interrupt_handler,
6314         .name    = "perfmon"
6315 };
6316 
6317 static void
6318 pfm_alt_save_pmu_state(void *data)
6319 {
6320         struct pt_regs *regs;
6321 
6322         regs = task_pt_regs(current);
6323 
6324         DPRINT(("called\n"));
6325 
6326         /*
6327          * should not be necessary but
6328          * let's take not risk
6329          */
6330         pfm_clear_psr_up();
6331         pfm_clear_psr_pp();
6332         ia64_psr(regs)->pp = 0;
6333 
6334         /*
6335          * This call is required
6336          * May cause a spurious interrupt on some processors
6337          */
6338         pfm_freeze_pmu();
6339 
6340         ia64_srlz_d();
6341 }
6342 
6343 void
6344 pfm_alt_restore_pmu_state(void *data)
6345 {
6346         struct pt_regs *regs;
6347 
6348         regs = task_pt_regs(current);
6349 
6350         DPRINT(("called\n"));
6351 
6352         /*
6353          * put PMU back in state expected
6354          * by perfmon
6355          */
6356         pfm_clear_psr_up();
6357         pfm_clear_psr_pp();
6358         ia64_psr(regs)->pp = 0;
6359 
6360         /*
6361          * perfmon runs with PMU unfrozen at all times
6362          */
6363         pfm_unfreeze_pmu();
6364 
6365         ia64_srlz_d();
6366 }
6367 
6368 int
6369 pfm_install_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
6370 {
6371         int ret, i;
6372         int reserve_cpu;
6373 
6374         /* some sanity checks */
6375         if (hdl == NULL || hdl->handler == NULL) return -EINVAL;
6376 
6377         /* do the easy test first */
6378         if (pfm_alt_intr_handler) return -EBUSY;
6379 
6380         /* one at a time in the install or remove, just fail the others */
6381         if (!spin_trylock(&pfm_alt_install_check)) {
6382                 return -EBUSY;
6383         }
6384 
6385         /* reserve our session */
6386         for_each_online_cpu(reserve_cpu) {
6387                 ret = pfm_reserve_session(NULL, 1, reserve_cpu);
6388                 if (ret) goto cleanup_reserve;
6389         }
6390 
6391         /* save the current system wide pmu states */
6392         ret = on_each_cpu(pfm_alt_save_pmu_state, NULL, 1);
6393         if (ret) {
6394                 DPRINT(("on_each_cpu() failed: %d\n", ret));
6395                 goto cleanup_reserve;
6396         }
6397 
6398         /* officially change to the alternate interrupt handler */
6399         pfm_alt_intr_handler = hdl;
6400 
6401         spin_unlock(&pfm_alt_install_check);
6402 
6403         return 0;
6404 
6405 cleanup_reserve:
6406         for_each_online_cpu(i) {
6407                 /* don't unreserve more than we reserved */
6408                 if (i >= reserve_cpu) break;
6409 
6410                 pfm_unreserve_session(NULL, 1, i);
6411         }
6412 
6413         spin_unlock(&pfm_alt_install_check);
6414 
6415         return ret;
6416 }
6417 EXPORT_SYMBOL_GPL(pfm_install_alt_pmu_interrupt);
6418 
6419 int
6420 pfm_remove_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
6421 {
6422         int i;
6423         int ret;
6424 
6425         if (hdl == NULL) return -EINVAL;
6426 
6427         /* cannot remove someone else's handler! */
6428         if (pfm_alt_intr_handler != hdl) return -EINVAL;
6429 
6430         /* one at a time in the install or remove, just fail the others */
6431         if (!spin_trylock(&pfm_alt_install_check)) {
6432                 return -EBUSY;
6433         }
6434 
6435         pfm_alt_intr_handler = NULL;
6436 
6437         ret = on_each_cpu(pfm_alt_restore_pmu_state, NULL, 1);
6438         if (ret) {
6439                 DPRINT(("on_each_cpu() failed: %d\n", ret));
6440         }
6441 
6442         for_each_online_cpu(i) {
6443                 pfm_unreserve_session(NULL, 1, i);
6444         }
6445 
6446         spin_unlock(&pfm_alt_install_check);
6447 
6448         return 0;
6449 }
6450 EXPORT_SYMBOL_GPL(pfm_remove_alt_pmu_interrupt);
6451 
6452 /*
6453  * perfmon initialization routine, called from the initcall() table
6454  */
6455 static int init_pfm_fs(void);
6456 
6457 static int __init
6458 pfm_probe_pmu(void)
6459 {
6460         pmu_config_t **p;
6461         int family;
6462 
6463         family = local_cpu_data->family;
6464         p      = pmu_confs;
6465 
6466         while(*p) {
6467                 if ((*p)->probe) {
6468                         if ((*p)->probe() == 0) goto found;
6469                 } else if ((*p)->pmu_family == family || (*p)->pmu_family == 0xff) {
6470                         goto found;
6471                 }
6472                 p++;
6473         }
6474         return -1;
6475 found:
6476         pmu_conf = *p;
6477         return 0;
6478 }
6479 
6480 int __init
6481 pfm_init(void)
6482 {
6483         unsigned int n, n_counters, i;
6484 
6485         printk("perfmon: version %u.%u IRQ %u\n",
6486                 PFM_VERSION_MAJ,
6487                 PFM_VERSION_MIN,
6488                 IA64_PERFMON_VECTOR);
6489 
6490         if (pfm_probe_pmu()) {
6491                 printk(KERN_INFO "perfmon: disabled, there is no support for processor family %d\n", 
6492                                 local_cpu_data->family);
6493                 return -ENODEV;
6494         }
6495 
6496         /*
6497          * compute the number of implemented PMD/PMC from the
6498          * description tables
6499          */
6500         n = 0;
6501         for (i=0; PMC_IS_LAST(i) == 0;  i++) {
6502                 if (PMC_IS_IMPL(i) == 0) continue;
6503                 pmu_conf->impl_pmcs[i>>6] |= 1UL << (i&63);
6504                 n++;
6505         }
6506         pmu_conf->num_pmcs = n;
6507 
6508         n = 0; n_counters = 0;
6509         for (i=0; PMD_IS_LAST(i) == 0;  i++) {
6510                 if (PMD_IS_IMPL(i) == 0) continue;
6511                 pmu_conf->impl_pmds[i>>6] |= 1UL << (i&63);
6512                 n++;
6513                 if (PMD_IS_COUNTING(i)) n_counters++;
6514         }
6515         pmu_conf->num_pmds      = n;
6516         pmu_conf->num_counters  = n_counters;
6517 
6518         /*
6519          * sanity checks on the number of debug registers
6520          */
6521         if (pmu_conf->use_rr_dbregs) {
6522                 if (pmu_conf->num_ibrs > IA64_NUM_DBG_REGS) {
6523                         printk(KERN_INFO "perfmon: unsupported number of code debug registers (%u)\n", pmu_conf->num_ibrs);
6524                         pmu_conf = NULL;
6525                         return -1;
6526                 }
6527                 if (pmu_conf->num_dbrs > IA64_NUM_DBG_REGS) {
6528                         printk(KERN_INFO "perfmon: unsupported number of data debug registers (%u)\n", pmu_conf->num_ibrs);
6529                         pmu_conf = NULL;
6530                         return -1;
6531                 }
6532         }
6533 
6534         printk("perfmon: %s PMU detected, %u PMCs, %u PMDs, %u counters (%lu bits)\n",
6535                pmu_conf->pmu_name,
6536                pmu_conf->num_pmcs,
6537                pmu_conf->num_pmds,
6538                pmu_conf->num_counters,
6539                ffz(pmu_conf->ovfl_val));
6540 
6541         /* sanity check */
6542         if (pmu_conf->num_pmds >= PFM_NUM_PMD_REGS || pmu_conf->num_pmcs >= PFM_NUM_PMC_REGS) {
6543                 printk(KERN_ERR "perfmon: not enough pmc/pmd, perfmon disabled\n");
6544                 pmu_conf = NULL;
6545                 return -1;
6546         }
6547 
6548         /*
6549          * create /proc/perfmon (mostly for debugging purposes)
6550          */
6551         perfmon_dir = proc_create_seq("perfmon", S_IRUGO, NULL, &pfm_seq_ops);
6552         if (perfmon_dir == NULL) {
6553                 printk(KERN_ERR "perfmon: cannot create /proc entry, perfmon disabled\n");
6554                 pmu_conf = NULL;
6555                 return -1;
6556         }
6557 
6558         /*
6559          * create /proc/sys/kernel/perfmon (for debugging purposes)
6560          */
6561         pfm_sysctl_header = register_sysctl_table(pfm_sysctl_root);
6562 
6563         /*
6564          * initialize all our spinlocks
6565          */
6566         spin_lock_init(&pfm_sessions.pfs_lock);
6567         spin_lock_init(&pfm_buffer_fmt_lock);
6568 
6569         init_pfm_fs();
6570 
6571         for(i=0; i < NR_CPUS; i++) pfm_stats[i].pfm_ovfl_intr_cycles_min = ~0UL;
6572 
6573         return 0;
6574 }
6575 
6576 __initcall(pfm_init);
6577 
6578 /*
6579  * this function is called before pfm_init()
6580  */
6581 void
6582 pfm_init_percpu (void)
6583 {
6584         static int first_time=1;
6585         /*
6586          * make sure no measurement is active
6587          * (may inherit programmed PMCs from EFI).
6588          */
6589         pfm_clear_psr_pp();
6590         pfm_clear_psr_up();
6591 
6592         /*
6593          * we run with the PMU not frozen at all times
6594          */
6595         pfm_unfreeze_pmu();
6596 
6597         if (first_time) {
6598                 register_percpu_irq(IA64_PERFMON_VECTOR, &perfmon_irqaction);
6599                 first_time=0;
6600         }
6601 
6602         ia64_setreg(_IA64_REG_CR_PMV, IA64_PERFMON_VECTOR);
6603         ia64_srlz_d();
6604 }
6605 
6606 /*
6607  * used for debug purposes only
6608  */
6609 void
6610 dump_pmu_state(const char *from)
6611 {
6612         struct task_struct *task;
6613         struct pt_regs *regs;
6614         pfm_context_t *ctx;
6615         unsigned long psr, dcr, info, flags;
6616         int i, this_cpu;
6617 
6618         local_irq_save(flags);
6619 
6620         this_cpu = smp_processor_id();
6621         regs     = task_pt_regs(current);
6622         info     = PFM_CPUINFO_GET();
6623         dcr      = ia64_getreg(_IA64_REG_CR_DCR);
6624 
6625         if (info == 0 && ia64_psr(regs)->pp == 0 && (dcr & IA64_DCR_PP) == 0) {
6626                 local_irq_restore(flags);
6627                 return;
6628         }
6629 
6630         printk("CPU%d from %s() current [%d] iip=0x%lx %s\n", 
6631                 this_cpu, 
6632                 from, 
6633                 task_pid_nr(current),
6634                 regs->cr_iip,
6635                 current->comm);
6636 
6637         task = GET_PMU_OWNER();
6638         ctx  = GET_PMU_CTX();
6639 
6640         printk("->CPU%d owner [%d] ctx=%p\n", this_cpu, task ? task_pid_nr(task) : -1, ctx);
6641 
6642         psr = pfm_get_psr();
6643 
6644         printk("->CPU%d pmc0=0x%lx psr.pp=%d psr.up=%d dcr.pp=%d syst_info=0x%lx user_psr.up=%d user_psr.pp=%d\n", 
6645                 this_cpu,
6646                 ia64_get_pmc(0),
6647                 psr & IA64_PSR_PP ? 1 : 0,
6648                 psr & IA64_PSR_UP ? 1 : 0,
6649                 dcr & IA64_DCR_PP ? 1 : 0,
6650                 info,
6651                 ia64_psr(regs)->up,
6652                 ia64_psr(regs)->pp);
6653 
6654         ia64_psr(regs)->up = 0;
6655         ia64_psr(regs)->pp = 0;
6656 
6657         for (i=1; PMC_IS_LAST(i) == 0; i++) {
6658                 if (PMC_IS_IMPL(i) == 0) continue;
6659                 printk("->CPU%d pmc[%d]=0x%lx thread_pmc[%d]=0x%lx\n", this_cpu, i, ia64_get_pmc(i), i, ctx->th_pmcs[i]);
6660         }
6661 
6662         for (i=1; PMD_IS_LAST(i) == 0; i++) {
6663                 if (PMD_IS_IMPL(i) == 0) continue;
6664                 printk("->CPU%d pmd[%d]=0x%lx thread_pmd[%d]=0x%lx\n", this_cpu, i, ia64_get_pmd(i), i, ctx->th_pmds[i]);
6665         }
6666 
6667         if (ctx) {
6668                 printk("->CPU%d ctx_state=%d vaddr=%p addr=%p fd=%d ctx_task=[%d] saved_psr_up=0x%lx\n",
6669                                 this_cpu,
6670                                 ctx->ctx_state,
6671                                 ctx->ctx_smpl_vaddr,
6672                                 ctx->ctx_smpl_hdr,
6673                                 ctx->ctx_msgq_head,
6674                                 ctx->ctx_msgq_tail,
6675                                 ctx->ctx_saved_psr_up);
6676         }
6677         local_irq_restore(flags);
6678 }
6679 
6680 /*
6681  * called from process.c:copy_thread(). task is new child.
6682  */
6683 void
6684 pfm_inherit(struct task_struct *task, struct pt_regs *regs)
6685 {
6686         struct thread_struct *thread;
6687 
6688         DPRINT(("perfmon: pfm_inherit clearing state for [%d]\n", task_pid_nr(task)));
6689 
6690         thread = &task->thread;
6691 
6692         /*
6693          * cut links inherited from parent (current)
6694          */
6695         thread->pfm_context = NULL;
6696 
6697         PFM_SET_WORK_PENDING(task, 0);
6698 
6699         /*
6700          * the psr bits are already set properly in copy_threads()
6701          */
6702 }
6703 #else  /* !CONFIG_PERFMON */
6704 asmlinkage long
6705 sys_perfmonctl (int fd, int cmd, void *arg, int count)
6706 {
6707         return -ENOSYS;
6708 }
6709 #endif /* CONFIG_PERFMON */
6710 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp