~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/events/core.c

Version: ~ [ linux-5.10-rc1 ] ~ [ linux-5.9.1 ] ~ [ linux-5.8.16 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.72 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.152 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.202 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.240 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.240 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Performance events core code:
  3  *
  4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  5  *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
  6  *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
  7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  8  *
  9  * For licensing details see kernel-base/COPYING
 10  */
 11 
 12 #include <linux/fs.h>
 13 #include <linux/mm.h>
 14 #include <linux/cpu.h>
 15 #include <linux/smp.h>
 16 #include <linux/idr.h>
 17 #include <linux/file.h>
 18 #include <linux/poll.h>
 19 #include <linux/slab.h>
 20 #include <linux/hash.h>
 21 #include <linux/tick.h>
 22 #include <linux/sysfs.h>
 23 #include <linux/dcache.h>
 24 #include <linux/percpu.h>
 25 #include <linux/ptrace.h>
 26 #include <linux/reboot.h>
 27 #include <linux/vmstat.h>
 28 #include <linux/device.h>
 29 #include <linux/export.h>
 30 #include <linux/vmalloc.h>
 31 #include <linux/hardirq.h>
 32 #include <linux/rculist.h>
 33 #include <linux/uaccess.h>
 34 #include <linux/syscalls.h>
 35 #include <linux/anon_inodes.h>
 36 #include <linux/kernel_stat.h>
 37 #include <linux/perf_event.h>
 38 #include <linux/ftrace_event.h>
 39 #include <linux/hw_breakpoint.h>
 40 #include <linux/mm_types.h>
 41 #include <linux/cgroup.h>
 42 #include <linux/compat.h>
 43 
 44 #include "internal.h"
 45 
 46 #include <asm/irq_regs.h>
 47 
 48 struct remote_function_call {
 49         struct task_struct      *p;
 50         int                     (*func)(void *info);
 51         void                    *info;
 52         int                     ret;
 53 };
 54 
 55 static void remote_function(void *data)
 56 {
 57         struct remote_function_call *tfc = data;
 58         struct task_struct *p = tfc->p;
 59 
 60         if (p) {
 61                 tfc->ret = -EAGAIN;
 62                 if (task_cpu(p) != smp_processor_id() || !task_curr(p))
 63                         return;
 64         }
 65 
 66         tfc->ret = tfc->func(tfc->info);
 67 }
 68 
 69 /**
 70  * task_function_call - call a function on the cpu on which a task runs
 71  * @p:          the task to evaluate
 72  * @func:       the function to be called
 73  * @info:       the function call argument
 74  *
 75  * Calls the function @func when the task is currently running. This might
 76  * be on the current CPU, which just calls the function directly
 77  *
 78  * returns: @func return value, or
 79  *          -ESRCH  - when the process isn't running
 80  *          -EAGAIN - when the process moved away
 81  */
 82 static int
 83 task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
 84 {
 85         struct remote_function_call data = {
 86                 .p      = p,
 87                 .func   = func,
 88                 .info   = info,
 89                 .ret    = -ESRCH, /* No such (running) process */
 90         };
 91 
 92         if (task_curr(p))
 93                 smp_call_function_single(task_cpu(p), remote_function, &data, 1);
 94 
 95         return data.ret;
 96 }
 97 
 98 /**
 99  * cpu_function_call - call a function on the cpu
100  * @func:       the function to be called
101  * @info:       the function call argument
102  *
103  * Calls the function @func on the remote cpu.
104  *
105  * returns: @func return value or -ENXIO when the cpu is offline
106  */
107 static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
108 {
109         struct remote_function_call data = {
110                 .p      = NULL,
111                 .func   = func,
112                 .info   = info,
113                 .ret    = -ENXIO, /* No such CPU */
114         };
115 
116         smp_call_function_single(cpu, remote_function, &data, 1);
117 
118         return data.ret;
119 }
120 
121 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
122                        PERF_FLAG_FD_OUTPUT  |\
123                        PERF_FLAG_PID_CGROUP)
124 
125 /*
126  * branch priv levels that need permission checks
127  */
128 #define PERF_SAMPLE_BRANCH_PERM_PLM \
129         (PERF_SAMPLE_BRANCH_KERNEL |\
130          PERF_SAMPLE_BRANCH_HV)
131 
132 enum event_type_t {
133         EVENT_FLEXIBLE = 0x1,
134         EVENT_PINNED = 0x2,
135         EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
136 };
137 
138 /*
139  * perf_sched_events : >0 events exist
140  * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
141  */
142 struct static_key_deferred perf_sched_events __read_mostly;
143 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
144 static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
145 
146 static atomic_t nr_mmap_events __read_mostly;
147 static atomic_t nr_comm_events __read_mostly;
148 static atomic_t nr_task_events __read_mostly;
149 
150 static LIST_HEAD(pmus);
151 static DEFINE_MUTEX(pmus_lock);
152 static struct srcu_struct pmus_srcu;
153 
154 /*
155  * perf event paranoia level:
156  *  -1 - not paranoid at all
157  *   0 - disallow raw tracepoint access for unpriv
158  *   1 - disallow cpu events for unpriv
159  *   2 - disallow kernel profiling for unpriv
160  */
161 int sysctl_perf_event_paranoid __read_mostly = 1;
162 
163 /* Minimum for 512 kiB + 1 user control page */
164 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
165 
166 /*
167  * max perf event sample rate
168  */
169 #define DEFAULT_MAX_SAMPLE_RATE         100000
170 #define DEFAULT_SAMPLE_PERIOD_NS        (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)
171 #define DEFAULT_CPU_TIME_MAX_PERCENT    25
172 
173 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
174 
175 static int max_samples_per_tick __read_mostly   = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
176 static int perf_sample_period_ns __read_mostly  = DEFAULT_SAMPLE_PERIOD_NS;
177 
178 static atomic_t perf_sample_allowed_ns __read_mostly =
179         ATOMIC_INIT( DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100);
180 
181 void update_perf_cpu_limits(void)
182 {
183         u64 tmp = perf_sample_period_ns;
184 
185         tmp *= sysctl_perf_cpu_time_max_percent;
186         do_div(tmp, 100);
187         atomic_set(&perf_sample_allowed_ns, tmp);
188 }
189 
190 int perf_proc_update_handler(struct ctl_table *table, int write,
191                 void __user *buffer, size_t *lenp,
192                 loff_t *ppos)
193 {
194         int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
195 
196         if (ret || !write)
197                 return ret;
198 
199         max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
200         perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
201         update_perf_cpu_limits();
202 
203         return 0;
204 }
205 
206 int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT;
207 
208 int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
209                                 void __user *buffer, size_t *lenp,
210                                 loff_t *ppos)
211 {
212         int ret = proc_dointvec(table, write, buffer, lenp, ppos);
213 
214         if (ret || !write)
215                 return ret;
216 
217         update_perf_cpu_limits();
218 
219         return 0;
220 }
221 
222 /*
223  * perf samples are done in some very critical code paths (NMIs).
224  * If they take too much CPU time, the system can lock up and not
225  * get any real work done.  This will drop the sample rate when
226  * we detect that events are taking too long.
227  */
228 #define NR_ACCUMULATED_SAMPLES 128
229 DEFINE_PER_CPU(u64, running_sample_length);
230 
231 void perf_sample_event_took(u64 sample_len_ns)
232 {
233         u64 avg_local_sample_len;
234         u64 local_samples_len;
235 
236         if (atomic_read(&perf_sample_allowed_ns) == 0)
237                 return;
238 
239         /* decay the counter by 1 average sample */
240         local_samples_len = __get_cpu_var(running_sample_length);
241         local_samples_len -= local_samples_len/NR_ACCUMULATED_SAMPLES;
242         local_samples_len += sample_len_ns;
243         __get_cpu_var(running_sample_length) = local_samples_len;
244 
245         /*
246          * note: this will be biased artifically low until we have
247          * seen NR_ACCUMULATED_SAMPLES.  Doing it this way keeps us
248          * from having to maintain a count.
249          */
250         avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES;
251 
252         if (avg_local_sample_len <= atomic_read(&perf_sample_allowed_ns))
253                 return;
254 
255         if (max_samples_per_tick <= 1)
256                 return;
257 
258         max_samples_per_tick = DIV_ROUND_UP(max_samples_per_tick, 2);
259         sysctl_perf_event_sample_rate = max_samples_per_tick * HZ;
260         perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
261 
262         printk_ratelimited(KERN_WARNING
263                         "perf samples too long (%lld > %d), lowering "
264                         "kernel.perf_event_max_sample_rate to %d\n",
265                         avg_local_sample_len,
266                         atomic_read(&perf_sample_allowed_ns),
267                         sysctl_perf_event_sample_rate);
268 
269         update_perf_cpu_limits();
270 }
271 
272 static atomic64_t perf_event_id;
273 
274 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
275                               enum event_type_t event_type);
276 
277 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
278                              enum event_type_t event_type,
279                              struct task_struct *task);
280 
281 static void update_context_time(struct perf_event_context *ctx);
282 static u64 perf_event_time(struct perf_event *event);
283 
284 void __weak perf_event_print_debug(void)        { }
285 
286 extern __weak const char *perf_pmu_name(void)
287 {
288         return "pmu";
289 }
290 
291 static inline u64 perf_clock(void)
292 {
293         return local_clock();
294 }
295 
296 static inline struct perf_cpu_context *
297 __get_cpu_context(struct perf_event_context *ctx)
298 {
299         return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
300 }
301 
302 static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
303                           struct perf_event_context *ctx)
304 {
305         raw_spin_lock(&cpuctx->ctx.lock);
306         if (ctx)
307                 raw_spin_lock(&ctx->lock);
308 }
309 
310 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
311                             struct perf_event_context *ctx)
312 {
313         if (ctx)
314                 raw_spin_unlock(&ctx->lock);
315         raw_spin_unlock(&cpuctx->ctx.lock);
316 }
317 
318 #ifdef CONFIG_CGROUP_PERF
319 
320 /*
321  * perf_cgroup_info keeps track of time_enabled for a cgroup.
322  * This is a per-cpu dynamically allocated data structure.
323  */
324 struct perf_cgroup_info {
325         u64                             time;
326         u64                             timestamp;
327 };
328 
329 struct perf_cgroup {
330         struct cgroup_subsys_state      css;
331         struct perf_cgroup_info __percpu *info;
332 };
333 
334 /*
335  * Must ensure cgroup is pinned (css_get) before calling
336  * this function. In other words, we cannot call this function
337  * if there is no cgroup event for the current CPU context.
338  */
339 static inline struct perf_cgroup *
340 perf_cgroup_from_task(struct task_struct *task)
341 {
342         return container_of(task_subsys_state(task, perf_subsys_id),
343                         struct perf_cgroup, css);
344 }
345 
346 static inline bool
347 perf_cgroup_match(struct perf_event *event)
348 {
349         struct perf_event_context *ctx = event->ctx;
350         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
351 
352         /* @event doesn't care about cgroup */
353         if (!event->cgrp)
354                 return true;
355 
356         /* wants specific cgroup scope but @cpuctx isn't associated with any */
357         if (!cpuctx->cgrp)
358                 return false;
359 
360         /*
361          * Cgroup scoping is recursive.  An event enabled for a cgroup is
362          * also enabled for all its descendant cgroups.  If @cpuctx's
363          * cgroup is a descendant of @event's (the test covers identity
364          * case), it's a match.
365          */
366         return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,
367                                     event->cgrp->css.cgroup);
368 }
369 
370 static inline bool perf_tryget_cgroup(struct perf_event *event)
371 {
372         return css_tryget(&event->cgrp->css);
373 }
374 
375 static inline void perf_put_cgroup(struct perf_event *event)
376 {
377         css_put(&event->cgrp->css);
378 }
379 
380 static inline void perf_detach_cgroup(struct perf_event *event)
381 {
382         perf_put_cgroup(event);
383         event->cgrp = NULL;
384 }
385 
386 static inline int is_cgroup_event(struct perf_event *event)
387 {
388         return event->cgrp != NULL;
389 }
390 
391 static inline u64 perf_cgroup_event_time(struct perf_event *event)
392 {
393         struct perf_cgroup_info *t;
394 
395         t = per_cpu_ptr(event->cgrp->info, event->cpu);
396         return t->time;
397 }
398 
399 static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
400 {
401         struct perf_cgroup_info *info;
402         u64 now;
403 
404         now = perf_clock();
405 
406         info = this_cpu_ptr(cgrp->info);
407 
408         info->time += now - info->timestamp;
409         info->timestamp = now;
410 }
411 
412 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
413 {
414         struct perf_cgroup *cgrp_out = cpuctx->cgrp;
415         if (cgrp_out)
416                 __update_cgrp_time(cgrp_out);
417 }
418 
419 static inline void update_cgrp_time_from_event(struct perf_event *event)
420 {
421         struct perf_cgroup *cgrp;
422 
423         /*
424          * ensure we access cgroup data only when needed and
425          * when we know the cgroup is pinned (css_get)
426          */
427         if (!is_cgroup_event(event))
428                 return;
429 
430         cgrp = perf_cgroup_from_task(current);
431         /*
432          * Do not update time when cgroup is not active
433          */
434         if (cgrp == event->cgrp)
435                 __update_cgrp_time(event->cgrp);
436 }
437 
438 static inline void
439 perf_cgroup_set_timestamp(struct task_struct *task,
440                           struct perf_event_context *ctx)
441 {
442         struct perf_cgroup *cgrp;
443         struct perf_cgroup_info *info;
444 
445         /*
446          * ctx->lock held by caller
447          * ensure we do not access cgroup data
448          * unless we have the cgroup pinned (css_get)
449          */
450         if (!task || !ctx->nr_cgroups)
451                 return;
452 
453         cgrp = perf_cgroup_from_task(task);
454         info = this_cpu_ptr(cgrp->info);
455         info->timestamp = ctx->timestamp;
456 }
457 
458 #define PERF_CGROUP_SWOUT       0x1 /* cgroup switch out every event */
459 #define PERF_CGROUP_SWIN        0x2 /* cgroup switch in events based on task */
460 
461 /*
462  * reschedule events based on the cgroup constraint of task.
463  *
464  * mode SWOUT : schedule out everything
465  * mode SWIN : schedule in based on cgroup for next
466  */
467 void perf_cgroup_switch(struct task_struct *task, int mode)
468 {
469         struct perf_cpu_context *cpuctx;
470         struct pmu *pmu;
471         unsigned long flags;
472 
473         /*
474          * disable interrupts to avoid geting nr_cgroup
475          * changes via __perf_event_disable(). Also
476          * avoids preemption.
477          */
478         local_irq_save(flags);
479 
480         /*
481          * we reschedule only in the presence of cgroup
482          * constrained events.
483          */
484         rcu_read_lock();
485 
486         list_for_each_entry_rcu(pmu, &pmus, entry) {
487                 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
488                 if (cpuctx->unique_pmu != pmu)
489                         continue; /* ensure we process each cpuctx once */
490 
491                 /*
492                  * perf_cgroup_events says at least one
493                  * context on this CPU has cgroup events.
494                  *
495                  * ctx->nr_cgroups reports the number of cgroup
496                  * events for a context.
497                  */
498                 if (cpuctx->ctx.nr_cgroups > 0) {
499                         perf_ctx_lock(cpuctx, cpuctx->task_ctx);
500                         perf_pmu_disable(cpuctx->ctx.pmu);
501 
502                         if (mode & PERF_CGROUP_SWOUT) {
503                                 cpu_ctx_sched_out(cpuctx, EVENT_ALL);
504                                 /*
505                                  * must not be done before ctxswout due
506                                  * to event_filter_match() in event_sched_out()
507                                  */
508                                 cpuctx->cgrp = NULL;
509                         }
510 
511                         if (mode & PERF_CGROUP_SWIN) {
512                                 WARN_ON_ONCE(cpuctx->cgrp);
513                                 /*
514                                  * set cgrp before ctxsw in to allow
515                                  * event_filter_match() to not have to pass
516                                  * task around
517                                  */
518                                 cpuctx->cgrp = perf_cgroup_from_task(task);
519                                 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
520                         }
521                         perf_pmu_enable(cpuctx->ctx.pmu);
522                         perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
523                 }
524         }
525 
526         rcu_read_unlock();
527 
528         local_irq_restore(flags);
529 }
530 
531 static inline void perf_cgroup_sched_out(struct task_struct *task,
532                                          struct task_struct *next)
533 {
534         struct perf_cgroup *cgrp1;
535         struct perf_cgroup *cgrp2 = NULL;
536 
537         /*
538          * we come here when we know perf_cgroup_events > 0
539          */
540         cgrp1 = perf_cgroup_from_task(task);
541 
542         /*
543          * next is NULL when called from perf_event_enable_on_exec()
544          * that will systematically cause a cgroup_switch()
545          */
546         if (next)
547                 cgrp2 = perf_cgroup_from_task(next);
548 
549         /*
550          * only schedule out current cgroup events if we know
551          * that we are switching to a different cgroup. Otherwise,
552          * do no touch the cgroup events.
553          */
554         if (cgrp1 != cgrp2)
555                 perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
556 }
557 
558 static inline void perf_cgroup_sched_in(struct task_struct *prev,
559                                         struct task_struct *task)
560 {
561         struct perf_cgroup *cgrp1;
562         struct perf_cgroup *cgrp2 = NULL;
563 
564         /*
565          * we come here when we know perf_cgroup_events > 0
566          */
567         cgrp1 = perf_cgroup_from_task(task);
568 
569         /* prev can never be NULL */
570         cgrp2 = perf_cgroup_from_task(prev);
571 
572         /*
573          * only need to schedule in cgroup events if we are changing
574          * cgroup during ctxsw. Cgroup events were not scheduled
575          * out of ctxsw out if that was not the case.
576          */
577         if (cgrp1 != cgrp2)
578                 perf_cgroup_switch(task, PERF_CGROUP_SWIN);
579 }
580 
581 static inline int perf_cgroup_connect(int fd, struct perf_event *event,
582                                       struct perf_event_attr *attr,
583                                       struct perf_event *group_leader)
584 {
585         struct perf_cgroup *cgrp;
586         struct cgroup_subsys_state *css;
587         struct fd f = fdget(fd);
588         int ret = 0;
589 
590         if (!f.file)
591                 return -EBADF;
592 
593         css = cgroup_css_from_dir(f.file, perf_subsys_id);
594         if (IS_ERR(css)) {
595                 ret = PTR_ERR(css);
596                 goto out;
597         }
598 
599         cgrp = container_of(css, struct perf_cgroup, css);
600         event->cgrp = cgrp;
601 
602         /* must be done before we fput() the file */
603         if (!perf_tryget_cgroup(event)) {
604                 event->cgrp = NULL;
605                 ret = -ENOENT;
606                 goto out;
607         }
608 
609         /*
610          * all events in a group must monitor
611          * the same cgroup because a task belongs
612          * to only one perf cgroup at a time
613          */
614         if (group_leader && group_leader->cgrp != cgrp) {
615                 perf_detach_cgroup(event);
616                 ret = -EINVAL;
617         }
618 out:
619         fdput(f);
620         return ret;
621 }
622 
623 static inline void
624 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
625 {
626         struct perf_cgroup_info *t;
627         t = per_cpu_ptr(event->cgrp->info, event->cpu);
628         event->shadow_ctx_time = now - t->timestamp;
629 }
630 
631 static inline void
632 perf_cgroup_defer_enabled(struct perf_event *event)
633 {
634         /*
635          * when the current task's perf cgroup does not match
636          * the event's, we need to remember to call the
637          * perf_mark_enable() function the first time a task with
638          * a matching perf cgroup is scheduled in.
639          */
640         if (is_cgroup_event(event) && !perf_cgroup_match(event))
641                 event->cgrp_defer_enabled = 1;
642 }
643 
644 static inline void
645 perf_cgroup_mark_enabled(struct perf_event *event,
646                          struct perf_event_context *ctx)
647 {
648         struct perf_event *sub;
649         u64 tstamp = perf_event_time(event);
650 
651         if (!event->cgrp_defer_enabled)
652                 return;
653 
654         event->cgrp_defer_enabled = 0;
655 
656         event->tstamp_enabled = tstamp - event->total_time_enabled;
657         list_for_each_entry(sub, &event->sibling_list, group_entry) {
658                 if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
659                         sub->tstamp_enabled = tstamp - sub->total_time_enabled;
660                         sub->cgrp_defer_enabled = 0;
661                 }
662         }
663 }
664 #else /* !CONFIG_CGROUP_PERF */
665 
666 static inline bool
667 perf_cgroup_match(struct perf_event *event)
668 {
669         return true;
670 }
671 
672 static inline void perf_detach_cgroup(struct perf_event *event)
673 {}
674 
675 static inline int is_cgroup_event(struct perf_event *event)
676 {
677         return 0;
678 }
679 
680 static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
681 {
682         return 0;
683 }
684 
685 static inline void update_cgrp_time_from_event(struct perf_event *event)
686 {
687 }
688 
689 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
690 {
691 }
692 
693 static inline void perf_cgroup_sched_out(struct task_struct *task,
694                                          struct task_struct *next)
695 {
696 }
697 
698 static inline void perf_cgroup_sched_in(struct task_struct *prev,
699                                         struct task_struct *task)
700 {
701 }
702 
703 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
704                                       struct perf_event_attr *attr,
705                                       struct perf_event *group_leader)
706 {
707         return -EINVAL;
708 }
709 
710 static inline void
711 perf_cgroup_set_timestamp(struct task_struct *task,
712                           struct perf_event_context *ctx)
713 {
714 }
715 
716 void
717 perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
718 {
719 }
720 
721 static inline void
722 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
723 {
724 }
725 
726 static inline u64 perf_cgroup_event_time(struct perf_event *event)
727 {
728         return 0;
729 }
730 
731 static inline void
732 perf_cgroup_defer_enabled(struct perf_event *event)
733 {
734 }
735 
736 static inline void
737 perf_cgroup_mark_enabled(struct perf_event *event,
738                          struct perf_event_context *ctx)
739 {
740 }
741 #endif
742 
743 void perf_pmu_disable(struct pmu *pmu)
744 {
745         int *count = this_cpu_ptr(pmu->pmu_disable_count);
746         if (!(*count)++)
747                 pmu->pmu_disable(pmu);
748 }
749 
750 void perf_pmu_enable(struct pmu *pmu)
751 {
752         int *count = this_cpu_ptr(pmu->pmu_disable_count);
753         if (!--(*count))
754                 pmu->pmu_enable(pmu);
755 }
756 
757 static DEFINE_PER_CPU(struct list_head, rotation_list);
758 
759 /*
760  * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
761  * because they're strictly cpu affine and rotate_start is called with IRQs
762  * disabled, while rotate_context is called from IRQ context.
763  */
764 static void perf_pmu_rotate_start(struct pmu *pmu)
765 {
766         struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
767         struct list_head *head = &__get_cpu_var(rotation_list);
768 
769         WARN_ON(!irqs_disabled());
770 
771         if (list_empty(&cpuctx->rotation_list)) {
772                 int was_empty = list_empty(head);
773                 list_add(&cpuctx->rotation_list, head);
774                 if (was_empty)
775                         tick_nohz_full_kick();
776         }
777 }
778 
779 static void get_ctx(struct perf_event_context *ctx)
780 {
781         WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
782 }
783 
784 static void put_ctx(struct perf_event_context *ctx)
785 {
786         if (atomic_dec_and_test(&ctx->refcount)) {
787                 if (ctx->parent_ctx)
788                         put_ctx(ctx->parent_ctx);
789                 if (ctx->task)
790                         put_task_struct(ctx->task);
791                 kfree_rcu(ctx, rcu_head);
792         }
793 }
794 
795 static void unclone_ctx(struct perf_event_context *ctx)
796 {
797         if (ctx->parent_ctx) {
798                 put_ctx(ctx->parent_ctx);
799                 ctx->parent_ctx = NULL;
800         }
801 }
802 
803 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
804 {
805         /*
806          * only top level events have the pid namespace they were created in
807          */
808         if (event->parent)
809                 event = event->parent;
810 
811         return task_tgid_nr_ns(p, event->ns);
812 }
813 
814 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
815 {
816         /*
817          * only top level events have the pid namespace they were created in
818          */
819         if (event->parent)
820                 event = event->parent;
821 
822         return task_pid_nr_ns(p, event->ns);
823 }
824 
825 /*
826  * If we inherit events we want to return the parent event id
827  * to userspace.
828  */
829 static u64 primary_event_id(struct perf_event *event)
830 {
831         u64 id = event->id;
832 
833         if (event->parent)
834                 id = event->parent->id;
835 
836         return id;
837 }
838 
839 /*
840  * Get the perf_event_context for a task and lock it.
841  * This has to cope with with the fact that until it is locked,
842  * the context could get moved to another task.
843  */
844 static struct perf_event_context *
845 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
846 {
847         struct perf_event_context *ctx;
848 
849 retry:
850         /*
851          * One of the few rules of preemptible RCU is that one cannot do
852          * rcu_read_unlock() while holding a scheduler (or nested) lock when
853          * part of the read side critical section was preemptible -- see
854          * rcu_read_unlock_special().
855          *
856          * Since ctx->lock nests under rq->lock we must ensure the entire read
857          * side critical section is non-preemptible.
858          */
859         preempt_disable();
860         rcu_read_lock();
861         ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
862         if (ctx) {
863                 /*
864                  * If this context is a clone of another, it might
865                  * get swapped for another underneath us by
866                  * perf_event_task_sched_out, though the
867                  * rcu_read_lock() protects us from any context
868                  * getting freed.  Lock the context and check if it
869                  * got swapped before we could get the lock, and retry
870                  * if so.  If we locked the right context, then it
871                  * can't get swapped on us any more.
872                  */
873                 raw_spin_lock_irqsave(&ctx->lock, *flags);
874                 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
875                         raw_spin_unlock_irqrestore(&ctx->lock, *flags);
876                         rcu_read_unlock();
877                         preempt_enable();
878                         goto retry;
879                 }
880 
881                 if (!atomic_inc_not_zero(&ctx->refcount)) {
882                         raw_spin_unlock_irqrestore(&ctx->lock, *flags);
883                         ctx = NULL;
884                 }
885         }
886         rcu_read_unlock();
887         preempt_enable();
888         return ctx;
889 }
890 
891 /*
892  * Get the context for a task and increment its pin_count so it
893  * can't get swapped to another task.  This also increments its
894  * reference count so that the context can't get freed.
895  */
896 static struct perf_event_context *
897 perf_pin_task_context(struct task_struct *task, int ctxn)
898 {
899         struct perf_event_context *ctx;
900         unsigned long flags;
901 
902         ctx = perf_lock_task_context(task, ctxn, &flags);
903         if (ctx) {
904                 ++ctx->pin_count;
905                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
906         }
907         return ctx;
908 }
909 
910 static void perf_unpin_context(struct perf_event_context *ctx)
911 {
912         unsigned long flags;
913 
914         raw_spin_lock_irqsave(&ctx->lock, flags);
915         --ctx->pin_count;
916         raw_spin_unlock_irqrestore(&ctx->lock, flags);
917 }
918 
919 /*
920  * Update the record of the current time in a context.
921  */
922 static void update_context_time(struct perf_event_context *ctx)
923 {
924         u64 now = perf_clock();
925 
926         ctx->time += now - ctx->timestamp;
927         ctx->timestamp = now;
928 }
929 
930 static u64 perf_event_time(struct perf_event *event)
931 {
932         struct perf_event_context *ctx = event->ctx;
933 
934         if (is_cgroup_event(event))
935                 return perf_cgroup_event_time(event);
936 
937         return ctx ? ctx->time : 0;
938 }
939 
940 /*
941  * Update the total_time_enabled and total_time_running fields for a event.
942  * The caller of this function needs to hold the ctx->lock.
943  */
944 static void update_event_times(struct perf_event *event)
945 {
946         struct perf_event_context *ctx = event->ctx;
947         u64 run_end;
948 
949         if (event->state < PERF_EVENT_STATE_INACTIVE ||
950             event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
951                 return;
952         /*
953          * in cgroup mode, time_enabled represents
954          * the time the event was enabled AND active
955          * tasks were in the monitored cgroup. This is
956          * independent of the activity of the context as
957          * there may be a mix of cgroup and non-cgroup events.
958          *
959          * That is why we treat cgroup events differently
960          * here.
961          */
962         if (is_cgroup_event(event))
963                 run_end = perf_cgroup_event_time(event);
964         else if (ctx->is_active)
965                 run_end = ctx->time;
966         else
967                 run_end = event->tstamp_stopped;
968 
969         event->total_time_enabled = run_end - event->tstamp_enabled;
970 
971         if (event->state == PERF_EVENT_STATE_INACTIVE)
972                 run_end = event->tstamp_stopped;
973         else
974                 run_end = perf_event_time(event);
975 
976         event->total_time_running = run_end - event->tstamp_running;
977 
978 }
979 
980 /*
981  * Update total_time_enabled and total_time_running for all events in a group.
982  */
983 static void update_group_times(struct perf_event *leader)
984 {
985         struct perf_event *event;
986 
987         update_event_times(leader);
988         list_for_each_entry(event, &leader->sibling_list, group_entry)
989                 update_event_times(event);
990 }
991 
992 static struct list_head *
993 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
994 {
995         if (event->attr.pinned)
996                 return &ctx->pinned_groups;
997         else
998                 return &ctx->flexible_groups;
999 }
1000 
1001 /*
1002  * Add a event from the lists for its context.
1003  * Must be called with ctx->mutex and ctx->lock held.
1004  */
1005 static void
1006 list_add_event(struct perf_event *event, struct perf_event_context *ctx)
1007 {
1008         WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
1009         event->attach_state |= PERF_ATTACH_CONTEXT;
1010 
1011         /*
1012          * If we're a stand alone event or group leader, we go to the context
1013          * list, group events are kept attached to the group so that
1014          * perf_group_detach can, at all times, locate all siblings.
1015          */
1016         if (event->group_leader == event) {
1017                 struct list_head *list;
1018 
1019                 if (is_software_event(event))
1020                         event->group_flags |= PERF_GROUP_SOFTWARE;
1021 
1022                 list = ctx_group_list(event, ctx);
1023                 list_add_tail(&event->group_entry, list);
1024         }
1025 
1026         if (is_cgroup_event(event))
1027                 ctx->nr_cgroups++;
1028 
1029         if (has_branch_stack(event))
1030                 ctx->nr_branch_stack++;
1031 
1032         list_add_rcu(&event->event_entry, &ctx->event_list);
1033         if (!ctx->nr_events)
1034                 perf_pmu_rotate_start(ctx->pmu);
1035         ctx->nr_events++;
1036         if (event->attr.inherit_stat)
1037                 ctx->nr_stat++;
1038 }
1039 
1040 /*
1041  * Initialize event state based on the perf_event_attr::disabled.
1042  */
1043 static inline void perf_event__state_init(struct perf_event *event)
1044 {
1045         event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
1046                                               PERF_EVENT_STATE_INACTIVE;
1047 }
1048 
1049 /*
1050  * Called at perf_event creation and when events are attached/detached from a
1051  * group.
1052  */
1053 static void perf_event__read_size(struct perf_event *event)
1054 {
1055         int entry = sizeof(u64); /* value */
1056         int size = 0;
1057         int nr = 1;
1058 
1059         if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1060                 size += sizeof(u64);
1061 
1062         if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1063                 size += sizeof(u64);
1064 
1065         if (event->attr.read_format & PERF_FORMAT_ID)
1066                 entry += sizeof(u64);
1067 
1068         if (event->attr.read_format & PERF_FORMAT_GROUP) {
1069                 nr += event->group_leader->nr_siblings;
1070                 size += sizeof(u64);
1071         }
1072 
1073         size += entry * nr;
1074         event->read_size = size;
1075 }
1076 
1077 static void perf_event__header_size(struct perf_event *event)
1078 {
1079         struct perf_sample_data *data;
1080         u64 sample_type = event->attr.sample_type;
1081         u16 size = 0;
1082 
1083         perf_event__read_size(event);
1084 
1085         if (sample_type & PERF_SAMPLE_IP)
1086                 size += sizeof(data->ip);
1087 
1088         if (sample_type & PERF_SAMPLE_ADDR)
1089                 size += sizeof(data->addr);
1090 
1091         if (sample_type & PERF_SAMPLE_PERIOD)
1092                 size += sizeof(data->period);
1093 
1094         if (sample_type & PERF_SAMPLE_WEIGHT)
1095                 size += sizeof(data->weight);
1096 
1097         if (sample_type & PERF_SAMPLE_READ)
1098                 size += event->read_size;
1099 
1100         if (sample_type & PERF_SAMPLE_DATA_SRC)
1101                 size += sizeof(data->data_src.val);
1102 
1103         event->header_size = size;
1104 }
1105 
1106 static void perf_event__id_header_size(struct perf_event *event)
1107 {
1108         struct perf_sample_data *data;
1109         u64 sample_type = event->attr.sample_type;
1110         u16 size = 0;
1111 
1112         if (sample_type & PERF_SAMPLE_TID)
1113                 size += sizeof(data->tid_entry);
1114 
1115         if (sample_type & PERF_SAMPLE_TIME)
1116                 size += sizeof(data->time);
1117 
1118         if (sample_type & PERF_SAMPLE_ID)
1119                 size += sizeof(data->id);
1120 
1121         if (sample_type & PERF_SAMPLE_STREAM_ID)
1122                 size += sizeof(data->stream_id);
1123 
1124         if (sample_type & PERF_SAMPLE_CPU)
1125                 size += sizeof(data->cpu_entry);
1126 
1127         event->id_header_size = size;
1128 }
1129 
1130 static void perf_group_attach(struct perf_event *event)
1131 {
1132         struct perf_event *group_leader = event->group_leader, *pos;
1133 
1134         /*
1135          * We can have double attach due to group movement in perf_event_open.
1136          */
1137         if (event->attach_state & PERF_ATTACH_GROUP)
1138                 return;
1139 
1140         event->attach_state |= PERF_ATTACH_GROUP;
1141 
1142         if (group_leader == event)
1143                 return;
1144 
1145         if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
1146                         !is_software_event(event))
1147                 group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
1148 
1149         list_add_tail(&event->group_entry, &group_leader->sibling_list);
1150         group_leader->nr_siblings++;
1151 
1152         perf_event__header_size(group_leader);
1153 
1154         list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
1155                 perf_event__header_size(pos);
1156 }
1157 
1158 /*
1159  * Remove a event from the lists for its context.
1160  * Must be called with ctx->mutex and ctx->lock held.
1161  */
1162 static void
1163 list_del_event(struct perf_event *event, struct perf_event_context *ctx)
1164 {
1165         struct perf_cpu_context *cpuctx;
1166         /*
1167          * We can have double detach due to exit/hot-unplug + close.
1168          */
1169         if (!(event->attach_state & PERF_ATTACH_CONTEXT))
1170                 return;
1171 
1172         event->attach_state &= ~PERF_ATTACH_CONTEXT;
1173 
1174         if (is_cgroup_event(event)) {
1175                 ctx->nr_cgroups--;
1176                 cpuctx = __get_cpu_context(ctx);
1177                 /*
1178                  * if there are no more cgroup events
1179                  * then cler cgrp to avoid stale pointer
1180                  * in update_cgrp_time_from_cpuctx()
1181                  */
1182                 if (!ctx->nr_cgroups)
1183                         cpuctx->cgrp = NULL;
1184         }
1185 
1186         if (has_branch_stack(event))
1187                 ctx->nr_branch_stack--;
1188 
1189         ctx->nr_events--;
1190         if (event->attr.inherit_stat)
1191                 ctx->nr_stat--;
1192 
1193         list_del_rcu(&event->event_entry);
1194 
1195         if (event->group_leader == event)
1196                 list_del_init(&event->group_entry);
1197 
1198         update_group_times(event);
1199 
1200         /*
1201          * If event was in error state, then keep it
1202          * that way, otherwise bogus counts will be
1203          * returned on read(). The only way to get out
1204          * of error state is by explicit re-enabling
1205          * of the event
1206          */
1207         if (event->state > PERF_EVENT_STATE_OFF)
1208                 event->state = PERF_EVENT_STATE_OFF;
1209 }
1210 
1211 static void perf_group_detach(struct perf_event *event)
1212 {
1213         struct perf_event *sibling, *tmp;
1214         struct list_head *list = NULL;
1215 
1216         /*
1217          * We can have double detach due to exit/hot-unplug + close.
1218          */
1219         if (!(event->attach_state & PERF_ATTACH_GROUP))
1220                 return;
1221 
1222         event->attach_state &= ~PERF_ATTACH_GROUP;
1223 
1224         /*
1225          * If this is a sibling, remove it from its group.
1226          */
1227         if (event->group_leader != event) {
1228                 list_del_init(&event->group_entry);
1229                 event->group_leader->nr_siblings--;
1230                 goto out;
1231         }
1232 
1233         if (!list_empty(&event->group_entry))
1234                 list = &event->group_entry;
1235 
1236         /*
1237          * If this was a group event with sibling events then
1238          * upgrade the siblings to singleton events by adding them
1239          * to whatever list we are on.
1240          */
1241         list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
1242                 if (list)
1243                         list_move_tail(&sibling->group_entry, list);
1244                 sibling->group_leader = sibling;
1245 
1246                 /* Inherit group flags from the previous leader */
1247                 sibling->group_flags = event->group_flags;
1248         }
1249 
1250 out:
1251         perf_event__header_size(event->group_leader);
1252 
1253         list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
1254                 perf_event__header_size(tmp);
1255 }
1256 
1257 static inline int
1258 event_filter_match(struct perf_event *event)
1259 {
1260         return (event->cpu == -1 || event->cpu == smp_processor_id())
1261             && perf_cgroup_match(event);
1262 }
1263 
1264 static void
1265 event_sched_out(struct perf_event *event,
1266                   struct perf_cpu_context *cpuctx,
1267                   struct perf_event_context *ctx)
1268 {
1269         u64 tstamp = perf_event_time(event);
1270         u64 delta;
1271         /*
1272          * An event which could not be activated because of
1273          * filter mismatch still needs to have its timings
1274          * maintained, otherwise bogus information is return
1275          * via read() for time_enabled, time_running:
1276          */
1277         if (event->state == PERF_EVENT_STATE_INACTIVE
1278             && !event_filter_match(event)) {
1279                 delta = tstamp - event->tstamp_stopped;
1280                 event->tstamp_running += delta;
1281                 event->tstamp_stopped = tstamp;
1282         }
1283 
1284         if (event->state != PERF_EVENT_STATE_ACTIVE)
1285                 return;
1286 
1287         event->state = PERF_EVENT_STATE_INACTIVE;
1288         if (event->pending_disable) {
1289                 event->pending_disable = 0;
1290                 event->state = PERF_EVENT_STATE_OFF;
1291         }
1292         event->tstamp_stopped = tstamp;
1293         event->pmu->del(event, 0);
1294         event->oncpu = -1;
1295 
1296         if (!is_software_event(event))
1297                 cpuctx->active_oncpu--;
1298         ctx->nr_active--;
1299         if (event->attr.freq && event->attr.sample_freq)
1300                 ctx->nr_freq--;
1301         if (event->attr.exclusive || !cpuctx->active_oncpu)
1302                 cpuctx->exclusive = 0;
1303 }
1304 
1305 static void
1306 group_sched_out(struct perf_event *group_event,
1307                 struct perf_cpu_context *cpuctx,
1308                 struct perf_event_context *ctx)
1309 {
1310         struct perf_event *event;
1311         int state = group_event->state;
1312 
1313         event_sched_out(group_event, cpuctx, ctx);
1314 
1315         /*
1316          * Schedule out siblings (if any):
1317          */
1318         list_for_each_entry(event, &group_event->sibling_list, group_entry)
1319                 event_sched_out(event, cpuctx, ctx);
1320 
1321         if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
1322                 cpuctx->exclusive = 0;
1323 }
1324 
1325 struct remove_event {
1326         struct perf_event *event;
1327         bool detach_group;
1328 };
1329 
1330 /*
1331  * Cross CPU call to remove a performance event
1332  *
1333  * We disable the event on the hardware level first. After that we
1334  * remove it from the context list.
1335  */
1336 static int __perf_remove_from_context(void *info)
1337 {
1338         struct remove_event *re = info;
1339         struct perf_event *event = re->event;
1340         struct perf_event_context *ctx = event->ctx;
1341         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1342 
1343         raw_spin_lock(&ctx->lock);
1344         event_sched_out(event, cpuctx, ctx);
1345         if (re->detach_group)
1346                 perf_group_detach(event);
1347         list_del_event(event, ctx);
1348         if (!ctx->nr_events && cpuctx->task_ctx == ctx) {
1349                 ctx->is_active = 0;
1350                 cpuctx->task_ctx = NULL;
1351         }
1352         raw_spin_unlock(&ctx->lock);
1353 
1354         return 0;
1355 }
1356 
1357 
1358 /*
1359  * Remove the event from a task's (or a CPU's) list of events.
1360  *
1361  * CPU events are removed with a smp call. For task events we only
1362  * call when the task is on a CPU.
1363  *
1364  * If event->ctx is a cloned context, callers must make sure that
1365  * every task struct that event->ctx->task could possibly point to
1366  * remains valid.  This is OK when called from perf_release since
1367  * that only calls us on the top-level context, which can't be a clone.
1368  * When called from perf_event_exit_task, it's OK because the
1369  * context has been detached from its task.
1370  */
1371 static void perf_remove_from_context(struct perf_event *event, bool detach_group)
1372 {
1373         struct perf_event_context *ctx = event->ctx;
1374         struct task_struct *task = ctx->task;
1375         struct remove_event re = {
1376                 .event = event,
1377                 .detach_group = detach_group,
1378         };
1379 
1380         lockdep_assert_held(&ctx->mutex);
1381 
1382         if (!task) {
1383                 /*
1384                  * Per cpu events are removed via an smp call and
1385                  * the removal is always successful.
1386                  */
1387                 cpu_function_call(event->cpu, __perf_remove_from_context, &re);
1388                 return;
1389         }
1390 
1391 retry:
1392         if (!task_function_call(task, __perf_remove_from_context, &re))
1393                 return;
1394 
1395         raw_spin_lock_irq(&ctx->lock);
1396         /*
1397          * If we failed to find a running task, but find the context active now
1398          * that we've acquired the ctx->lock, retry.
1399          */
1400         if (ctx->is_active) {
1401                 raw_spin_unlock_irq(&ctx->lock);
1402                 /*
1403                  * Reload the task pointer, it might have been changed by
1404                  * a concurrent perf_event_context_sched_out().
1405                  */
1406                 task = ctx->task;
1407                 goto retry;
1408         }
1409 
1410         /*
1411          * Since the task isn't running, its safe to remove the event, us
1412          * holding the ctx->lock ensures the task won't get scheduled in.
1413          */
1414         if (detach_group)
1415                 perf_group_detach(event);
1416         list_del_event(event, ctx);
1417         raw_spin_unlock_irq(&ctx->lock);
1418 }
1419 
1420 /*
1421  * Cross CPU call to disable a performance event
1422  */
1423 int __perf_event_disable(void *info)
1424 {
1425         struct perf_event *event = info;
1426         struct perf_event_context *ctx = event->ctx;
1427         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1428 
1429         /*
1430          * If this is a per-task event, need to check whether this
1431          * event's task is the current task on this cpu.
1432          *
1433          * Can trigger due to concurrent perf_event_context_sched_out()
1434          * flipping contexts around.
1435          */
1436         if (ctx->task && cpuctx->task_ctx != ctx)
1437                 return -EINVAL;
1438 
1439         raw_spin_lock(&ctx->lock);
1440 
1441         /*
1442          * If the event is on, turn it off.
1443          * If it is in error state, leave it in error state.
1444          */
1445         if (event->state >= PERF_EVENT_STATE_INACTIVE) {
1446                 update_context_time(ctx);
1447                 update_cgrp_time_from_event(event);
1448                 update_group_times(event);
1449                 if (event == event->group_leader)
1450                         group_sched_out(event, cpuctx, ctx);
1451                 else
1452                         event_sched_out(event, cpuctx, ctx);
1453                 event->state = PERF_EVENT_STATE_OFF;
1454         }
1455 
1456         raw_spin_unlock(&ctx->lock);
1457 
1458         return 0;
1459 }
1460 
1461 /*
1462  * Disable a event.
1463  *
1464  * If event->ctx is a cloned context, callers must make sure that
1465  * every task struct that event->ctx->task could possibly point to
1466  * remains valid.  This condition is satisifed when called through
1467  * perf_event_for_each_child or perf_event_for_each because they
1468  * hold the top-level event's child_mutex, so any descendant that
1469  * goes to exit will block in sync_child_event.
1470  * When called from perf_pending_event it's OK because event->ctx
1471  * is the current context on this CPU and preemption is disabled,
1472  * hence we can't get into perf_event_task_sched_out for this context.
1473  */
1474 void perf_event_disable(struct perf_event *event)
1475 {
1476         struct perf_event_context *ctx = event->ctx;
1477         struct task_struct *task = ctx->task;
1478 
1479         if (!task) {
1480                 /*
1481                  * Disable the event on the cpu that it's on
1482                  */
1483                 cpu_function_call(event->cpu, __perf_event_disable, event);
1484                 return;
1485         }
1486 
1487 retry:
1488         if (!task_function_call(task, __perf_event_disable, event))
1489                 return;
1490 
1491         raw_spin_lock_irq(&ctx->lock);
1492         /*
1493          * If the event is still active, we need to retry the cross-call.
1494          */
1495         if (event->state == PERF_EVENT_STATE_ACTIVE) {
1496                 raw_spin_unlock_irq(&ctx->lock);
1497                 /*
1498                  * Reload the task pointer, it might have been changed by
1499                  * a concurrent perf_event_context_sched_out().
1500                  */
1501                 task = ctx->task;
1502                 goto retry;
1503         }
1504 
1505         /*
1506          * Since we have the lock this context can't be scheduled
1507          * in, so we can change the state safely.
1508          */
1509         if (event->state == PERF_EVENT_STATE_INACTIVE) {
1510                 update_group_times(event);
1511                 event->state = PERF_EVENT_STATE_OFF;
1512         }
1513         raw_spin_unlock_irq(&ctx->lock);
1514 }
1515 EXPORT_SYMBOL_GPL(perf_event_disable);
1516 
1517 static void perf_set_shadow_time(struct perf_event *event,
1518                                  struct perf_event_context *ctx,
1519                                  u64 tstamp)
1520 {
1521         /*
1522          * use the correct time source for the time snapshot
1523          *
1524          * We could get by without this by leveraging the
1525          * fact that to get to this function, the caller
1526          * has most likely already called update_context_time()
1527          * and update_cgrp_time_xx() and thus both timestamp
1528          * are identical (or very close). Given that tstamp is,
1529          * already adjusted for cgroup, we could say that:
1530          *    tstamp - ctx->timestamp
1531          * is equivalent to
1532          *    tstamp - cgrp->timestamp.
1533          *
1534          * Then, in perf_output_read(), the calculation would
1535          * work with no changes because:
1536          * - event is guaranteed scheduled in
1537          * - no scheduled out in between
1538          * - thus the timestamp would be the same
1539          *
1540          * But this is a bit hairy.
1541          *
1542          * So instead, we have an explicit cgroup call to remain
1543          * within the time time source all along. We believe it
1544          * is cleaner and simpler to understand.
1545          */
1546         if (is_cgroup_event(event))
1547                 perf_cgroup_set_shadow_time(event, tstamp);
1548         else
1549                 event->shadow_ctx_time = tstamp - ctx->timestamp;
1550 }
1551 
1552 #define MAX_INTERRUPTS (~0ULL)
1553 
1554 static void perf_log_throttle(struct perf_event *event, int enable);
1555 
1556 static int
1557 event_sched_in(struct perf_event *event,
1558                  struct perf_cpu_context *cpuctx,
1559                  struct perf_event_context *ctx)
1560 {
1561         u64 tstamp = perf_event_time(event);
1562 
1563         if (event->state <= PERF_EVENT_STATE_OFF)
1564                 return 0;
1565 
1566         event->state = PERF_EVENT_STATE_ACTIVE;
1567         event->oncpu = smp_processor_id();
1568 
1569         /*
1570          * Unthrottle events, since we scheduled we might have missed several
1571          * ticks already, also for a heavily scheduling task there is little
1572          * guarantee it'll get a tick in a timely manner.
1573          */
1574         if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
1575                 perf_log_throttle(event, 1);
1576                 event->hw.interrupts = 0;
1577         }
1578 
1579         /*
1580          * The new state must be visible before we turn it on in the hardware:
1581          */
1582         smp_wmb();
1583 
1584         if (event->pmu->add(event, PERF_EF_START)) {
1585                 event->state = PERF_EVENT_STATE_INACTIVE;
1586                 event->oncpu = -1;
1587                 return -EAGAIN;
1588         }
1589 
1590         event->tstamp_running += tstamp - event->tstamp_stopped;
1591 
1592         perf_set_shadow_time(event, ctx, tstamp);
1593 
1594         if (!is_software_event(event))
1595                 cpuctx->active_oncpu++;
1596         ctx->nr_active++;
1597         if (event->attr.freq && event->attr.sample_freq)
1598                 ctx->nr_freq++;
1599 
1600         if (event->attr.exclusive)
1601                 cpuctx->exclusive = 1;
1602 
1603         return 0;
1604 }
1605 
1606 static int
1607 group_sched_in(struct perf_event *group_event,
1608                struct perf_cpu_context *cpuctx,
1609                struct perf_event_context *ctx)
1610 {
1611         struct perf_event *event, *partial_group = NULL;
1612         struct pmu *pmu = group_event->pmu;
1613         u64 now = ctx->time;
1614         bool simulate = false;
1615 
1616         if (group_event->state == PERF_EVENT_STATE_OFF)
1617                 return 0;
1618 
1619         pmu->start_txn(pmu);
1620 
1621         if (event_sched_in(group_event, cpuctx, ctx)) {
1622                 pmu->cancel_txn(pmu);
1623                 return -EAGAIN;
1624         }
1625 
1626         /*
1627          * Schedule in siblings as one group (if any):
1628          */
1629         list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1630                 if (event_sched_in(event, cpuctx, ctx)) {
1631                         partial_group = event;
1632                         goto group_error;
1633                 }
1634         }
1635 
1636         if (!pmu->commit_txn(pmu))
1637                 return 0;
1638 
1639 group_error:
1640         /*
1641          * Groups can be scheduled in as one unit only, so undo any
1642          * partial group before returning:
1643          * The events up to the failed event are scheduled out normally,
1644          * tstamp_stopped will be updated.
1645          *
1646          * The failed events and the remaining siblings need to have
1647          * their timings updated as if they had gone thru event_sched_in()
1648          * and event_sched_out(). This is required to get consistent timings
1649          * across the group. This also takes care of the case where the group
1650          * could never be scheduled by ensuring tstamp_stopped is set to mark
1651          * the time the event was actually stopped, such that time delta
1652          * calculation in update_event_times() is correct.
1653          */
1654         list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1655                 if (event == partial_group)
1656                         simulate = true;
1657 
1658                 if (simulate) {
1659                         event->tstamp_running += now - event->tstamp_stopped;
1660                         event->tstamp_stopped = now;
1661                 } else {
1662                         event_sched_out(event, cpuctx, ctx);
1663                 }
1664         }
1665         event_sched_out(group_event, cpuctx, ctx);
1666 
1667         pmu->cancel_txn(pmu);
1668 
1669         return -EAGAIN;
1670 }
1671 
1672 /*
1673  * Work out whether we can put this event group on the CPU now.
1674  */
1675 static int group_can_go_on(struct perf_event *event,
1676                            struct perf_cpu_context *cpuctx,
1677                            int can_add_hw)
1678 {
1679         /*
1680          * Groups consisting entirely of software events can always go on.
1681          */
1682         if (event->group_flags & PERF_GROUP_SOFTWARE)
1683                 return 1;
1684         /*
1685          * If an exclusive group is already on, no other hardware
1686          * events can go on.
1687          */
1688         if (cpuctx->exclusive)
1689                 return 0;
1690         /*
1691          * If this group is exclusive and there are already
1692          * events on the CPU, it can't go on.
1693          */
1694         if (event->attr.exclusive && cpuctx->active_oncpu)
1695                 return 0;
1696         /*
1697          * Otherwise, try to add it if all previous groups were able
1698          * to go on.
1699          */
1700         return can_add_hw;
1701 }
1702 
1703 static void add_event_to_ctx(struct perf_event *event,
1704                                struct perf_event_context *ctx)
1705 {
1706         u64 tstamp = perf_event_time(event);
1707 
1708         list_add_event(event, ctx);
1709         perf_group_attach(event);
1710         event->tstamp_enabled = tstamp;
1711         event->tstamp_running = tstamp;
1712         event->tstamp_stopped = tstamp;
1713 }
1714 
1715 static void task_ctx_sched_out(struct perf_event_context *ctx);
1716 static void
1717 ctx_sched_in(struct perf_event_context *ctx,
1718              struct perf_cpu_context *cpuctx,
1719              enum event_type_t event_type,
1720              struct task_struct *task);
1721 
1722 static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
1723                                 struct perf_event_context *ctx,
1724                                 struct task_struct *task)
1725 {
1726         cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
1727         if (ctx)
1728                 ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
1729         cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
1730         if (ctx)
1731                 ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
1732 }
1733 
1734 /*
1735  * Cross CPU call to install and enable a performance event
1736  *
1737  * Must be called with ctx->mutex held
1738  */
1739 static int  __perf_install_in_context(void *info)
1740 {
1741         struct perf_event *event = info;
1742         struct perf_event_context *ctx = event->ctx;
1743         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1744         struct perf_event_context *task_ctx = cpuctx->task_ctx;
1745         struct task_struct *task = current;
1746 
1747         perf_ctx_lock(cpuctx, task_ctx);
1748         perf_pmu_disable(cpuctx->ctx.pmu);
1749 
1750         /*
1751          * If there was an active task_ctx schedule it out.
1752          */
1753         if (task_ctx)
1754                 task_ctx_sched_out(task_ctx);
1755 
1756         /*
1757          * If the context we're installing events in is not the
1758          * active task_ctx, flip them.
1759          */
1760         if (ctx->task && task_ctx != ctx) {
1761                 if (task_ctx)
1762                         raw_spin_unlock(&task_ctx->lock);
1763                 raw_spin_lock(&ctx->lock);
1764                 task_ctx = ctx;
1765         }
1766 
1767         if (task_ctx) {
1768                 cpuctx->task_ctx = task_ctx;
1769                 task = task_ctx->task;
1770         }
1771 
1772         cpu_ctx_sched_out(cpuctx, EVENT_ALL);
1773 
1774         update_context_time(ctx);
1775         /*
1776          * update cgrp time only if current cgrp
1777          * matches event->cgrp. Must be done before
1778          * calling add_event_to_ctx()
1779          */
1780         update_cgrp_time_from_event(event);
1781 
1782         add_event_to_ctx(event, ctx);
1783 
1784         /*
1785          * Schedule everything back in
1786          */
1787         perf_event_sched_in(cpuctx, task_ctx, task);
1788 
1789         perf_pmu_enable(cpuctx->ctx.pmu);
1790         perf_ctx_unlock(cpuctx, task_ctx);
1791 
1792         return 0;
1793 }
1794 
1795 /*
1796  * Attach a performance event to a context
1797  *
1798  * First we add the event to the list with the hardware enable bit
1799  * in event->hw_config cleared.
1800  *
1801  * If the event is attached to a task which is on a CPU we use a smp
1802  * call to enable it in the task context. The task might have been
1803  * scheduled away, but we check this in the smp call again.
1804  */
1805 static void
1806 perf_install_in_context(struct perf_event_context *ctx,
1807                         struct perf_event *event,
1808                         int cpu)
1809 {
1810         struct task_struct *task = ctx->task;
1811 
1812         lockdep_assert_held(&ctx->mutex);
1813 
1814         event->ctx = ctx;
1815         if (event->cpu != -1)
1816                 event->cpu = cpu;
1817 
1818         if (!task) {
1819                 /*
1820                  * Per cpu events are installed via an smp call and
1821                  * the install is always successful.
1822                  */
1823                 cpu_function_call(cpu, __perf_install_in_context, event);
1824                 return;
1825         }
1826 
1827 retry:
1828         if (!task_function_call(task, __perf_install_in_context, event))
1829                 return;
1830 
1831         raw_spin_lock_irq(&ctx->lock);
1832         /*
1833          * If we failed to find a running task, but find the context active now
1834          * that we've acquired the ctx->lock, retry.
1835          */
1836         if (ctx->is_active) {
1837                 raw_spin_unlock_irq(&ctx->lock);
1838                 /*
1839                  * Reload the task pointer, it might have been changed by
1840                  * a concurrent perf_event_context_sched_out().
1841                  */
1842                 task = ctx->task;
1843                 goto retry;
1844         }
1845 
1846         /*
1847          * Since the task isn't running, its safe to add the event, us holding
1848          * the ctx->lock ensures the task won't get scheduled in.
1849          */
1850         add_event_to_ctx(event, ctx);
1851         raw_spin_unlock_irq(&ctx->lock);
1852 }
1853 
1854 /*
1855  * Put a event into inactive state and update time fields.
1856  * Enabling the leader of a group effectively enables all
1857  * the group members that aren't explicitly disabled, so we
1858  * have to update their ->tstamp_enabled also.
1859  * Note: this works for group members as well as group leaders
1860  * since the non-leader members' sibling_lists will be empty.
1861  */
1862 static void __perf_event_mark_enabled(struct perf_event *event)
1863 {
1864         struct perf_event *sub;
1865         u64 tstamp = perf_event_time(event);
1866 
1867         event->state = PERF_EVENT_STATE_INACTIVE;
1868         event->tstamp_enabled = tstamp - event->total_time_enabled;
1869         list_for_each_entry(sub, &event->sibling_list, group_entry) {
1870                 if (sub->state >= PERF_EVENT_STATE_INACTIVE)
1871                         sub->tstamp_enabled = tstamp - sub->total_time_enabled;
1872         }
1873 }
1874 
1875 /*
1876  * Cross CPU call to enable a performance event
1877  */
1878 static int __perf_event_enable(void *info)
1879 {
1880         struct perf_event *event = info;
1881         struct perf_event_context *ctx = event->ctx;
1882         struct perf_event *leader = event->group_leader;
1883         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1884         int err;
1885 
1886         /*
1887          * There's a time window between 'ctx->is_active' check
1888          * in perf_event_enable function and this place having:
1889          *   - IRQs on
1890          *   - ctx->lock unlocked
1891          *
1892          * where the task could be killed and 'ctx' deactivated
1893          * by perf_event_exit_task.
1894          */
1895         if (!ctx->is_active)
1896                 return -EINVAL;
1897 
1898         raw_spin_lock(&ctx->lock);
1899         update_context_time(ctx);
1900 
1901         if (event->state >= PERF_EVENT_STATE_INACTIVE)
1902                 goto unlock;
1903 
1904         /*
1905          * set current task's cgroup time reference point
1906          */
1907         perf_cgroup_set_timestamp(current, ctx);
1908 
1909         __perf_event_mark_enabled(event);
1910 
1911         if (!event_filter_match(event)) {
1912                 if (is_cgroup_event(event))
1913                         perf_cgroup_defer_enabled(event);
1914                 goto unlock;
1915         }
1916 
1917         /*
1918          * If the event is in a group and isn't the group leader,
1919          * then don't put it on unless the group is on.
1920          */
1921         if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
1922                 goto unlock;
1923 
1924         if (!group_can_go_on(event, cpuctx, 1)) {
1925                 err = -EEXIST;
1926         } else {
1927                 if (event == leader)
1928                         err = group_sched_in(event, cpuctx, ctx);
1929                 else
1930                         err = event_sched_in(event, cpuctx, ctx);
1931         }
1932 
1933         if (err) {
1934                 /*
1935                  * If this event can't go on and it's part of a
1936                  * group, then the whole group has to come off.
1937                  */
1938                 if (leader != event)
1939                         group_sched_out(leader, cpuctx, ctx);
1940                 if (leader->attr.pinned) {
1941                         update_group_times(leader);
1942                         leader->state = PERF_EVENT_STATE_ERROR;
1943                 }
1944         }
1945 
1946 unlock:
1947         raw_spin_unlock(&ctx->lock);
1948 
1949         return 0;
1950 }
1951 
1952 /*
1953  * Enable a event.
1954  *
1955  * If event->ctx is a cloned context, callers must make sure that
1956  * every task struct that event->ctx->task could possibly point to
1957  * remains valid.  This condition is satisfied when called through
1958  * perf_event_for_each_child or perf_event_for_each as described
1959  * for perf_event_disable.
1960  */
1961 void perf_event_enable(struct perf_event *event)
1962 {
1963         struct perf_event_context *ctx = event->ctx;
1964         struct task_struct *task = ctx->task;
1965 
1966         if (!task) {
1967                 /*
1968                  * Enable the event on the cpu that it's on
1969                  */
1970                 cpu_function_call(event->cpu, __perf_event_enable, event);
1971                 return;
1972         }
1973 
1974         raw_spin_lock_irq(&ctx->lock);
1975         if (event->state >= PERF_EVENT_STATE_INACTIVE)
1976                 goto out;
1977 
1978         /*
1979          * If the event is in error state, clear that first.
1980          * That way, if we see the event in error state below, we
1981          * know that it has gone back into error state, as distinct
1982          * from the task having been scheduled away before the
1983          * cross-call arrived.
1984          */
1985         if (event->state == PERF_EVENT_STATE_ERROR)
1986                 event->state = PERF_EVENT_STATE_OFF;
1987 
1988 retry:
1989         if (!ctx->is_active) {
1990                 __perf_event_mark_enabled(event);
1991                 goto out;
1992         }
1993 
1994         raw_spin_unlock_irq(&ctx->lock);
1995 
1996         if (!task_function_call(task, __perf_event_enable, event))
1997                 return;
1998 
1999         raw_spin_lock_irq(&ctx->lock);
2000 
2001         /*
2002          * If the context is active and the event is still off,
2003          * we need to retry the cross-call.
2004          */
2005         if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
2006                 /*
2007                  * task could have been flipped by a concurrent
2008                  * perf_event_context_sched_out()
2009                  */
2010                 task = ctx->task;
2011                 goto retry;
2012         }
2013 
2014 out:
2015         raw_spin_unlock_irq(&ctx->lock);
2016 }
2017 EXPORT_SYMBOL_GPL(perf_event_enable);
2018 
2019 int perf_event_refresh(struct perf_event *event, int refresh)
2020 {
2021         /*
2022          * not supported on inherited events
2023          */
2024         if (event->attr.inherit || !is_sampling_event(event))
2025                 return -EINVAL;
2026 
2027         atomic_add(refresh, &event->event_limit);
2028         perf_event_enable(event);
2029 
2030         return 0;
2031 }
2032 EXPORT_SYMBOL_GPL(perf_event_refresh);
2033 
2034 static void ctx_sched_out(struct perf_event_context *ctx,
2035                           struct perf_cpu_context *cpuctx,
2036                           enum event_type_t event_type)
2037 {
2038         struct perf_event *event;
2039         int is_active = ctx->is_active;
2040 
2041         ctx->is_active &= ~event_type;
2042         if (likely(!ctx->nr_events))
2043                 return;
2044 
2045         update_context_time(ctx);
2046         update_cgrp_time_from_cpuctx(cpuctx);
2047         if (!ctx->nr_active)
2048                 return;
2049 
2050         perf_pmu_disable(ctx->pmu);
2051         if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {
2052                 list_for_each_entry(event, &ctx->pinned_groups, group_entry)
2053                         group_sched_out(event, cpuctx, ctx);
2054         }
2055 
2056         if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {
2057                 list_for_each_entry(event, &ctx->flexible_groups, group_entry)
2058                         group_sched_out(event, cpuctx, ctx);
2059         }
2060         perf_pmu_enable(ctx->pmu);
2061 }
2062 
2063 /*
2064  * Test whether two contexts are equivalent, i.e. whether they
2065  * have both been cloned from the same version of the same context
2066  * and they both have the same number of enabled events.
2067  * If the number of enabled events is the same, then the set
2068  * of enabled events should be the same, because these are both
2069  * inherited contexts, therefore we can't access individual events
2070  * in them directly with an fd; we can only enable/disable all
2071  * events via prctl, or enable/disable all events in a family
2072  * via ioctl, which will have the same effect on both contexts.
2073  */
2074 static int context_equiv(struct perf_event_context *ctx1,
2075                          struct perf_event_context *ctx2)
2076 {
2077         return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
2078                 && ctx1->parent_gen == ctx2->parent_gen
2079                 && !ctx1->pin_count && !ctx2->pin_count;
2080 }
2081 
2082 static void __perf_event_sync_stat(struct perf_event *event,
2083                                      struct perf_event *next_event)
2084 {
2085         u64 value;
2086 
2087         if (!event->attr.inherit_stat)
2088                 return;
2089 
2090         /*
2091          * Update the event value, we cannot use perf_event_read()
2092          * because we're in the middle of a context switch and have IRQs
2093          * disabled, which upsets smp_call_function_single(), however
2094          * we know the event must be on the current CPU, therefore we
2095          * don't need to use it.
2096          */
2097         switch (event->state) {
2098         case PERF_EVENT_STATE_ACTIVE:
2099                 event->pmu->read(event);
2100                 /* fall-through */
2101 
2102         case PERF_EVENT_STATE_INACTIVE:
2103                 update_event_times(event);
2104                 break;
2105 
2106         default:
2107                 break;
2108         }
2109 
2110         /*
2111          * In order to keep per-task stats reliable we need to flip the event
2112          * values when we flip the contexts.
2113          */
2114         value = local64_read(&next_event->count);
2115         value = local64_xchg(&event->count, value);
2116         local64_set(&next_event->count, value);
2117 
2118         swap(event->total_time_enabled, next_event->total_time_enabled);
2119         swap(event->total_time_running, next_event->total_time_running);
2120 
2121         /*
2122          * Since we swizzled the values, update the user visible data too.
2123          */
2124         perf_event_update_userpage(event);
2125         perf_event_update_userpage(next_event);
2126 }
2127 
2128 static void perf_event_sync_stat(struct perf_event_context *ctx,
2129                                    struct perf_event_context *next_ctx)
2130 {
2131         struct perf_event *event, *next_event;
2132 
2133         if (!ctx->nr_stat)
2134                 return;
2135 
2136         update_context_time(ctx);
2137 
2138         event = list_first_entry(&ctx->event_list,
2139                                    struct perf_event, event_entry);
2140 
2141         next_event = list_first_entry(&next_ctx->event_list,
2142                                         struct perf_event, event_entry);
2143 
2144         while (&event->event_entry != &ctx->event_list &&
2145                &next_event->event_entry != &next_ctx->event_list) {
2146 
2147                 __perf_event_sync_stat(event, next_event);
2148 
2149                 event = list_next_entry(event, event_entry);
2150                 next_event = list_next_entry(next_event, event_entry);
2151         }
2152 }
2153 
2154 static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
2155                                          struct task_struct *next)
2156 {
2157         struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
2158         struct perf_event_context *next_ctx;
2159         struct perf_event_context *parent;
2160         struct perf_cpu_context *cpuctx;
2161         int do_switch = 1;
2162 
2163         if (likely(!ctx))
2164                 return;
2165 
2166         cpuctx = __get_cpu_context(ctx);
2167         if (!cpuctx->task_ctx)
2168                 return;
2169 
2170         rcu_read_lock();
2171         parent = rcu_dereference(ctx->parent_ctx);
2172         next_ctx = next->perf_event_ctxp[ctxn];
2173         if (parent && next_ctx &&
2174             rcu_dereference(next_ctx->parent_ctx) == parent) {
2175                 /*
2176                  * Looks like the two contexts are clones, so we might be
2177                  * able to optimize the context switch.  We lock both
2178                  * contexts and check that they are clones under the
2179                  * lock (including re-checking that neither has been
2180                  * uncloned in the meantime).  It doesn't matter which
2181                  * order we take the locks because no other cpu could
2182                  * be trying to lock both of these tasks.
2183                  */
2184                 raw_spin_lock(&ctx->lock);
2185                 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
2186                 if (context_equiv(ctx, next_ctx)) {
2187                         /*
2188                          * XXX do we need a memory barrier of sorts
2189                          * wrt to rcu_dereference() of perf_event_ctxp
2190                          */
2191                         task->perf_event_ctxp[ctxn] = next_ctx;
2192                         next->perf_event_ctxp[ctxn] = ctx;
2193                         ctx->task = next;
2194                         next_ctx->task = task;
2195                         do_switch = 0;
2196 
2197                         perf_event_sync_stat(ctx, next_ctx);
2198                 }
2199                 raw_spin_unlock(&next_ctx->lock);
2200                 raw_spin_unlock(&ctx->lock);
2201         }
2202         rcu_read_unlock();
2203 
2204         if (do_switch) {
2205                 raw_spin_lock(&ctx->lock);
2206                 ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2207                 cpuctx->task_ctx = NULL;
2208                 raw_spin_unlock(&ctx->lock);
2209         }
2210 }
2211 
2212 #define for_each_task_context_nr(ctxn)                                  \
2213         for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
2214 
2215 /*
2216  * Called from scheduler to remove the events of the current task,
2217  * with interrupts disabled.
2218  *
2219  * We stop each event and update the event value in event->count.
2220  *
2221  * This does not protect us against NMI, but disable()
2222  * sets the disabled bit in the control field of event _before_
2223  * accessing the event control register. If a NMI hits, then it will
2224  * not restart the event.
2225  */
2226 void __perf_event_task_sched_out(struct task_struct *task,
2227                                  struct task_struct *next)
2228 {
2229         int ctxn;
2230 
2231         for_each_task_context_nr(ctxn)
2232                 perf_event_context_sched_out(task, ctxn, next);
2233 
2234         /*
2235          * if cgroup events exist on this CPU, then we need
2236          * to check if we have to switch out PMU state.
2237          * cgroup event are system-wide mode only
2238          */
2239         if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
2240                 perf_cgroup_sched_out(task, next);
2241 }
2242 
2243 static void task_ctx_sched_out(struct perf_event_context *ctx)
2244 {
2245         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2246 
2247         if (!cpuctx->task_ctx)
2248                 return;
2249 
2250         if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2251                 return;
2252 
2253         ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2254         cpuctx->task_ctx = NULL;
2255 }
2256 
2257 /*
2258  * Called with IRQs disabled
2259  */
2260 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
2261                               enum event_type_t event_type)
2262 {
2263         ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
2264 }
2265 
2266 static void
2267 ctx_pinned_sched_in(struct perf_event_context *ctx,
2268                     struct perf_cpu_context *cpuctx)
2269 {
2270         struct perf_event *event;
2271 
2272         list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
2273                 if (event->state <= PERF_EVENT_STATE_OFF)
2274                         continue;
2275                 if (!event_filter_match(event))
2276                         continue;
2277 
2278                 /* may need to reset tstamp_enabled */
2279                 if (is_cgroup_event(event))
2280                         perf_cgroup_mark_enabled(event, ctx);
2281 
2282                 if (group_can_go_on(event, cpuctx, 1))
2283                         group_sched_in(event, cpuctx, ctx);
2284 
2285                 /*
2286                  * If this pinned group hasn't been scheduled,
2287                  * put it in error state.
2288                  */
2289                 if (event->state == PERF_EVENT_STATE_INACTIVE) {
2290                         update_group_times(event);
2291                         event->state = PERF_EVENT_STATE_ERROR;
2292                 }
2293         }
2294 }
2295 
2296 static void
2297 ctx_flexible_sched_in(struct perf_event_context *ctx,
2298                       struct perf_cpu_context *cpuctx)
2299 {
2300         struct perf_event *event;
2301         int can_add_hw = 1;
2302 
2303         list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
2304                 /* Ignore events in OFF or ERROR state */
2305                 if (event->state <= PERF_EVENT_STATE_OFF)
2306                         continue;
2307                 /*
2308                  * Listen to the 'cpu' scheduling filter constraint
2309                  * of events:
2310                  */
2311                 if (!event_filter_match(event))
2312                         continue;
2313 
2314                 /* may need to reset tstamp_enabled */
2315                 if (is_cgroup_event(event))
2316                         perf_cgroup_mark_enabled(event, ctx);
2317 
2318                 if (group_can_go_on(event, cpuctx, can_add_hw)) {
2319                         if (group_sched_in(event, cpuctx, ctx))
2320                                 can_add_hw = 0;
2321                 }
2322         }
2323 }
2324 
2325 static void
2326 ctx_sched_in(struct perf_event_context *ctx,
2327              struct perf_cpu_context *cpuctx,
2328              enum event_type_t event_type,
2329              struct task_struct *task)
2330 {
2331         u64 now;
2332         int is_active = ctx->is_active;
2333 
2334         ctx->is_active |= event_type;
2335         if (likely(!ctx->nr_events))
2336                 return;
2337 
2338         now = perf_clock();
2339         ctx->timestamp = now;
2340         perf_cgroup_set_timestamp(task, ctx);
2341         /*
2342          * First go through the list and put on any pinned groups
2343          * in order to give them the best chance of going on.
2344          */
2345         if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))
2346                 ctx_pinned_sched_in(ctx, cpuctx);
2347 
2348         /* Then walk through the lower prio flexible groups */
2349         if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))
2350                 ctx_flexible_sched_in(ctx, cpuctx);
2351 }
2352 
2353 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
2354                              enum event_type_t event_type,
2355                              struct task_struct *task)
2356 {
2357         struct perf_event_context *ctx = &cpuctx->ctx;
2358 
2359         ctx_sched_in(ctx, cpuctx, event_type, task);
2360 }
2361 
2362 static void perf_event_context_sched_in(struct perf_event_context *ctx,
2363                                         struct task_struct *task)
2364 {
2365         struct perf_cpu_context *cpuctx;
2366 
2367         cpuctx = __get_cpu_context(ctx);
2368         if (cpuctx->task_ctx == ctx)
2369                 return;
2370 
2371         perf_ctx_lock(cpuctx, ctx);
2372         perf_pmu_disable(ctx->pmu);
2373         /*
2374          * We want to keep the following priority order:
2375          * cpu pinned (that don't need to move), task pinned,
2376          * cpu flexible, task flexible.
2377          */
2378         cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2379 
2380         if (ctx->nr_events)
2381                 cpuctx->task_ctx = ctx;
2382 
2383         perf_event_sched_in(cpuctx, cpuctx->task_ctx, task);
2384 
2385         perf_pmu_enable(ctx->pmu);
2386         perf_ctx_unlock(cpuctx, ctx);
2387 
2388         /*
2389          * Since these rotations are per-cpu, we need to ensure the
2390          * cpu-context we got scheduled on is actually rotating.
2391          */
2392         perf_pmu_rotate_start(ctx->pmu);
2393 }
2394 
2395 /*
2396  * When sampling the branck stack in system-wide, it may be necessary
2397  * to flush the stack on context switch. This happens when the branch
2398  * stack does not tag its entries with the pid of the current task.
2399  * Otherwise it becomes impossible to associate a branch entry with a
2400  * task. This ambiguity is more likely to appear when the branch stack
2401  * supports priv level filtering and the user sets it to monitor only
2402  * at the user level (which could be a useful measurement in system-wide
2403  * mode). In that case, the risk is high of having a branch stack with
2404  * branch from multiple tasks. Flushing may mean dropping the existing
2405  * entries or stashing them somewhere in the PMU specific code layer.
2406  *
2407  * This function provides the context switch callback to the lower code
2408  * layer. It is invoked ONLY when there is at least one system-wide context
2409  * with at least one active event using taken branch sampling.
2410  */
2411 static void perf_branch_stack_sched_in(struct task_struct *prev,
2412                                        struct task_struct *task)
2413 {
2414         struct perf_cpu_context *cpuctx;
2415         struct pmu *pmu;
2416         unsigned long flags;
2417 
2418         /* no need to flush branch stack if not changing task */
2419         if (prev == task)
2420                 return;
2421 
2422         local_irq_save(flags);
2423 
2424         rcu_read_lock();
2425 
2426         list_for_each_entry_rcu(pmu, &pmus, entry) {
2427                 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
2428 
2429                 /*
2430                  * check if the context has at least one
2431                  * event using PERF_SAMPLE_BRANCH_STACK
2432                  */
2433                 if (cpuctx->ctx.nr_branch_stack > 0
2434                     && pmu->flush_branch_stack) {
2435 
2436                         pmu = cpuctx->ctx.pmu;
2437 
2438                         perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2439 
2440                         perf_pmu_disable(pmu);
2441 
2442                         pmu->flush_branch_stack();
2443 
2444                         perf_pmu_enable(pmu);
2445 
2446                         perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2447                 }
2448         }
2449 
2450         rcu_read_unlock();
2451 
2452         local_irq_restore(flags);
2453 }
2454 
2455 /*
2456  * Called from scheduler to add the events of the current task
2457  * with interrupts disabled.
2458  *
2459  * We restore the event value and then enable it.
2460  *
2461  * This does not protect us against NMI, but enable()
2462  * sets the enabled bit in the control field of event _before_
2463  * accessing the event control register. If a NMI hits, then it will
2464  * keep the event running.
2465  */
2466 void __perf_event_task_sched_in(struct task_struct *prev,
2467                                 struct task_struct *task)
2468 {
2469         struct perf_event_context *ctx;
2470         int ctxn;
2471 
2472         for_each_task_context_nr(ctxn) {
2473                 ctx = task->perf_event_ctxp[ctxn];
2474                 if (likely(!ctx))
2475                         continue;
2476 
2477                 perf_event_context_sched_in(ctx, task);
2478         }
2479         /*
2480          * if cgroup events exist on this CPU, then we need
2481          * to check if we have to switch in PMU state.
2482          * cgroup event are system-wide mode only
2483          */
2484         if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
2485                 perf_cgroup_sched_in(prev, task);
2486 
2487         /* check for system-wide branch_stack events */
2488         if (atomic_read(&__get_cpu_var(perf_branch_stack_events)))
2489                 perf_branch_stack_sched_in(prev, task);
2490 }
2491 
2492 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
2493 {
2494         u64 frequency = event->attr.sample_freq;
2495         u64 sec = NSEC_PER_SEC;
2496         u64 divisor, dividend;
2497 
2498         int count_fls, nsec_fls, frequency_fls, sec_fls;
2499 
2500         count_fls = fls64(count);
2501         nsec_fls = fls64(nsec);
2502         frequency_fls = fls64(frequency);
2503         sec_fls = 30;
2504 
2505         /*
2506          * We got @count in @nsec, with a target of sample_freq HZ
2507          * the target period becomes:
2508          *
2509          *             @count * 10^9
2510          * period = -------------------
2511          *          @nsec * sample_freq
2512          *
2513          */
2514 
2515         /*
2516          * Reduce accuracy by one bit such that @a and @b converge
2517          * to a similar magnitude.
2518          */
2519 #define REDUCE_FLS(a, b)                \
2520 do {                                    \
2521         if (a##_fls > b##_fls) {        \
2522                 a >>= 1;                \
2523                 a##_fls--;              \
2524         } else {                        \
2525                 b >>= 1;                \
2526                 b##_fls--;              \
2527         }                               \
2528 } while (0)
2529 
2530         /*
2531          * Reduce accuracy until either term fits in a u64, then proceed with
2532          * the other, so that finally we can do a u64/u64 division.
2533          */
2534         while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
2535                 REDUCE_FLS(nsec, frequency);
2536                 REDUCE_FLS(sec, count);
2537         }
2538 
2539         if (count_fls + sec_fls > 64) {
2540                 divisor = nsec * frequency;
2541 
2542                 while (count_fls + sec_fls > 64) {
2543                         REDUCE_FLS(count, sec);
2544                         divisor >>= 1;
2545                 }
2546 
2547                 dividend = count * sec;
2548         } else {
2549                 dividend = count * sec;
2550 
2551                 while (nsec_fls + frequency_fls > 64) {
2552                         REDUCE_FLS(nsec, frequency);
2553                         dividend >>= 1;
2554                 }
2555 
2556                 divisor = nsec * frequency;
2557         }
2558 
2559         if (!divisor)
2560                 return dividend;
2561 
2562         return div64_u64(dividend, divisor);
2563 }
2564 
2565 static DEFINE_PER_CPU(int, perf_throttled_count);
2566 static DEFINE_PER_CPU(u64, perf_throttled_seq);
2567 
2568 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
2569 {
2570         struct hw_perf_event *hwc = &event->hw;
2571         s64 period, sample_period;
2572         s64 delta;
2573 
2574         period = perf_calculate_period(event, nsec, count);
2575 
2576         delta = (s64)(period - hwc->sample_period);
2577         delta = (delta + 7) / 8; /* low pass filter */
2578 
2579         sample_period = hwc->sample_period + delta;
2580 
2581         if (!sample_period)
2582                 sample_period = 1;
2583 
2584         hwc->sample_period = sample_period;
2585 
2586         if (local64_read(&hwc->period_left) > 8*sample_period) {
2587                 if (disable)
2588                         event->pmu->stop(event, PERF_EF_UPDATE);
2589 
2590                 local64_set(&hwc->period_left, 0);
2591 
2592                 if (disable)
2593                         event->pmu->start(event, PERF_EF_RELOAD);
2594         }
2595 }
2596 
2597 /*
2598  * combine freq adjustment with unthrottling to avoid two passes over the
2599  * events. At the same time, make sure, having freq events does not change
2600  * the rate of unthrottling as that would introduce bias.
2601  */
2602 static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
2603                                            int needs_unthr)
2604 {
2605         struct perf_event *event;
2606         struct hw_perf_event *hwc;
2607         u64 now, period = TICK_NSEC;
2608         s64 delta;
2609 
2610         /*
2611          * only need to iterate over all events iff:
2612          * - context have events in frequency mode (needs freq adjust)
2613          * - there are events to unthrottle on this cpu
2614          */
2615         if (!(ctx->nr_freq || needs_unthr))
2616                 return;
2617 
2618         raw_spin_lock(&ctx->lock);
2619         perf_pmu_disable(ctx->pmu);
2620 
2621         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
2622                 if (event->state != PERF_EVENT_STATE_ACTIVE)
2623                         continue;
2624 
2625                 if (!event_filter_match(event))
2626                         continue;
2627 
2628                 hwc = &event->hw;
2629 
2630                 if (needs_unthr && hwc->interrupts == MAX_INTERRUPTS) {
2631                         hwc->interrupts = 0;
2632                         perf_log_throttle(event, 1);
2633                         event->pmu->start(event, 0);
2634                 }
2635 
2636                 if (!event->attr.freq || !event->attr.sample_freq)
2637                         continue;
2638 
2639                 /*
2640                  * stop the event and update event->count
2641                  */
2642                 event->pmu->stop(event, PERF_EF_UPDATE);
2643 
2644                 now = local64_read(&event->count);
2645                 delta = now - hwc->freq_count_stamp;
2646                 hwc->freq_count_stamp = now;
2647 
2648                 /*
2649                  * restart the event
2650                  * reload only if value has changed
2651                  * we have stopped the event so tell that
2652                  * to perf_adjust_period() to avoid stopping it
2653                  * twice.
2654                  */
2655                 if (delta > 0)
2656                         perf_adjust_period(event, period, delta, false);
2657 
2658                 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
2659         }
2660 
2661         perf_pmu_enable(ctx->pmu);
2662         raw_spin_unlock(&ctx->lock);
2663 }
2664 
2665 /*
2666  * Round-robin a context's events:
2667  */
2668 static void rotate_ctx(struct perf_event_context *ctx)
2669 {
2670         /*
2671          * Rotate the first entry last of non-pinned groups. Rotation might be
2672          * disabled by the inheritance code.
2673          */
2674         if (!ctx->rotate_disable)
2675                 list_rotate_left(&ctx->flexible_groups);
2676 }
2677 
2678 /*
2679  * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
2680  * because they're strictly cpu affine and rotate_start is called with IRQs
2681  * disabled, while rotate_context is called from IRQ context.
2682  */
2683 static void perf_rotate_context(struct perf_cpu_context *cpuctx)
2684 {
2685         struct perf_event_context *ctx = NULL;
2686         int rotate = 0, remove = 1;
2687 
2688         if (cpuctx->ctx.nr_events) {
2689                 remove = 0;
2690                 if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
2691                         rotate = 1;
2692         }
2693 
2694         ctx = cpuctx->task_ctx;
2695         if (ctx && ctx->nr_events) {
2696                 remove = 0;
2697                 if (ctx->nr_events != ctx->nr_active)
2698                         rotate = 1;
2699         }
2700 
2701         if (!rotate)
2702                 goto done;
2703 
2704         perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2705         perf_pmu_disable(cpuctx->ctx.pmu);
2706 
2707         cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2708         if (ctx)
2709                 ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
2710 
2711         rotate_ctx(&cpuctx->ctx);
2712         if (ctx)
2713                 rotate_ctx(ctx);
2714 
2715         perf_event_sched_in(cpuctx, ctx, current);
2716 
2717         perf_pmu_enable(cpuctx->ctx.pmu);
2718         perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2719 done:
2720         if (remove)
2721                 list_del_init(&cpuctx->rotation_list);
2722 }
2723 
2724 #ifdef CONFIG_NO_HZ_FULL
2725 bool perf_event_can_stop_tick(void)
2726 {
2727         if (list_empty(&__get_cpu_var(rotation_list)))
2728                 return true;
2729         else
2730                 return false;
2731 }
2732 #endif
2733 
2734 void perf_event_task_tick(void)
2735 {
2736         struct list_head *head = &__get_cpu_var(rotation_list);
2737         struct perf_cpu_context *cpuctx, *tmp;
2738         struct perf_event_context *ctx;
2739         int throttled;
2740 
2741         WARN_ON(!irqs_disabled());
2742 
2743         __this_cpu_inc(perf_throttled_seq);
2744         throttled = __this_cpu_xchg(perf_throttled_count, 0);
2745 
2746         list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
2747                 ctx = &cpuctx->ctx;
2748                 perf_adjust_freq_unthr_context(ctx, throttled);
2749 
2750                 ctx = cpuctx->task_ctx;
2751                 if (ctx)
2752                         perf_adjust_freq_unthr_context(ctx, throttled);
2753 
2754                 if (cpuctx->jiffies_interval == 1 ||
2755                                 !(jiffies % cpuctx->jiffies_interval))
2756                         perf_rotate_context(cpuctx);
2757         }
2758 }
2759 
2760 static int event_enable_on_exec(struct perf_event *event,
2761                                 struct perf_event_context *ctx)
2762 {
2763         if (!event->attr.enable_on_exec)
2764                 return 0;
2765 
2766         event->attr.enable_on_exec = 0;
2767         if (event->state >= PERF_EVENT_STATE_INACTIVE)
2768                 return 0;
2769 
2770         __perf_event_mark_enabled(event);
2771 
2772         return 1;
2773 }
2774 
2775 /*
2776  * Enable all of a task's events that have been marked enable-on-exec.
2777  * This expects task == current.
2778  */
2779 static void perf_event_enable_on_exec(struct perf_event_context *ctx)
2780 {
2781         struct perf_event *event;
2782         unsigned long flags;
2783         int enabled = 0;
2784         int ret;
2785 
2786         local_irq_save(flags);
2787         if (!ctx || !ctx->nr_events)
2788                 goto out;
2789 
2790         /*
2791          * We must ctxsw out cgroup events to avoid conflict
2792          * when invoking perf_task_event_sched_in() later on
2793          * in this function. Otherwise we end up trying to
2794          * ctxswin cgroup events which are already scheduled
2795          * in.
2796          */
2797         perf_cgroup_sched_out(current, NULL);
2798 
2799         raw_spin_lock(&ctx->lock);
2800         task_ctx_sched_out(ctx);
2801 
2802         list_for_each_entry(event, &ctx->event_list, event_entry) {
2803                 ret = event_enable_on_exec(event, ctx);
2804                 if (ret)
2805                         enabled = 1;
2806         }
2807 
2808         /*
2809          * Unclone this context if we enabled any event.
2810          */
2811         if (enabled)
2812                 unclone_ctx(ctx);
2813 
2814         raw_spin_unlock(&ctx->lock);
2815 
2816         /*
2817          * Also calls ctxswin for cgroup events, if any:
2818          */
2819         perf_event_context_sched_in(ctx, ctx->task);
2820 out:
2821         local_irq_restore(flags);
2822 }
2823 
2824 /*
2825  * Cross CPU call to read the hardware event
2826  */
2827 static void __perf_event_read(void *info)
2828 {
2829         struct perf_event *event = info;
2830         struct perf_event_context *ctx = event->ctx;
2831         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2832 
2833         /*
2834          * If this is a task context, we need to check whether it is
2835          * the current task context of this cpu.  If not it has been
2836          * scheduled out before the smp call arrived.  In that case
2837          * event->count would have been updated to a recent sample
2838          * when the event was scheduled out.
2839          */
2840         if (ctx->task && cpuctx->task_ctx != ctx)
2841                 return;
2842 
2843         raw_spin_lock(&ctx->lock);
2844         if (ctx->is_active) {
2845                 update_context_time(ctx);
2846                 update_cgrp_time_from_event(event);
2847         }
2848         update_event_times(event);
2849         if (event->state == PERF_EVENT_STATE_ACTIVE)
2850                 event->pmu->read(event);
2851         raw_spin_unlock(&ctx->lock);
2852 }
2853 
2854 static inline u64 perf_event_count(struct perf_event *event)
2855 {
2856         return local64_read(&event->count) + atomic64_read(&event->child_count);
2857 }
2858 
2859 static u64 perf_event_read(struct perf_event *event)
2860 {
2861         /*
2862          * If event is enabled and currently active on a CPU, update the
2863          * value in the event structure:
2864          */
2865         if (event->state == PERF_EVENT_STATE_ACTIVE) {
2866                 smp_call_function_single(event->oncpu,
2867                                          __perf_event_read, event, 1);
2868         } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
2869                 struct perf_event_context *ctx = event->ctx;
2870                 unsigned long flags;
2871 
2872                 raw_spin_lock_irqsave(&ctx->lock, flags);
2873                 /*
2874                  * may read while context is not active
2875                  * (e.g., thread is blocked), in that case
2876                  * we cannot update context time
2877                  */
2878                 if (ctx->is_active) {
2879                         update_context_time(ctx);
2880                         update_cgrp_time_from_event(event);
2881                 }
2882                 update_event_times(event);
2883                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
2884         }
2885 
2886         return perf_event_count(event);
2887 }
2888 
2889 /*
2890  * Initialize the perf_event context in a task_struct:
2891  */
2892 static void __perf_event_init_context(struct perf_event_context *ctx)
2893 {
2894         raw_spin_lock_init(&ctx->lock);
2895         mutex_init(&ctx->mutex);
2896         INIT_LIST_HEAD(&ctx->pinned_groups);
2897         INIT_LIST_HEAD(&ctx->flexible_groups);
2898         INIT_LIST_HEAD(&ctx->event_list);
2899         atomic_set(&ctx->refcount, 1);
2900 }
2901 
2902 static struct perf_event_context *
2903 alloc_perf_context(struct pmu *pmu, struct task_struct *task)
2904 {
2905         struct perf_event_context *ctx;
2906 
2907         ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
2908         if (!ctx)
2909                 return NULL;
2910 
2911         __perf_event_init_context(ctx);
2912         if (task) {
2913                 ctx->task = task;
2914                 get_task_struct(task);
2915         }
2916         ctx->pmu = pmu;
2917 
2918         return ctx;
2919 }
2920 
2921 static struct task_struct *
2922 find_lively_task_by_vpid(pid_t vpid)
2923 {
2924         struct task_struct *task;
2925         int err;
2926 
2927         rcu_read_lock();
2928         if (!vpid)
2929                 task = current;
2930         else
2931                 task = find_task_by_vpid(vpid);
2932         if (task)
2933                 get_task_struct(task);
2934         rcu_read_unlock();
2935 
2936         if (!task)
2937                 return ERR_PTR(-ESRCH);
2938 
2939         /* Reuse ptrace permission checks for now. */
2940         err = -EACCES;
2941         if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS))
2942                 goto errout;
2943 
2944         return task;
2945 errout:
2946         put_task_struct(task);
2947         return ERR_PTR(err);
2948 
2949 }
2950 
2951 /*
2952  * Returns a matching context with refcount and pincount.
2953  */
2954 static struct perf_event_context *
2955 find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
2956 {
2957         struct perf_event_context *ctx;
2958         struct perf_cpu_context *cpuctx;
2959         unsigned long flags;
2960         int ctxn, err;
2961 
2962         if (!task) {
2963                 /* Must be root to operate on a CPU event: */
2964                 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
2965                         return ERR_PTR(-EACCES);
2966 
2967                 /*
2968                  * We could be clever and allow to attach a event to an
2969                  * offline CPU and activate it when the CPU comes up, but
2970                  * that's for later.
2971                  */
2972                 if (!cpu_online(cpu))
2973                         return ERR_PTR(-ENODEV);
2974 
2975                 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
2976                 ctx = &cpuctx->ctx;
2977                 get_ctx(ctx);
2978                 ++ctx->pin_count;
2979 
2980                 return ctx;
2981         }
2982 
2983         err = -EINVAL;
2984         ctxn = pmu->task_ctx_nr;
2985         if (ctxn < 0)
2986                 goto errout;
2987 
2988 retry:
2989         ctx = perf_lock_task_context(task, ctxn, &flags);
2990         if (ctx) {
2991                 unclone_ctx(ctx);
2992                 ++ctx->pin_count;
2993                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
2994         } else {
2995                 ctx = alloc_perf_context(pmu, task);
2996                 err = -ENOMEM;
2997                 if (!ctx)
2998                         goto errout;
2999 
3000                 err = 0;
3001                 mutex_lock(&task->perf_event_mutex);
3002                 /*
3003                  * If it has already passed perf_event_exit_task().
3004                  * we must see PF_EXITING, it takes this mutex too.
3005                  */
3006                 if (task->flags & PF_EXITING)
3007                         err = -ESRCH;
3008                 else if (task->perf_event_ctxp[ctxn])
3009                         err = -EAGAIN;
3010                 else {
3011                         get_ctx(ctx);
3012                         ++ctx->pin_count;
3013                         rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
3014                 }
3015                 mutex_unlock(&task->perf_event_mutex);
3016 
3017                 if (unlikely(err)) {
3018                         put_ctx(ctx);
3019 
3020                         if (err == -EAGAIN)
3021                                 goto retry;
3022                         goto errout;
3023                 }
3024         }
3025 
3026         return ctx;
3027 
3028 errout:
3029         return ERR_PTR(err);
3030 }
3031 
3032 static void perf_event_free_filter(struct perf_event *event);
3033 
3034 static void free_event_rcu(struct rcu_head *head)
3035 {
3036         struct perf_event *event;
3037 
3038         event = container_of(head, struct perf_event, rcu_head);
3039         if (event->ns)
3040                 put_pid_ns(event->ns);
3041         perf_event_free_filter(event);
3042         kfree(event);
3043 }
3044 
3045 static void ring_buffer_put(struct ring_buffer *rb);
3046 static void ring_buffer_detach(struct perf_event *event, struct ring_buffer *rb);
3047 
3048 static void free_event(struct perf_event *event)
3049 {
3050         irq_work_sync(&event->pending);
3051 
3052         if (!event->parent) {
3053                 if (event->attach_state & PERF_ATTACH_TASK)
3054                         static_key_slow_dec_deferred(&perf_sched_events);
3055                 if (event->attr.mmap || event->attr.mmap_data)
3056                         atomic_dec(&nr_mmap_events);
3057                 if (event->attr.comm)
3058                         atomic_dec(&nr_comm_events);
3059                 if (event->attr.task)
3060                         atomic_dec(&nr_task_events);
3061                 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
3062                         put_callchain_buffers();
3063                 if (is_cgroup_event(event)) {
3064                         atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
3065                         static_key_slow_dec_deferred(&perf_sched_events);
3066                 }
3067 
3068                 if (has_branch_stack(event)) {
3069                         static_key_slow_dec_deferred(&perf_sched_events);
3070                         /* is system-wide event */
3071                         if (!(event->attach_state & PERF_ATTACH_TASK)) {
3072                                 atomic_dec(&per_cpu(perf_branch_stack_events,
3073                                                     event->cpu));
3074                         }
3075                 }
3076         }
3077 
3078         if (event->rb) {
3079                 struct ring_buffer *rb;
3080 
3081                 /*
3082                  * Can happen when we close an event with re-directed output.
3083                  *
3084                  * Since we have a 0 refcount, perf_mmap_close() will skip
3085                  * over us; possibly making our ring_buffer_put() the last.
3086                  */
3087                 mutex_lock(&event->mmap_mutex);
3088                 rb = event->rb;
3089                 if (rb) {
3090                         rcu_assign_pointer(event->rb, NULL);
3091                         ring_buffer_detach(event, rb);
3092                         ring_buffer_put(rb); /* could be last */
3093                 }
3094                 mutex_unlock(&event->mmap_mutex);
3095         }
3096 
3097         if (is_cgroup_event(event))
3098                 perf_detach_cgroup(event);
3099 
3100         if (event->destroy)
3101                 event->destroy(event);
3102 
3103         if (event->ctx)
3104                 put_ctx(event->ctx);
3105 
3106         call_rcu(&event->rcu_head, free_event_rcu);
3107 }
3108 
3109 int perf_event_release_kernel(struct perf_event *event)
3110 {
3111         struct perf_event_context *ctx = event->ctx;
3112 
3113         WARN_ON_ONCE(ctx->parent_ctx);
3114         /*
3115          * There are two ways this annotation is useful:
3116          *
3117          *  1) there is a lock recursion from perf_event_exit_task
3118          *     see the comment there.
3119          *
3120          *  2) there is a lock-inversion with mmap_sem through
3121          *     perf_event_read_group(), which takes faults while
3122          *     holding ctx->mutex, however this is called after
3123          *     the last filedesc died, so there is no possibility
3124          *     to trigger the AB-BA case.
3125          */
3126         mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
3127         perf_remove_from_context(event, true);
3128         mutex_unlock(&ctx->mutex);
3129 
3130         free_event(event);
3131 
3132         return 0;
3133 }
3134 EXPORT_SYMBOL_GPL(perf_event_release_kernel);
3135 
3136 /*
3137  * Called when the last reference to the file is gone.
3138  */
3139 static void put_event(struct perf_event *event)
3140 {
3141         struct task_struct *owner;
3142 
3143         if (!atomic_long_dec_and_test(&event->refcount))
3144                 return;
3145 
3146         rcu_read_lock();
3147         owner = ACCESS_ONCE(event->owner);
3148         /*
3149          * Matches the smp_wmb() in perf_event_exit_task(). If we observe
3150          * !owner it means the list deletion is complete and we can indeed
3151          * free this event, otherwise we need to serialize on
3152          * owner->perf_event_mutex.
3153          */
3154         smp_read_barrier_depends();
3155         if (owner) {
3156                 /*
3157                  * Since delayed_put_task_struct() also drops the last
3158                  * task reference we can safely take a new reference
3159                  * while holding the rcu_read_lock().
3160                  */
3161                 get_task_struct(owner);
3162         }
3163         rcu_read_unlock();
3164 
3165         if (owner) {
3166                 mutex_lock(&owner->perf_event_mutex);
3167                 /*
3168                  * We have to re-check the event->owner field, if it is cleared
3169                  * we raced with perf_event_exit_task(), acquiring the mutex
3170                  * ensured they're done, and we can proceed with freeing the
3171                  * event.
3172                  */
3173                 if (event->owner)
3174                         list_del_init(&event->owner_entry);
3175                 mutex_unlock(&owner->perf_event_mutex);
3176                 put_task_struct(owner);
3177         }
3178 
3179         perf_event_release_kernel(event);
3180 }
3181 
3182 static int perf_release(struct inode *inode, struct file *file)
3183 {
3184         put_event(file->private_data);
3185         return 0;
3186 }
3187 
3188 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
3189 {
3190         struct perf_event *child;
3191         u64 total = 0;
3192 
3193         *enabled = 0;
3194         *running = 0;
3195 
3196         mutex_lock(&event->child_mutex);
3197         total += perf_event_read(event);
3198         *enabled += event->total_time_enabled +
3199                         atomic64_read(&event->child_total_time_enabled);
3200         *running += event->total_time_running +
3201                         atomic64_read(&event->child_total_time_running);
3202 
3203         list_for_each_entry(child, &event->child_list, child_list) {
3204                 total += perf_event_read(child);
3205                 *enabled += child->total_time_enabled;
3206                 *running += child->total_time_running;
3207         }
3208         mutex_unlock(&event->child_mutex);
3209 
3210         return total;
3211 }
3212 EXPORT_SYMBOL_GPL(perf_event_read_value);
3213 
3214 static int perf_event_read_group(struct perf_event *event,
3215                                    u64 read_format, char __user *buf)
3216 {
3217         struct perf_event *leader = event->group_leader, *sub;
3218         int n = 0, size = 0, ret = -EFAULT;
3219         struct perf_event_context *ctx = leader->ctx;
3220         u64 values[5];
3221         u64 count, enabled, running;
3222 
3223         mutex_lock(&ctx->mutex);
3224         count = perf_event_read_value(leader, &enabled, &running);
3225 
3226         values[n++] = 1 + leader->nr_siblings;
3227         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3228                 values[n++] = enabled;
3229         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3230                 values[n++] = running;
3231         values[n++] = count;
3232         if (read_format & PERF_FORMAT_ID)
3233                 values[n++] = primary_event_id(leader);
3234 
3235         size = n * sizeof(u64);
3236 
3237         if (copy_to_user(buf, values, size))
3238                 goto unlock;
3239 
3240         ret = size;
3241 
3242         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3243                 n = 0;
3244 
3245                 values[n++] = perf_event_read_value(sub, &enabled, &running);
3246                 if (read_format & PERF_FORMAT_ID)
3247                         values[n++] = primary_event_id(sub);
3248 
3249                 size = n * sizeof(u64);
3250 
3251                 if (copy_to_user(buf + ret, values, size)) {
3252                         ret = -EFAULT;
3253                         goto unlock;
3254                 }
3255 
3256                 ret += size;
3257         }
3258 unlock:
3259         mutex_unlock(&ctx->mutex);
3260 
3261         return ret;
3262 }
3263 
3264 static int perf_event_read_one(struct perf_event *event,
3265                                  u64 read_format, char __user *buf)
3266 {
3267         u64 enabled, running;
3268         u64 values[4];
3269         int n = 0;
3270 
3271         values[n++] = perf_event_read_value(event, &enabled, &running);
3272         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3273                 values[n++] = enabled;
3274         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3275                 values[n++] = running;
3276         if (read_format & PERF_FORMAT_ID)
3277                 values[n++] = primary_event_id(event);
3278 
3279         if (copy_to_user(buf, values, n * sizeof(u64)))
3280                 return -EFAULT;
3281 
3282         return n * sizeof(u64);
3283 }
3284 
3285 /*
3286  * Read the performance event - simple non blocking version for now
3287  */
3288 static ssize_t
3289 perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
3290 {
3291         u64 read_format = event->attr.read_format;
3292         int ret;
3293 
3294         /*
3295          * Return end-of-file for a read on a event that is in
3296          * error state (i.e. because it was pinned but it couldn't be
3297          * scheduled on to the CPU at some point).
3298          */
3299         if (event->state == PERF_EVENT_STATE_ERROR)
3300                 return 0;
3301 
3302         if (count < event->read_size)
3303                 return -ENOSPC;
3304 
3305         WARN_ON_ONCE(event->ctx->parent_ctx);
3306         if (read_format & PERF_FORMAT_GROUP)
3307                 ret = perf_event_read_group(event, read_format, buf);
3308         else
3309                 ret = perf_event_read_one(event, read_format, buf);
3310 
3311         return ret;
3312 }
3313 
3314 static ssize_t
3315 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
3316 {
3317         struct perf_event *event = file->private_data;
3318 
3319         return perf_read_hw(event, buf, count);
3320 }
3321 
3322 static unsigned int perf_poll(struct file *file, poll_table *wait)
3323 {
3324         struct perf_event *event = file->private_data;
3325         struct ring_buffer *rb;
3326         unsigned int events = POLL_HUP;
3327 
3328         /*
3329          * Pin the event->rb by taking event->mmap_mutex; otherwise
3330          * perf_event_set_output() can swizzle our rb and make us miss wakeups.
3331          */
3332         mutex_lock(&event->mmap_mutex);
3333         rb = event->rb;
3334         if (rb)
3335                 events = atomic_xchg(&rb->poll, 0);
3336         mutex_unlock(&event->mmap_mutex);
3337 
3338         poll_wait(file, &event->waitq, wait);
3339 
3340         return events;
3341 }
3342 
3343 static void perf_event_reset(struct perf_event *event)
3344 {
3345         (void)perf_event_read(event);
3346         local64_set(&event->count, 0);
3347         perf_event_update_userpage(event);
3348 }
3349 
3350 /*
3351  * Holding the top-level event's child_mutex means that any
3352  * descendant process that has inherited this event will block
3353  * in sync_child_event if it goes to exit, thus satisfying the
3354  * task existence requirements of perf_event_enable/disable.
3355  */
3356 static void perf_event_for_each_child(struct perf_event *event,
3357                                         void (*func)(struct perf_event *))
3358 {
3359         struct perf_event *child;
3360 
3361         WARN_ON_ONCE(event->ctx->parent_ctx);
3362         mutex_lock(&event->child_mutex);
3363         func(event);
3364         list_for_each_entry(child, &event->child_list, child_list)
3365                 func(child);
3366         mutex_unlock(&event->child_mutex);
3367 }
3368 
3369 static void perf_event_for_each(struct perf_event *event,
3370                                   void (*func)(struct perf_event *))
3371 {
3372         struct perf_event_context *ctx = event->ctx;
3373         struct perf_event *sibling;
3374 
3375         WARN_ON_ONCE(ctx->parent_ctx);
3376         mutex_lock(&ctx->mutex);
3377         event = event->group_leader;
3378 
3379         perf_event_for_each_child(event, func);
3380         list_for_each_entry(sibling, &event->sibling_list, group_entry)
3381                 perf_event_for_each_child(sibling, func);
3382         mutex_unlock(&ctx->mutex);
3383 }
3384 
3385 static int perf_event_period(struct perf_event *event, u64 __user *arg)
3386 {
3387         struct perf_event_context *ctx = event->ctx;
3388         int ret = 0;
3389         u64 value;
3390 
3391         if (!is_sampling_event(event))
3392                 return -EINVAL;
3393 
3394         if (copy_from_user(&value, arg, sizeof(value)))
3395                 return -EFAULT;
3396 
3397         if (!value)
3398                 return -EINVAL;
3399 
3400         raw_spin_lock_irq(&ctx->lock);
3401         if (event->attr.freq) {
3402                 if (value > sysctl_perf_event_sample_rate) {
3403                         ret = -EINVAL;
3404                         goto unlock;
3405                 }
3406 
3407                 event->attr.sample_freq = value;
3408         } else {
3409                 event->attr.sample_period = value;
3410                 event->hw.sample_period = value;
3411         }
3412 unlock:
3413         raw_spin_unlock_irq(&ctx->lock);
3414 
3415         return ret;
3416 }
3417 
3418 static const struct file_operations perf_fops;
3419 
3420 static inline int perf_fget_light(int fd, struct fd *p)
3421 {
3422         struct fd f = fdget(fd);
3423         if (!f.file)
3424                 return -EBADF;
3425 
3426         if (f.file->f_op != &perf_fops) {
3427                 fdput(f);
3428                 return -EBADF;
3429         }
3430         *p = f;
3431         return 0;
3432 }
3433 
3434 static int perf_event_set_output(struct perf_event *event,
3435                                  struct perf_event *output_event);
3436 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
3437 
3438 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
3439 {
3440         struct perf_event *event = file->private_data;
3441         void (*func)(struct perf_event *);
3442         u32 flags = arg;
3443 
3444         switch (cmd) {
3445         case PERF_EVENT_IOC_ENABLE:
3446                 func = perf_event_enable;
3447                 break;
3448         case PERF_EVENT_IOC_DISABLE:
3449                 func = perf_event_disable;
3450                 break;
3451         case PERF_EVENT_IOC_RESET:
3452                 func = perf_event_reset;
3453                 break;
3454 
3455         case PERF_EVENT_IOC_REFRESH:
3456                 return perf_event_refresh(event, arg);
3457 
3458         case PERF_EVENT_IOC_PERIOD:
3459                 return perf_event_period(event, (u64 __user *)arg);
3460 
3461         case PERF_EVENT_IOC_SET_OUTPUT:
3462         {
3463                 int ret;
3464                 if (arg != -1) {
3465                         struct perf_event *output_event;
3466                         struct fd output;
3467                         ret = perf_fget_light(arg, &output);
3468                         if (ret)
3469                                 return ret;
3470                         output_event = output.file->private_data;
3471                         ret = perf_event_set_output(event, output_event);
3472                         fdput(output);
3473                 } else {
3474                         ret = perf_event_set_output(event, NULL);
3475                 }
3476                 return ret;
3477         }
3478 
3479         case PERF_EVENT_IOC_SET_FILTER:
3480                 return perf_event_set_filter(event, (void __user *)arg);
3481 
3482         default:
3483                 return -ENOTTY;
3484         }
3485 
3486         if (flags & PERF_IOC_FLAG_GROUP)
3487                 perf_event_for_each(event, func);
3488         else
3489                 perf_event_for_each_child(event, func);
3490 
3491         return 0;
3492 }
3493 
3494 #ifdef CONFIG_COMPAT
3495 static long perf_compat_ioctl(struct file *file, unsigned int cmd,
3496                                 unsigned long arg)
3497 {
3498         switch (_IOC_NR(cmd)) {
3499         case _IOC_NR(PERF_EVENT_IOC_SET_FILTER):
3500                 /* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */
3501                 if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) {
3502                         cmd &= ~IOCSIZE_MASK;
3503                         cmd |= sizeof(void *) << IOCSIZE_SHIFT;
3504                 }
3505                 break;
3506         }
3507         return perf_ioctl(file, cmd, arg);
3508 }
3509 #else
3510 # define perf_compat_ioctl NULL
3511 #endif
3512 
3513 int perf_event_task_enable(void)
3514 {
3515         struct perf_event *event;
3516 
3517         mutex_lock(&current->perf_event_mutex);
3518         list_for_each_entry(event, &current->perf_event_list, owner_entry)
3519                 perf_event_for_each_child(event, perf_event_enable);
3520         mutex_unlock(&current->perf_event_mutex);
3521 
3522         return 0;
3523 }
3524 
3525 int perf_event_task_disable(void)
3526 {
3527         struct perf_event *event;
3528 
3529         mutex_lock(&current->perf_event_mutex);
3530         list_for_each_entry(event, &current->perf_event_list, owner_entry)
3531                 perf_event_for_each_child(event, perf_event_disable);
3532         mutex_unlock(&current->perf_event_mutex);
3533 
3534         return 0;
3535 }
3536 
3537 static int perf_event_index(struct perf_event *event)
3538 {
3539         if (event->hw.state & PERF_HES_STOPPED)
3540                 return 0;
3541 
3542         if (event->state != PERF_EVENT_STATE_ACTIVE)
3543                 return 0;
3544 
3545         return event->pmu->event_idx(event);
3546 }
3547 
3548 static void calc_timer_values(struct perf_event *event,
3549                                 u64 *now,
3550                                 u64 *enabled,
3551                                 u64 *running)
3552 {
3553         u64 ctx_time;
3554 
3555         *now = perf_clock();
3556         ctx_time = event->shadow_ctx_time + *now;
3557         *enabled = ctx_time - event->tstamp_enabled;
3558         *running = ctx_time - event->tstamp_running;
3559 }
3560 
3561 void __weak arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
3562 {
3563 }
3564 
3565 /*
3566  * Callers need to ensure there can be no nesting of this function, otherwise
3567  * the seqlock logic goes bad. We can not serialize this because the arch
3568  * code calls this from NMI context.
3569  */
3570 void perf_event_update_userpage(struct perf_event *event)
3571 {
3572         struct perf_event_mmap_page *userpg;
3573         struct ring_buffer *rb;
3574         u64 enabled, running, now;
3575 
3576         rcu_read_lock();
3577         /*
3578          * compute total_time_enabled, total_time_running
3579          * based on snapshot values taken when the event
3580          * was last scheduled in.
3581          *
3582          * we cannot simply called update_context_time()
3583          * because of locking issue as we can be called in
3584          * NMI context
3585          */
3586         calc_timer_values(event, &now, &enabled, &running);
3587         rb = rcu_dereference(event->rb);
3588         if (!rb)
3589                 goto unlock;
3590 
3591         userpg = rb->user_page;
3592 
3593         /*
3594          * Disable preemption so as to not let the corresponding user-space
3595          * spin too long if we get preempted.
3596          */
3597         preempt_disable();
3598         ++userpg->lock;
3599         barrier();
3600         userpg->index = perf_event_index(event);
3601         userpg->offset = perf_event_count(event);
3602         if (userpg->index)
3603                 userpg->offset -= local64_read(&event->hw.prev_count);
3604 
3605         userpg->time_enabled = enabled +
3606                         atomic64_read(&event->child_total_time_enabled);
3607 
3608         userpg->time_running = running +
3609                         atomic64_read(&event->child_total_time_running);
3610 
3611         arch_perf_update_userpage(userpg, now);
3612 
3613         barrier();
3614         ++userpg->lock;
3615         preempt_enable();
3616 unlock:
3617         rcu_read_unlock();
3618 }
3619 
3620 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
3621 {
3622         struct perf_event *event = vma->vm_file->private_data;
3623         struct ring_buffer *rb;
3624         int ret = VM_FAULT_SIGBUS;
3625 
3626         if (vmf->flags & FAULT_FLAG_MKWRITE) {
3627                 if (vmf->pgoff == 0)
3628                         ret = 0;
3629                 return ret;
3630         }
3631 
3632         rcu_read_lock();
3633         rb = rcu_dereference(event->rb);
3634         if (!rb)
3635                 goto unlock;
3636 
3637         if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
3638                 goto unlock;
3639 
3640         vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
3641         if (!vmf->page)
3642                 goto unlock;
3643 
3644         get_page(vmf->page);
3645         vmf->page->mapping = vma->vm_file->f_mapping;
3646         vmf->page->index   = vmf->pgoff;
3647 
3648         ret = 0;
3649 unlock:
3650         rcu_read_unlock();
3651 
3652         return ret;
3653 }
3654 
3655 static void ring_buffer_attach(struct perf_event *event,
3656                                struct ring_buffer *rb)
3657 {
3658         unsigned long flags;
3659 
3660         if (!list_empty(&event->rb_entry))
3661                 return;
3662 
3663         spin_lock_irqsave(&rb->event_lock, flags);
3664         if (list_empty(&event->rb_entry))
3665                 list_add(&event->rb_entry, &rb->event_list);
3666         spin_unlock_irqrestore(&rb->event_lock, flags);
3667 }
3668 
3669 static void ring_buffer_detach(struct perf_event *event, struct ring_buffer *rb)
3670 {
3671         unsigned long flags;
3672 
3673         if (list_empty(&event->rb_entry))
3674                 return;
3675 
3676         spin_lock_irqsave(&rb->event_lock, flags);
3677         list_del_init(&event->rb_entry);
3678         wake_up_all(&event->waitq);
3679         spin_unlock_irqrestore(&rb->event_lock, flags);
3680 }
3681 
3682 static void ring_buffer_wakeup(struct perf_event *event)
3683 {
3684         struct ring_buffer *rb;
3685 
3686         rcu_read_lock();
3687         rb = rcu_dereference(event->rb);
3688         if (rb) {
3689                 list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
3690                         wake_up_all(&event->waitq);
3691         }
3692         rcu_read_unlock();
3693 }
3694 
3695 static void rb_free_rcu(struct rcu_head *rcu_head)
3696 {
3697         struct ring_buffer *rb;
3698 
3699         rb = container_of(rcu_head, struct ring_buffer, rcu_head);
3700         rb_free(rb);
3701 }
3702 
3703 static struct ring_buffer *ring_buffer_get(struct perf_event *event)
3704 {
3705         struct ring_buffer *rb;
3706 
3707         rcu_read_lock();
3708         rb = rcu_dereference(event->rb);
3709         if (rb) {
3710                 if (!atomic_inc_not_zero(&rb->refcount))
3711                         rb = NULL;
3712         }
3713         rcu_read_unlock();
3714 
3715         return rb;
3716 }
3717 
3718 static void ring_buffer_put(struct ring_buffer *rb)
3719 {
3720         if (!atomic_dec_and_test(&rb->refcount))
3721                 return;
3722 
3723         WARN_ON_ONCE(!list_empty(&rb->event_list));
3724 
3725         call_rcu(&rb->rcu_head, rb_free_rcu);
3726 }
3727 
3728 static void perf_mmap_open(struct vm_area_struct *vma)
3729 {
3730         struct perf_event *event = vma->vm_file->private_data;
3731 
3732         atomic_inc(&event->mmap_count);
3733         atomic_inc(&event->rb->mmap_count);
3734 }
3735 
3736 /*
3737  * A buffer can be mmap()ed multiple times; either directly through the same
3738  * event, or through other events by use of perf_event_set_output().
3739  *
3740  * In order to undo the VM accounting done by perf_mmap() we need to destroy
3741  * the buffer here, where we still have a VM context. This means we need
3742  * to detach all events redirecting to us.
3743  */
3744 static void perf_mmap_close(struct vm_area_struct *vma)
3745 {
3746         struct perf_event *event = vma->vm_file->private_data;
3747 
3748         struct ring_buffer *rb = event->rb;
3749         struct user_struct *mmap_user = rb->mmap_user;
3750         int mmap_locked = rb->mmap_locked;
3751         unsigned long size = perf_data_size(rb);
3752 
3753         atomic_dec(&rb->mmap_count);
3754 
3755         if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
3756                 return;
3757 
3758         /* Detach current event from the buffer. */
3759         rcu_assign_pointer(event->rb, NULL);
3760         ring_buffer_detach(event, rb);
3761         mutex_unlock(&event->mmap_mutex);
3762 
3763         /* If there's still other mmap()s of this buffer, we're done. */
3764         if (atomic_read(&rb->mmap_count)) {
3765                 ring_buffer_put(rb); /* can't be last */
3766                 return;
3767         }
3768 
3769         /*
3770          * No other mmap()s, detach from all other events that might redirect
3771          * into the now unreachable buffer. Somewhat complicated by the
3772          * fact that rb::event_lock otherwise nests inside mmap_mutex.
3773          */
3774 again:
3775         rcu_read_lock();
3776         list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
3777                 if (!atomic_long_inc_not_zero(&event->refcount)) {
3778                         /*
3779                          * This event is en-route to free_event() which will
3780                          * detach it and remove it from the list.
3781                          */
3782                         continue;
3783                 }
3784                 rcu_read_unlock();
3785 
3786                 mutex_lock(&event->mmap_mutex);
3787                 /*
3788                  * Check we didn't race with perf_event_set_output() which can
3789                  * swizzle the rb from under us while we were waiting to
3790                  * acquire mmap_mutex.
3791                  *
3792                  * If we find a different rb; ignore this event, a next
3793                  * iteration will no longer find it on the list. We have to
3794                  * still restart the iteration to make sure we're not now
3795                  * iterating the wrong list.
3796                  */
3797                 if (event->rb == rb) {
3798                         rcu_assign_pointer(event->rb, NULL);
3799                         ring_buffer_detach(event, rb);
3800                         ring_buffer_put(rb); /* can't be last, we still have one */
3801                 }
3802                 mutex_unlock(&event->mmap_mutex);
3803                 put_event(event);
3804 
3805                 /*
3806                  * Restart the iteration; either we're on the wrong list or
3807                  * destroyed its integrity by doing a deletion.
3808                  */
3809                 goto again;
3810         }
3811         rcu_read_unlock();
3812 
3813         /*
3814          * It could be there's still a few 0-ref events on the list; they'll
3815          * get cleaned up by free_event() -- they'll also still have their
3816          * ref on the rb and will free it whenever they are done with it.
3817          *
3818          * Aside from that, this buffer is 'fully' detached and unmapped,
3819          * undo the VM accounting.
3820          */
3821 
3822         atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm);
3823         vma->vm_mm->pinned_vm -= mmap_locked;
3824         free_uid(mmap_user);
3825 
3826         ring_buffer_put(rb); /* could be last */
3827 }
3828 
3829 static const struct vm_operations_struct perf_mmap_vmops = {
3830         .open           = perf_mmap_open,
3831         .close          = perf_mmap_close,
3832         .fault          = perf_mmap_fault,
3833         .page_mkwrite   = perf_mmap_fault,
3834 };
3835 
3836 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
3837 {
3838         struct perf_event *event = file->private_data;
3839         unsigned long user_locked, user_lock_limit;
3840         struct user_struct *user = current_user();
3841         unsigned long locked, lock_limit;
3842         struct ring_buffer *rb;
3843         unsigned long vma_size;
3844         unsigned long nr_pages;
3845         long user_extra, extra;
3846         int ret = 0, flags = 0;
3847 
3848         /*
3849          * Don't allow mmap() of inherited per-task counters. This would
3850          * create a performance issue due to all children writing to the
3851          * same rb.
3852          */
3853         if (event->cpu == -1 && event->attr.inherit)
3854                 return -EINVAL;
3855 
3856         if (!(vma->vm_flags & VM_SHARED))
3857                 return -EINVAL;
3858 
3859         vma_size = vma->vm_end - vma->vm_start;
3860         nr_pages = (vma_size / PAGE_SIZE) - 1;
3861 
3862         /*
3863          * If we have rb pages ensure they're a power-of-two number, so we
3864          * can do bitmasks instead of modulo.
3865          */
3866         if (nr_pages != 0 && !is_power_of_2(nr_pages))
3867                 return -EINVAL;
3868 
3869         if (vma_size != PAGE_SIZE * (1 + nr_pages))
3870                 return -EINVAL;
3871 
3872         if (vma->vm_pgoff != 0)
3873                 return -EINVAL;
3874 
3875         WARN_ON_ONCE(event->ctx->parent_ctx);
3876 again:
3877         mutex_lock(&event->mmap_mutex);
3878         if (event->rb) {
3879                 if (event->rb->nr_pages != nr_pages) {
3880                         ret = -EINVAL;
3881                         goto unlock;
3882                 }
3883 
3884                 if (!atomic_inc_not_zero(&event->rb->mmap_count)) {
3885                         /*
3886                          * Raced against perf_mmap_close() through
3887                          * perf_event_set_output(). Try again, hope for better
3888                          * luck.
3889                          */
3890                         mutex_unlock(&event->mmap_mutex);
3891                         goto again;
3892                 }
3893 
3894                 goto unlock;
3895         }
3896 
3897         user_extra = nr_pages + 1;
3898         user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
3899 
3900         /*
3901          * Increase the limit linearly with more CPUs:
3902          */
3903         user_lock_limit *= num_online_cpus();
3904 
3905         user_locked = atomic_long_read(&user->locked_vm) + user_extra;
3906 
3907         extra = 0;
3908         if (user_locked > user_lock_limit)
3909                 extra = user_locked - user_lock_limit;
3910 
3911         lock_limit = rlimit(RLIMIT_MEMLOCK);
3912         lock_limit >>= PAGE_SHIFT;
3913         locked = vma->vm_mm->pinned_vm + extra;
3914 
3915         if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
3916                 !capable(CAP_IPC_LOCK)) {
3917                 ret = -EPERM;
3918                 goto unlock;
3919         }
3920 
3921         WARN_ON(event->rb);
3922 
3923         if (vma->vm_flags & VM_WRITE)
3924                 flags |= RING_BUFFER_WRITABLE;
3925 
3926         rb = rb_alloc(nr_pages, 
3927                 event->attr.watermark ? event->attr.wakeup_watermark : 0,
3928                 event->cpu, flags);
3929 
3930         if (!rb) {
3931                 ret = -ENOMEM;
3932                 goto unlock;
3933         }
3934 
3935         atomic_set(&rb->mmap_count, 1);
3936         rb->mmap_locked = extra;
3937         rb->mmap_user = get_current_user();
3938 
3939         atomic_long_add(user_extra, &user->locked_vm);
3940         vma->vm_mm->pinned_vm += extra;
3941 
3942         ring_buffer_attach(event, rb);
3943         rcu_assign_pointer(event->rb, rb);
3944 
3945         perf_event_update_userpage(event);
3946 
3947 unlock:
3948         if (!ret)
3949                 atomic_inc(&event->mmap_count);
3950         mutex_unlock(&event->mmap_mutex);
3951 
3952         /*
3953          * Since pinned accounting is per vm we cannot allow fork() to copy our
3954          * vma.
3955          */
3956         vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;
3957         vma->vm_ops = &perf_mmap_vmops;
3958 
3959         return ret;
3960 }
3961 
3962 static int perf_fasync(int fd, struct file *filp, int on)
3963 {
3964         struct inode *inode = file_inode(filp);
3965         struct perf_event *event = filp->private_data;
3966         int retval;
3967 
3968         mutex_lock(&inode->i_mutex);
3969         retval = fasync_helper(fd, filp, on, &event->fasync);
3970         mutex_unlock(&inode->i_mutex);
3971 
3972         if (retval < 0)
3973                 return retval;
3974 
3975         return 0;
3976 }
3977 
3978 static const struct file_operations perf_fops = {
3979         .llseek                 = no_llseek,
3980         .release                = perf_release,
3981         .read                   = perf_read,
3982         .poll                   = perf_poll,
3983         .unlocked_ioctl         = perf_ioctl,
3984         .compat_ioctl           = perf_compat_ioctl,
3985         .mmap                   = perf_mmap,
3986         .fasync                 = perf_fasync,
3987 };
3988 
3989 /*
3990  * Perf event wakeup
3991  *
3992  * If there's data, ensure we set the poll() state and publish everything
3993  * to user-space before waking everybody up.
3994  */
3995 
3996 static inline struct fasync_struct **perf_event_fasync(struct perf_event *event)
3997 {
3998         /* only the parent has fasync state */
3999         if (event->parent)
4000                 event = event->parent;
4001         return &event->fasync;
4002 }
4003 
4004 void perf_event_wakeup(struct perf_event *event)
4005 {
4006         ring_buffer_wakeup(event);
4007 
4008         if (event->pending_kill) {
4009                 kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill);
4010                 event->pending_kill = 0;
4011         }
4012 }
4013 
4014 static void perf_pending_event(struct irq_work *entry)
4015 {
4016         struct perf_event *event = container_of(entry,
4017                         struct perf_event, pending);
4018         int rctx;
4019 
4020         rctx = perf_swevent_get_recursion_context();
4021         /*
4022          * If we 'fail' here, that's OK, it means recursion is already disabled
4023          * and we won't recurse 'further'.
4024          */
4025 
4026         if (event->pending_disable) {
4027                 event->pending_disable = 0;
4028                 __perf_event_disable(event);
4029         }
4030 
4031         if (event->pending_wakeup) {
4032                 event->pending_wakeup = 0;
4033                 perf_event_wakeup(event);
4034         }
4035 
4036         if (rctx >= 0)
4037                 perf_swevent_put_recursion_context(rctx);
4038 }
4039 
4040 /*
4041  * We assume there is only KVM supporting the callbacks.
4042  * Later on, we might change it to a list if there is
4043  * another virtualization implementation supporting the callbacks.
4044  */
4045 struct perf_guest_info_callbacks *perf_guest_cbs;
4046 
4047 int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
4048 {
4049         perf_guest_cbs = cbs;
4050         return 0;
4051 }
4052 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
4053 
4054 int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
4055 {
4056         perf_guest_cbs = NULL;
4057         return 0;
4058 }
4059 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
4060 
4061 static void
4062 perf_output_sample_regs(struct perf_output_handle *handle,
4063                         struct pt_regs *regs, u64 mask)
4064 {
4065         int bit;
4066 
4067         for_each_set_bit(bit, (const unsigned long *) &mask,
4068                          sizeof(mask) * BITS_PER_BYTE) {
4069                 u64 val;
4070 
4071                 val = perf_reg_value(regs, bit);
4072                 perf_output_put(handle, val);
4073         }
4074 }
4075 
4076 static void perf_sample_regs_user(struct perf_regs_user *regs_user,
4077                                   struct pt_regs *regs)
4078 {
4079         if (!user_mode(regs)) {
4080                 if (current->mm)
4081                         regs = task_pt_regs(current);
4082                 else
4083                         regs = NULL;
4084         }
4085 
4086         if (regs) {
4087                 regs_user->regs = regs;
4088                 regs_user->abi  = perf_reg_abi(current);
4089         }
4090 }
4091 
4092 /*
4093  * Get remaining task size from user stack pointer.
4094  *
4095  * It'd be better to take stack vma map and limit this more
4096  * precisly, but there's no way to get it safely under interrupt,
4097  * so using TASK_SIZE as limit.
4098  */
4099 static u64 perf_ustack_task_size(struct pt_regs *regs)
4100 {
4101         unsigned long addr = perf_user_stack_pointer(regs);
4102 
4103         if (!addr || addr >= TASK_SIZE)
4104                 return 0;
4105 
4106         return TASK_SIZE - addr;
4107 }
4108 
4109 static u16
4110 perf_sample_ustack_size(u16 stack_size, u16 header_size,
4111                         struct pt_regs *regs)
4112 {
4113         u64 task_size;
4114 
4115         /* No regs, no stack pointer, no dump. */
4116         if (!regs)
4117                 return 0;
4118 
4119         /*
4120          * Check if we fit in with the requested stack size into the:
4121          * - TASK_SIZE
4122          *   If we don't, we limit the size to the TASK_SIZE.
4123          *
4124          * - remaining sample size
4125          *   If we don't, we customize the stack size to
4126          *   fit in to the remaining sample size.
4127          */
4128 
4129         task_size  = min((u64) USHRT_MAX, perf_ustack_task_size(regs));
4130         stack_size = min(stack_size, (u16) task_size);
4131 
4132         /* Current header size plus static size and dynamic size. */
4133         header_size += 2 * sizeof(u64);
4134 
4135         /* Do we fit in with the current stack dump size? */
4136         if ((u16) (header_size + stack_size) < header_size) {
4137                 /*
4138                  * If we overflow the maximum size for the sample,
4139                  * we customize the stack dump size to fit in.
4140                  */
4141                 stack_size = USHRT_MAX - header_size - sizeof(u64);
4142                 stack_size = round_up(stack_size, sizeof(u64));
4143         }
4144 
4145         return stack_size;
4146 }
4147 
4148 static void
4149 perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size,
4150                           struct pt_regs *regs)
4151 {
4152         /* Case of a kernel thread, nothing to dump */
4153         if (!regs) {
4154                 u64 size = 0;
4155                 perf_output_put(handle, size);
4156         } else {
4157                 unsigned long sp;
4158                 unsigned int rem;
4159                 u64 dyn_size;
4160 
4161                 /*
4162                  * We dump:
4163                  * static size
4164                  *   - the size requested by user or the best one we can fit
4165                  *     in to the sample max size
4166                  * data
4167                  *   - user stack dump data
4168                  * dynamic size
4169                  *   - the actual dumped size
4170                  */
4171 
4172                 /* Static size. */
4173                 perf_output_put(handle, dump_size);
4174 
4175                 /* Data. */
4176                 sp = perf_user_stack_pointer(regs);
4177                 rem = __output_copy_user(handle, (void *) sp, dump_size);
4178                 dyn_size = dump_size - rem;
4179 
4180                 perf_output_skip(handle, rem);
4181 
4182                 /* Dynamic size. */
4183                 perf_output_put(handle, dyn_size);
4184         }
4185 }
4186 
4187 static void __perf_event_header__init_id(struct perf_event_header *header,
4188                                          struct perf_sample_data *data,
4189                                          struct perf_event *event)
4190 {
4191         u64 sample_type = event->attr.sample_type;
4192 
4193         data->type = sample_type;
4194         header->size += event->id_header_size;
4195 
4196         if (sample_type & PERF_SAMPLE_TID) {
4197                 /* namespace issues */
4198                 data->tid_entry.pid = perf_event_pid(event, current);
4199                 data->tid_entry.tid = perf_event_tid(event, current);
4200         }
4201 
4202         if (sample_type & PERF_SAMPLE_TIME)
4203                 data->time = perf_clock();
4204 
4205         if (sample_type & PERF_SAMPLE_ID)
4206                 data->id = primary_event_id(event);
4207 
4208         if (sample_type & PERF_SAMPLE_STREAM_ID)
4209                 data->stream_id = event->id;
4210 
4211         if (sample_type & PERF_SAMPLE_CPU) {
4212                 data->cpu_entry.cpu      = raw_smp_processor_id();
4213                 data->cpu_entry.reserved = 0;
4214         }
4215 }
4216 
4217 void perf_event_header__init_id(struct perf_event_header *header,
4218                                 struct perf_sample_data *data,
4219                                 struct perf_event *event)
4220 {
4221         if (event->attr.sample_id_all)
4222                 __perf_event_header__init_id(header, data, event);
4223 }
4224 
4225 static void __perf_event__output_id_sample(struct perf_output_handle *handle,
4226                                            struct perf_sample_data *data)
4227 {
4228         u64 sample_type = data->type;
4229 
4230         if (sample_type & PERF_SAMPLE_TID)
4231                 perf_output_put(handle, data->tid_entry);
4232 
4233         if (sample_type & PERF_SAMPLE_TIME)
4234                 perf_output_put(handle, data->time);
4235 
4236         if (sample_type & PERF_SAMPLE_ID)
4237                 perf_output_put(handle, data->id);
4238 
4239         if (sample_type & PERF_SAMPLE_STREAM_ID)
4240                 perf_output_put(handle, data->stream_id);
4241 
4242         if (sample_type & PERF_SAMPLE_CPU)
4243                 perf_output_put(handle, data->cpu_entry);
4244 }
4245 
4246 void perf_event__output_id_sample(struct perf_event *event,
4247                                   struct perf_output_handle *handle,
4248                                   struct perf_sample_data *sample)
4249 {
4250         if (event->attr.sample_id_all)
4251                 __perf_event__output_id_sample(handle, sample);
4252 }
4253 
4254 static void perf_output_read_one(struct perf_output_handle *handle,
4255                                  struct perf_event *event,
4256                                  u64 enabled, u64 running)
4257 {
4258         u64 read_format = event->attr.read_format;
4259         u64 values[4];
4260         int n = 0;
4261 
4262         values[n++] = perf_event_count(event);
4263         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
4264                 values[n++] = enabled +
4265                         atomic64_read(&event->child_total_time_enabled);
4266         }
4267         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
4268                 values[n++] = running +
4269                         atomic64_read(&event->child_total_time_running);
4270         }
4271         if (read_format & PERF_FORMAT_ID)
4272                 values[n++] = primary_event_id(event);
4273 
4274         __output_copy(handle, values, n * sizeof(u64));
4275 }
4276 
4277 /*
4278  * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
4279  */
4280 static void perf_output_read_group(struct perf_output_handle *handle,
4281                             struct perf_event *event,
4282                             u64 enabled, u64 running)
4283 {
4284         struct perf_event *leader = event->group_leader, *sub;
4285         u64 read_format = event->attr.read_format;
4286         u64 values[5];
4287         int n = 0;
4288 
4289         values[n++] = 1 + leader->nr_siblings;
4290 
4291         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
4292                 values[n++] = enabled;
4293 
4294         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
4295                 values[n++] = running;
4296 
4297         if (leader != event)
4298                 leader->pmu->read(leader);
4299 
4300         values[n++] = perf_event_count(leader);
4301         if (read_format & PERF_FORMAT_ID)
4302                 values[n++] = primary_event_id(leader);
4303 
4304         __output_copy(handle, values, n * sizeof(u64));
4305 
4306         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
4307                 n = 0;
4308 
4309                 if (sub != event)
4310                         sub->pmu->read(sub);
4311 
4312                 values[n++] = perf_event_count(sub);
4313                 if (read_format & PERF_FORMAT_ID)
4314                         values[n++] = primary_event_id(sub);
4315 
4316                 __output_copy(handle, values, n * sizeof(u64));
4317         }
4318 }
4319 
4320 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
4321                                  PERF_FORMAT_TOTAL_TIME_RUNNING)
4322 
4323 static void perf_output_read(struct perf_output_handle *handle,
4324                              struct perf_event *event)
4325 {
4326         u64 enabled = 0, running = 0, now;
4327         u64 read_format = event->attr.read_format;
4328 
4329         /*
4330          * compute total_time_enabled, total_time_running
4331          * based on snapshot values taken when the event
4332          * was last scheduled in.
4333          *
4334          * we cannot simply called update_context_time()
4335          * because of locking issue as we are called in
4336          * NMI context
4337          */
4338         if (read_format & PERF_FORMAT_TOTAL_TIMES)
4339                 calc_timer_values(event, &now, &enabled, &running);
4340 
4341         if (event->attr.read_format & PERF_FORMAT_GROUP)
4342                 perf_output_read_group(handle, event, enabled, running);
4343         else
4344                 perf_output_read_one(handle, event, enabled, running);
4345 }
4346 
4347 void perf_output_sample(struct perf_output_handle *handle,
4348                         struct perf_event_header *header,
4349                         struct perf_sample_data *data,
4350                         struct perf_event *event)
4351 {
4352         u64 sample_type = data->type;
4353 
4354         perf_output_put(handle, *header);
4355 
4356         if (sample_type & PERF_SAMPLE_IP)
4357                 perf_output_put(handle, data->ip);
4358 
4359         if (sample_type & PERF_SAMPLE_TID)
4360                 perf_output_put(handle, data->tid_entry);
4361 
4362         if (sample_type & PERF_SAMPLE_TIME)
4363                 perf_output_put(handle, data->time);
4364 
4365         if (sample_type & PERF_SAMPLE_ADDR)
4366                 perf_output_put(handle, data->addr);
4367 
4368         if (sample_type & PERF_SAMPLE_ID)
4369                 perf_output_put(handle, data->id);
4370 
4371         if (sample_type & PERF_SAMPLE_STREAM_ID)
4372                 perf_output_put(handle, data->stream_id);
4373 
4374         if (sample_type & PERF_SAMPLE_CPU)
4375                 perf_output_put(handle, data->cpu_entry);
4376 
4377         if (sample_type & PERF_SAMPLE_PERIOD)
4378                 perf_output_put(handle, data->period);
4379 
4380         if (sample_type & PERF_SAMPLE_READ)
4381                 perf_output_read(handle, event);
4382 
4383         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
4384                 if (data->callchain) {
4385                         int size = 1;
4386 
4387                         if (data->callchain)
4388                                 size += data->callchain->nr;
4389 
4390                         size *= sizeof(u64);
4391 
4392                         __output_copy(handle, data->callchain, size);
4393                 } else {
4394                         u64 nr = 0;
4395                         perf_output_put(handle, nr);
4396                 }
4397         }
4398 
4399         if (sample_type & PERF_SAMPLE_RAW) {
4400                 if (data->raw) {
4401                         perf_output_put(handle, data->raw->size);
4402                         __output_copy(handle, data->raw->data,
4403                                            data->raw->size);
4404                 } else {
4405                         struct {
4406                                 u32     size;
4407                                 u32     data;
4408                         } raw = {
4409                                 .size = sizeof(u32),
4410                                 .data = 0,
4411                         };
4412                         perf_output_put(handle, raw);
4413                 }
4414         }
4415 
4416         if (!event->attr.watermark) {
4417                 int wakeup_events = event->attr.wakeup_events;
4418 
4419                 if (wakeup_events) {
4420                         struct ring_buffer *rb = handle->rb;
4421                         int events = local_inc_return(&rb->events);
4422 
4423                         if (events >= wakeup_events) {
4424                                 local_sub(wakeup_events, &rb->events);
4425                                 local_inc(&rb->wakeup);
4426                         }
4427                 }
4428         }
4429 
4430         if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4431                 if (data->br_stack) {
4432                         size_t size;
4433 
4434                         size = data->br_stack->nr
4435                              * sizeof(struct perf_branch_entry);
4436 
4437                         perf_output_put(handle, data->br_stack->nr);
4438                         perf_output_copy(handle, data->br_stack->entries, size);
4439                 } else {
4440                         /*
4441                          * we always store at least the value of nr
4442                          */
4443                         u64 nr = 0;
4444                         perf_output_put(handle, nr);
4445                 }
4446         }
4447 
4448         if (sample_type & PERF_SAMPLE_REGS_USER) {
4449                 u64 abi = data->regs_user.abi;
4450 
4451                 /*
4452                  * If there are no regs to dump, notice it through
4453                  * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
4454                  */
4455                 perf_output_put(handle, abi);
4456 
4457                 if (abi) {
4458                         u64 mask = event->attr.sample_regs_user;
4459                         perf_output_sample_regs(handle,
4460                                                 data->regs_user.regs,
4461                                                 mask);
4462                 }
4463         }
4464 
4465         if (sample_type & PERF_SAMPLE_STACK_USER)
4466                 perf_output_sample_ustack(handle,
4467                                           data->stack_user_size,
4468                                           data->regs_user.regs);
4469 
4470         if (sample_type & PERF_SAMPLE_WEIGHT)
4471                 perf_output_put(handle, data->weight);
4472 
4473         if (sample_type & PERF_SAMPLE_DATA_SRC)
4474                 perf_output_put(handle, data->data_src.val);
4475 }
4476 
4477 void perf_prepare_sample(struct perf_event_header *header,
4478                          struct perf_sample_data *data,
4479                          struct perf_event *event,
4480                          struct pt_regs *regs)
4481 {
4482         u64 sample_type = event->attr.sample_type;
4483 
4484         header->type = PERF_RECORD_SAMPLE;
4485         header->size = sizeof(*header) + event->header_size;
4486 
4487         header->misc = 0;
4488         header->misc |= perf_misc_flags(regs);
4489 
4490         __perf_event_header__init_id(header, data, event);
4491 
4492         if (sample_type & PERF_SAMPLE_IP)
4493                 data->ip = perf_instruction_pointer(regs);
4494 
4495         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
4496                 int size = 1;
4497 
4498                 data->callchain = perf_callchain(event, regs);
4499 
4500                 if (data->callchain)
4501                         size += data->callchain->nr;
4502 
4503                 header->size += size * sizeof(u64);
4504         }
4505 
4506         if (sample_type & PERF_SAMPLE_RAW) {
4507                 int size = sizeof(u32);
4508 
4509                 if (data->raw)
4510                         size += data->raw->size;
4511                 else
4512                         size += sizeof(u32);
4513 
4514                 WARN_ON_ONCE(size & (sizeof(u64)-1));
4515                 header->size += size;
4516         }
4517 
4518         if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4519                 int size = sizeof(u64); /* nr */
4520                 if (data->br_stack) {
4521                         size += data->br_stack->nr
4522                               * sizeof(struct perf_branch_entry);
4523                 }
4524                 header->size += size;
4525         }
4526 
4527         if (sample_type & PERF_SAMPLE_REGS_USER) {
4528                 /* regs dump ABI info */
4529                 int size = sizeof(u64);
4530 
4531                 perf_sample_regs_user(&data->regs_user, regs);
4532 
4533                 if (data->regs_user.regs) {
4534                         u64 mask = event->attr.sample_regs_user;
4535                         size += hweight64(mask) * sizeof(u64);
4536                 }
4537 
4538                 header->size += size;
4539         }
4540 
4541         if (sample_type & PERF_SAMPLE_STACK_USER) {
4542                 /*
4543                  * Either we need PERF_SAMPLE_STACK_USER bit to be allways
4544                  * processed as the last one or have additional check added
4545                  * in case new sample type is added, because we could eat
4546                  * up the rest of the sample size.
4547                  */
4548                 struct perf_regs_user *uregs = &data->regs_user;
4549                 u16 stack_size = event->attr.sample_stack_user;
4550                 u16 size = sizeof(u64);
4551 
4552                 if (!uregs->abi)
4553                         perf_sample_regs_user(uregs, regs);
4554 
4555                 stack_size = perf_sample_ustack_size(stack_size, header->size,
4556                                                      uregs->regs);
4557 
4558                 /*
4559                  * If there is something to dump, add space for the dump
4560                  * itself and for the field that tells the dynamic size,
4561                  * which is how many have been actually dumped.
4562                  */
4563                 if (stack_size)
4564                         size += sizeof(u64) + stack_size;
4565 
4566                 data->stack_user_size = stack_size;
4567                 header->size += size;
4568         }
4569 }
4570 
4571 static void perf_event_output(struct perf_event *event,
4572                                 struct perf_sample_data *data,
4573                                 struct pt_regs *regs)
4574 {
4575         struct perf_output_handle handle;
4576         struct perf_event_header header;
4577 
4578         /* protect the callchain buffers */
4579         rcu_read_lock();
4580 
4581         perf_prepare_sample(&header, data, event, regs);
4582 
4583         if (perf_output_begin(&handle, event, header.size))
4584                 goto exit;
4585 
4586         perf_output_sample(&handle, &header, data, event);
4587 
4588         perf_output_end(&handle);
4589 
4590 exit:
4591         rcu_read_unlock();
4592 }
4593 
4594 /*
4595  * read event_id
4596  */
4597 
4598 struct perf_read_event {
4599         struct perf_event_header        header;
4600 
4601         u32                             pid;
4602         u32                             tid;
4603 };
4604 
4605 static void
4606 perf_event_read_event(struct perf_event *event,
4607                         struct task_struct *task)
4608 {
4609         struct perf_output_handle handle;
4610         struct perf_sample_data sample;
4611         struct perf_read_event read_event = {
4612                 .header = {
4613                         .type = PERF_RECORD_READ,
4614                         .misc = 0,
4615                         .size = sizeof(read_event) + event->read_size,
4616                 },
4617                 .pid = perf_event_pid(event, task),
4618                 .tid = perf_event_tid(event, task),
4619         };
4620         int ret;
4621 
4622         perf_event_header__init_id(&read_event.header, &sample, event);
4623         ret = perf_output_begin(&handle, event, read_event.header.size);
4624         if (ret)
4625                 return;
4626 
4627         perf_output_put(&handle, read_event);
4628         perf_output_read(&handle, event);
4629         perf_event__output_id_sample(event, &handle, &sample);
4630 
4631         perf_output_end(&handle);
4632 }
4633 
4634 typedef int  (perf_event_aux_match_cb)(struct perf_event *event, void *data);
4635 typedef void (perf_event_aux_output_cb)(struct perf_event *event, void *data);
4636 
4637 static void
4638 perf_event_aux_ctx(struct perf_event_context *ctx,
4639                    perf_event_aux_match_cb match,
4640                    perf_event_aux_output_cb output,
4641                    void *data)
4642 {
4643         struct perf_event *event;
4644 
4645         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4646                 if (event->state < PERF_EVENT_STATE_INACTIVE)
4647                         continue;
4648                 if (!event_filter_match(event))
4649                         continue;
4650                 if (match(event, data))
4651                         output(event, data);
4652         }
4653 }
4654 
4655 static void
4656 perf_event_aux(perf_event_aux_match_cb match,
4657                perf_event_aux_output_cb output,
4658                void *data,
4659                struct perf_event_context *task_ctx)
4660 {
4661         struct perf_cpu_context *cpuctx;
4662         struct perf_event_context *ctx;
4663         struct pmu *pmu;
4664         int ctxn;
4665 
4666         rcu_read_lock();
4667         list_for_each_entry_rcu(pmu, &pmus, entry) {
4668                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4669                 if (cpuctx->unique_pmu != pmu)
4670                         goto next;
4671                 perf_event_aux_ctx(&cpuctx->ctx, match, output, data);
4672                 if (task_ctx)
4673                         goto next;
4674                 ctxn = pmu->task_ctx_nr;
4675                 if (ctxn < 0)
4676                         goto next;
4677                 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
4678                 if (ctx)
4679                         perf_event_aux_ctx(ctx, match, output, data);
4680 next:
4681                 put_cpu_ptr(pmu->pmu_cpu_context);
4682         }
4683 
4684         if (task_ctx) {
4685                 preempt_disable();
4686                 perf_event_aux_ctx(task_ctx, match, output, data);
4687                 preempt_enable();
4688         }
4689         rcu_read_unlock();
4690 }
4691 
4692 /*
4693  * task tracking -- fork/exit
4694  *
4695  * enabled by: attr.comm | attr.mmap | attr.mmap_data | attr.task
4696  */
4697 
4698 struct perf_task_event {
4699         struct task_struct              *task;
4700         struct perf_event_context       *task_ctx;
4701 
4702         struct {
4703                 struct perf_event_header        header;
4704 
4705                 u32                             pid;
4706                 u32                             ppid;
4707                 u32                             tid;
4708                 u32                             ptid;
4709                 u64                             time;
4710         } event_id;
4711 };
4712 
4713 static void perf_event_task_output(struct perf_event *event,
4714                                    void *data)
4715 {
4716         struct perf_task_event *task_event = data;
4717         struct perf_output_handle handle;
4718         struct perf_sample_data sample;
4719         struct task_struct *task = task_event->task;
4720         int ret, size = task_event->event_id.header.size;
4721 
4722         perf_event_header__init_id(&task_event->event_id.header, &sample, event);
4723 
4724         ret = perf_output_begin(&handle, event,
4725                                 task_event->event_id.header.size);
4726         if (ret)
4727                 goto out;
4728 
4729         task_event->event_id.pid = perf_event_pid(event, task);
4730         task_event->event_id.ppid = perf_event_pid(event, current);
4731 
4732         task_event->event_id.tid = perf_event_tid(event, task);
4733         task_event->event_id.ptid = perf_event_tid(event, current);
4734 
4735         perf_output_put(&handle, task_event->event_id);
4736 
4737         perf_event__output_id_sample(event, &handle, &sample);
4738 
4739         perf_output_end(&handle);
4740 out:
4741         task_event->event_id.header.size = size;
4742 }
4743 
4744 static int perf_event_task_match(struct perf_event *event,
4745                                  void *data __maybe_unused)
4746 {
4747         return event->attr.comm || event->attr.mmap ||
4748                event->attr.mmap_data || event->attr.task;
4749 }
4750 
4751 static void perf_event_task(struct task_struct *task,
4752                               struct perf_event_context *task_ctx,
4753                               int new)
4754 {
4755         struct perf_task_event task_event;
4756 
4757         if (!atomic_read(&nr_comm_events) &&
4758             !atomic_read(&nr_mmap_events) &&
4759             !atomic_read(&nr_task_events))
4760                 return;
4761 
4762         task_event = (struct perf_task_event){
4763                 .task     = task,
4764                 .task_ctx = task_ctx,
4765                 .event_id    = {
4766                         .header = {
4767                                 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
4768                                 .misc = 0,
4769                                 .size = sizeof(task_event.event_id),
4770                         },
4771                         /* .pid  */
4772                         /* .ppid */
4773                         /* .tid  */
4774                         /* .ptid */
4775                         .time = perf_clock(),
4776                 },
4777         };
4778 
4779         perf_event_aux(perf_event_task_match,
4780                        perf_event_task_output,
4781                        &task_event,
4782                        task_ctx);
4783 }
4784 
4785 void perf_event_fork(struct task_struct *task)
4786 {
4787         perf_event_task(task, NULL, 1);
4788 }
4789 
4790 /*
4791  * comm tracking
4792  */
4793 
4794 struct perf_comm_event {
4795         struct task_struct      *task;
4796         char                    *comm;
4797         int                     comm_size;
4798 
4799         struct {
4800                 struct perf_event_header        header;
4801 
4802                 u32                             pid;
4803                 u32                             tid;
4804         } event_id;
4805 };
4806 
4807 static void perf_event_comm_output(struct perf_event *event,
4808                                    void *data)
4809 {
4810         struct perf_comm_event *comm_event = data;
4811         struct perf_output_handle handle;
4812         struct perf_sample_data sample;
4813         int size = comm_event->event_id.header.size;
4814         int ret;
4815 
4816         perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
4817         ret = perf_output_begin(&handle, event,
4818                                 comm_event->event_id.header.size);
4819 
4820         if (ret)
4821                 goto out;
4822 
4823         comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
4824         comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
4825 
4826         perf_output_put(&handle, comm_event->event_id);
4827         __output_copy(&handle, comm_event->comm,
4828                                    comm_event->comm_size);
4829 
4830         perf_event__output_id_sample(event, &handle, &sample);
4831 
4832         perf_output_end(&handle);
4833 out:
4834         comm_event->event_id.header.size = size;
4835 }
4836 
4837 static int perf_event_comm_match(struct perf_event *event,
4838                                  void *data __maybe_unused)
4839 {
4840         return event->attr.comm;
4841 }
4842 
4843 static void perf_event_comm_event(struct perf_comm_event *comm_event)
4844 {
4845         char comm[TASK_COMM_LEN];
4846         unsigned int size;
4847 
4848         memset(comm, 0, sizeof(comm));
4849         strlcpy(comm, comm_event->task->comm, sizeof(comm));
4850         size = ALIGN(strlen(comm)+1, sizeof(u64));
4851 
4852         comm_event->comm = comm;
4853         comm_event->comm_size = size;
4854 
4855         comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
4856 
4857         perf_event_aux(perf_event_comm_match,
4858                        perf_event_comm_output,
4859                        comm_event,
4860                        NULL);
4861 }
4862 
4863 void perf_event_comm(struct task_struct *task)
4864 {
4865         struct perf_comm_event comm_event;
4866         struct perf_event_context *ctx;
4867         int ctxn;
4868 
4869         rcu_read_lock();
4870         for_each_task_context_nr(ctxn) {
4871                 ctx = task->perf_event_ctxp[ctxn];
4872                 if (!ctx)
4873                         continue;
4874 
4875                 perf_event_enable_on_exec(ctx);
4876         }
4877         rcu_read_unlock();
4878 
4879         if (!atomic_read(&nr_comm_events))
4880                 return;
4881 
4882         comm_event = (struct perf_comm_event){
4883                 .task   = task,
4884                 /* .comm      */
4885                 /* .comm_size */
4886                 .event_id  = {
4887                         .header = {
4888                                 .type = PERF_RECORD_COMM,
4889                                 .misc = 0,
4890                                 /* .size */
4891                         },
4892                         /* .pid */
4893                         /* .tid */
4894                 },
4895         };
4896 
4897         perf_event_comm_event(&comm_event);
4898 }
4899 
4900 /*
4901  * mmap tracking
4902  */
4903 
4904 struct perf_mmap_event {
4905         struct vm_area_struct   *vma;
4906 
4907         const char              *file_name;
4908         int                     file_size;
4909 
4910         struct {
4911                 struct perf_event_header        header;
4912 
4913                 u32                             pid;
4914                 u32                             tid;
4915                 u64                             start;
4916                 u64                             len;
4917                 u64                             pgoff;
4918         } event_id;
4919 };
4920 
4921 static void perf_event_mmap_output(struct perf_event *event,
4922                                    void *data)
4923 {
4924         struct perf_mmap_event *mmap_event = data;
4925         struct perf_output_handle handle;
4926         struct perf_sample_data sample;
4927         int size = mmap_event->event_id.header.size;
4928         int ret;
4929 
4930         perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
4931         ret = perf_output_begin(&handle, event,
4932                                 mmap_event->event_id.header.size);
4933         if (ret)
4934                 goto out;
4935 
4936         mmap_event->event_id.pid = perf_event_pid(event, current);
4937         mmap_event->event_id.tid = perf_event_tid(event, current);
4938 
4939         perf_output_put(&handle, mmap_event->event_id);
4940         __output_copy(&handle, mmap_event->file_name,
4941                                    mmap_event->file_size);
4942 
4943         perf_event__output_id_sample(event, &handle, &sample);
4944 
4945         perf_output_end(&handle);
4946 out:
4947         mmap_event->event_id.header.size = size;
4948 }
4949 
4950 static int perf_event_mmap_match(struct perf_event *event,
4951                                  void *data)
4952 {
4953         struct perf_mmap_event *mmap_event = data;
4954         struct vm_area_struct *vma = mmap_event->vma;
4955         int executable = vma->vm_flags & VM_EXEC;
4956 
4957         return (!executable && event->attr.mmap_data) ||
4958                (executable && event->attr.mmap);
4959 }
4960 
4961 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
4962 {
4963         struct vm_area_struct *vma = mmap_event->vma;
4964         struct file *file = vma->vm_file;
4965         unsigned int size;
4966         char tmp[16];
4967         char *buf = NULL;
4968         const char *name;
4969 
4970         memset(tmp, 0, sizeof(tmp));
4971 
4972         if (file) {
4973                 /*
4974                  * d_path works from the end of the rb backwards, so we
4975                  * need to add enough zero bytes after the string to handle
4976                  * the 64bit alignment we do later.
4977                  */
4978                 buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
4979                 if (!buf) {
4980                         name = strncpy(tmp, "//enomem", sizeof(tmp));
4981                         goto got_name;
4982                 }
4983                 name = d_path(&file->f_path, buf, PATH_MAX);
4984                 if (IS_ERR(name)) {
4985                         name = strncpy(tmp, "//toolong", sizeof(tmp));
4986                         goto got_name;
4987                 }
4988         } else {
4989                 if (arch_vma_name(mmap_event->vma)) {
4990                         name = strncpy(tmp, arch_vma_name(mmap_event->vma),
4991                                        sizeof(tmp) - 1);
4992                         tmp[sizeof(tmp) - 1] = '\0';
4993                         goto got_name;
4994                 }
4995 
4996                 if (!vma->vm_mm) {
4997                         name = strncpy(tmp, "[vdso]", sizeof(tmp));
4998                         goto got_name;
4999                 } else if (vma->vm_start <= vma->vm_mm->start_brk &&
5000                                 vma->vm_end >= vma->vm_mm->brk) {
5001                         name = strncpy(tmp, "[heap]", sizeof(tmp));
5002                         goto got_name;
5003                 } else if (vma->vm_start <= vma->vm_mm->start_stack &&
5004                                 vma->vm_end >= vma->vm_mm->start_stack) {
5005                         name = strncpy(tmp, "[stack]", sizeof(tmp));
5006                         goto got_name;
5007                 }
5008 
5009                 name = strncpy(tmp, "//anon", sizeof(tmp));
5010                 goto got_name;
5011         }
5012 
5013 got_name:
5014         size = ALIGN(strlen(name)+1, sizeof(u64));
5015 
5016         mmap_event->file_name = name;
5017         mmap_event->file_size = size;
5018 
5019         if (!(vma->vm_flags & VM_EXEC))
5020                 mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;
5021 
5022         mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
5023 
5024         perf_event_aux(perf_event_mmap_match,
5025                        perf_event_mmap_output,
5026                        mmap_event,
5027                        NULL);
5028 
5029         kfree(buf);
5030 }
5031 
5032 void perf_event_mmap(struct vm_area_struct *vma)
5033 {
5034         struct perf_mmap_event mmap_event;
5035 
5036         if (!atomic_read(&nr_mmap_events))
5037                 return;
5038 
5039         mmap_event = (struct perf_mmap_event){
5040                 .vma    = vma,
5041                 /* .file_name */
5042                 /* .file_size */
5043                 .event_id  = {
5044                         .header = {
5045                                 .type = PERF_RECORD_MMAP,
5046                                 .misc = PERF_RECORD_MISC_USER,
5047                                 /* .size */
5048                         },
5049                         /* .pid */
5050                         /* .tid */
5051                         .start  = vma->vm_start,
5052                         .len    = vma->vm_end - vma->vm_start,
5053                         .pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
5054                 },
5055         };
5056 
5057         perf_event_mmap_event(&mmap_event);
5058 }
5059 
5060 /*
5061  * IRQ throttle logging
5062  */
5063 
5064 static void perf_log_throttle(struct perf_event *event, int enable)
5065 {
5066         struct perf_output_handle handle;
5067         struct perf_sample_data sample;
5068         int ret;
5069 
5070         struct {
5071                 struct perf_event_header        header;
5072                 u64                             time;
5073                 u64                             id;
5074                 u64                             stream_id;
5075         } throttle_event = {
5076                 .header = {
5077                         .type = PERF_RECORD_THROTTLE,
5078                         .misc = 0,
5079                         .size = sizeof(throttle_event),
5080                 },
5081                 .time           = perf_clock(),
5082                 .id             = primary_event_id(event),
5083                 .stream_id      = event->id,
5084         };
5085 
5086         if (enable)
5087                 throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
5088 
5089         perf_event_header__init_id(&throttle_event.header, &sample, event);
5090 
5091         ret = perf_output_begin(&handle, event,
5092                                 throttle_event.header.size);
5093         if (ret)
5094                 return;
5095 
5096         perf_output_put(&handle, throttle_event);
5097         perf_event__output_id_sample(event, &handle, &sample);
5098         perf_output_end(&handle);
5099 }
5100 
5101 /*
5102  * Generic event overflow handling, sampling.
5103  */
5104 
5105 static int __perf_event_overflow(struct perf_event *event,
5106                                    int throttle, struct perf_sample_data *data,
5107                                    struct pt_regs *regs)
5108 {
5109         int events = atomic_read(&event->event_limit);
5110         struct hw_perf_event *hwc = &event->hw;
5111         u64 seq;
5112         int ret = 0;
5113 
5114         /*
5115          * Non-sampling counters might still use the PMI to fold short
5116          * hardware counters, ignore those.
5117          */
5118         if (unlikely(!is_sampling_event(event)))
5119                 return 0;
5120 
5121         seq = __this_cpu_read(perf_throttled_seq);
5122         if (seq != hwc->interrupts_seq) {
5123                 hwc->interrupts_seq = seq;
5124                 hwc->interrupts = 1;
5125         } else {
5126                 hwc->interrupts++;
5127                 if (unlikely(throttle
5128                              && hwc->interrupts >= max_samples_per_tick)) {
5129                         __this_cpu_inc(perf_throttled_count);
5130                         hwc->interrupts = MAX_INTERRUPTS;
5131                         perf_log_throttle(event, 0);
5132                         ret = 1;
5133                 }
5134         }
5135 
5136         if (event->attr.freq) {
5137                 u64 now = perf_clock();
5138                 s64 delta = now - hwc->freq_time_stamp;
5139 
5140                 hwc->freq_time_stamp = now;
5141 
5142                 if (delta > 0 && delta < 2*TICK_NSEC)
5143                         perf_adjust_period(event, delta, hwc->last_period, true);
5144         }
5145 
5146         /*
5147          * XXX event_limit might not quite work as expected on inherited
5148          * events
5149          */
5150 
5151         event->pending_kill = POLL_IN;
5152         if (events && atomic_dec_and_test(&event->event_limit)) {
5153                 ret = 1;
5154                 event->pending_kill = POLL_HUP;
5155                 event->pending_disable = 1;
5156                 irq_work_queue(&event->pending);
5157         }
5158 
5159         if (event->overflow_handler)
5160                 event->overflow_handler(event, data, regs);
5161         else
5162                 perf_event_output(event, data, regs);
5163 
5164         if (*perf_event_fasync(event) && event->pending_kill) {
5165                 event->pending_wakeup = 1;
5166                 irq_work_queue(&event->pending);
5167         }
5168 
5169         return ret;
5170 }
5171 
5172 int perf_event_overflow(struct perf_event *event,
5173                           struct perf_sample_data *data,
5174                           struct pt_regs *regs)
5175 {
5176         return __perf_event_overflow(event, 1, data, regs);
5177 }
5178 
5179 /*
5180  * Generic software event infrastructure
5181  */
5182 
5183 struct swevent_htable {
5184         struct swevent_hlist            *swevent_hlist;
5185         struct mutex                    hlist_mutex;
5186         int                             hlist_refcount;
5187 
5188         /* Recursion avoidance in each contexts */
5189         int                             recursion[PERF_NR_CONTEXTS];
5190 
5191         /* Keeps track of cpu being initialized/exited */
5192         bool                            online;
5193 };
5194 
5195 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
5196 
5197 /*
5198  * We directly increment event->count and keep a second value in
5199  * event->hw.period_left to count intervals. This period event
5200  * is kept in the range [-sample_period, 0] so that we can use the
5201  * sign as trigger.
5202  */
5203 
5204 static u64 perf_swevent_set_period(struct perf_event *event)
5205 {
5206         struct hw_perf_event *hwc = &event->hw;
5207         u64 period = hwc->last_period;
5208         u64 nr, offset;
5209         s64 old, val;
5210 
5211         hwc->last_period = hwc->sample_period;
5212 
5213 again:
5214         old = val = local64_read(&hwc->period_left);
5215         if (val < 0)
5216                 return 0;
5217 
5218         nr = div64_u64(period + val, period);
5219         offset = nr * period;
5220         val -= offset;
5221         if (local64_cmpxchg(&hwc->period_left, old, val) != old)
5222                 goto again;
5223 
5224         return nr;
5225 }
5226 
5227 static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
5228                                     struct perf_sample_data *data,
5229                                     struct pt_regs *regs)
5230 {
5231         struct hw_perf_event *hwc = &event->hw;
5232         int throttle = 0;
5233 
5234         if (!overflow)
5235                 overflow = perf_swevent_set_period(event);
5236 
5237         if (hwc->interrupts == MAX_INTERRUPTS)
5238                 return;
5239 
5240         for (; overflow; overflow--) {
5241                 if (__perf_event_overflow(event, throttle,
5242                                             data, regs)) {
5243                         /*
5244                          * We inhibit the overflow from happening when
5245                          * hwc->interrupts == MAX_INTERRUPTS.
5246                          */
5247                         break;
5248                 }
5249                 throttle = 1;
5250         }
5251 }
5252 
5253 static void perf_swevent_event(struct perf_event *event, u64 nr,
5254                                struct perf_sample_data *data,
5255                                struct pt_regs *regs)
5256 {
5257         struct hw_perf_event *hwc = &event->hw;
5258 
5259         local64_add(nr, &event->count);
5260 
5261         if (!regs)
5262                 return;
5263 
5264         if (!is_sampling_event(event))
5265                 return;
5266 
5267         if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
5268                 data->period = nr;
5269                 return perf_swevent_overflow(event, 1, data, regs);
5270         } else
5271                 data->period = event->hw.last_period;
5272 
5273         if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
5274                 return perf_swevent_overflow(event, 1, data, regs);
5275 
5276         if (local64_add_negative(nr, &hwc->period_left))
5277                 return;
5278 
5279         perf_swevent_overflow(event, 0, data, regs);
5280 }
5281 
5282 static int perf_exclude_event(struct perf_event *event,
5283                               struct pt_regs *regs)
5284 {
5285         if (event->hw.state & PERF_HES_STOPPED)
5286                 return 1;
5287 
5288         if (regs) {
5289                 if (event->attr.exclude_user && user_mode(regs))
5290                         return 1;
5291 
5292                 if (event->attr.exclude_kernel && !user_mode(regs))
5293                         return 1;
5294         }
5295 
5296         return 0;
5297 }
5298 
5299 static int perf_swevent_match(struct perf_event *event,
5300                                 enum perf_type_id type,
5301                                 u32 event_id,
5302                                 struct perf_sample_data *data,
5303                                 struct pt_regs *regs)
5304 {
5305         if (event->attr.type != type)
5306                 return 0;
5307 
5308         if (event->attr.config != event_id)
5309                 return 0;
5310 
5311         if (perf_exclude_event(event, regs))
5312                 return 0;
5313 
5314         return 1;
5315 }
5316 
5317 static inline u64 swevent_hash(u64 type, u32 event_id)
5318 {
5319         u64 val = event_id | (type << 32);
5320 
5321         return hash_64(val, SWEVENT_HLIST_BITS);
5322 }
5323 
5324 static inline struct hlist_head *
5325 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
5326 {
5327         u64 hash = swevent_hash(type, event_id);
5328 
5329         return &hlist->heads[hash];
5330 }
5331 
5332 /* For the read side: events when they trigger */
5333 static inline struct hlist_head *
5334 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
5335 {
5336         struct swevent_hlist *hlist;
5337 
5338         hlist = rcu_dereference(swhash->swevent_hlist);
5339         if (!hlist)
5340                 return NULL;
5341 
5342         return __find_swevent_head(hlist, type, event_id);
5343 }
5344 
5345 /* For the event head insertion and removal in the hlist */
5346 static inline struct hlist_head *
5347 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
5348 {
5349         struct swevent_hlist *hlist;
5350         u32 event_id = event->attr.config;
5351         u64 type = event->attr.type;
5352 
5353         /*
5354          * Event scheduling is always serialized against hlist allocation
5355          * and release. Which makes the protected version suitable here.
5356          * The context lock guarantees that.
5357          */
5358         hlist = rcu_dereference_protected(swhash->swevent_hlist,
5359                                           lockdep_is_held(&event->ctx->lock));
5360         if (!hlist)
5361                 return NULL;
5362 
5363         return __find_swevent_head(hlist, type, event_id);
5364 }
5365 
5366 static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
5367                                     u64 nr,
5368                                     struct perf_sample_data *data,
5369                                     struct pt_regs *regs)
5370 {
5371         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
5372         struct perf_event *event;
5373         struct hlist_head *head;
5374 
5375         rcu_read_lock();
5376         head = find_swevent_head_rcu(swhash, type, event_id);
5377         if (!head)
5378                 goto end;
5379 
5380         hlist_for_each_entry_rcu(event, head, hlist_entry) {
5381                 if (perf_swevent_match(event, type, event_id, data, regs))
5382                         perf_swevent_event(event, nr, data, regs);
5383         }
5384 end:
5385         rcu_read_unlock();
5386 }
5387 
5388 int perf_swevent_get_recursion_context(void)
5389 {
5390         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
5391 
5392         return get_recursion_context(swhash->recursion);
5393 }
5394 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
5395 
5396 inline void perf_swevent_put_recursion_context(int rctx)
5397 {
5398         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
5399 
5400         put_recursion_context(swhash->recursion, rctx);
5401 }
5402 
5403 void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
5404 {
5405         struct perf_sample_data data;
5406         int rctx;
5407 
5408         preempt_disable_notrace();
5409         rctx = perf_swevent_get_recursion_context();
5410         if (rctx < 0)
5411                 return;
5412 
5413         perf_sample_data_init(&data, addr, 0);
5414 
5415         do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
5416 
5417         perf_swevent_put_recursion_context(rctx);
5418         preempt_enable_notrace();
5419 }
5420 
5421 static void perf_swevent_read(struct perf_event *event)
5422 {
5423 }
5424 
5425 static int perf_swevent_add(struct perf_event *event, int flags)
5426 {
5427         struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
5428         struct hw_perf_event *hwc = &event->hw;
5429         struct hlist_head *head;
5430 
5431         if (is_sampling_event(event)) {
5432                 hwc->last_period = hwc->sample_period;
5433                 perf_swevent_set_period(event);
5434         }
5435 
5436         hwc->state = !(flags & PERF_EF_START);
5437 
5438         head = find_swevent_head(swhash, event);
5439         if (!head) {
5440                 /*
5441                  * We can race with cpu hotplug code. Do not
5442                  * WARN if the cpu just got unplugged.
5443                  */
5444                 WARN_ON_ONCE(swhash->online);
5445                 return -EINVAL;
5446         }
5447 
5448         hlist_add_head_rcu(&event->hlist_entry, head);
5449 
5450         return 0;
5451 }
5452 
5453 static void perf_swevent_del(struct perf_event *event, int flags)
5454 {
5455         hlist_del_rcu(&event->hlist_entry);
5456 }
5457 
5458 static void perf_swevent_start(struct perf_event *event, int flags)
5459 {
5460         event->hw.state = 0;
5461 }
5462 
5463 static void perf_swevent_stop(struct perf_event *event, int flags)
5464 {
5465         event->hw.state = PERF_HES_STOPPED;
5466 }
5467 
5468 /* Deref the hlist from the update side */
5469 static inline struct swevent_hlist *
5470 swevent_hlist_deref(struct swevent_htable *swhash)
5471 {
5472         return rcu_dereference_protected(swhash->swevent_hlist,
5473                                          lockdep_is_held(&swhash->hlist_mutex));
5474 }
5475 
5476 static void swevent_hlist_release(struct swevent_htable *swhash)
5477 {
5478         struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
5479 
5480         if (!hlist)
5481                 return;
5482 
5483         rcu_assign_pointer(swhash->swevent_hlist, NULL);
5484         kfree_rcu(hlist, rcu_head);
5485 }
5486 
5487 static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
5488 {
5489         struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5490 
5491         mutex_lock(&swhash->hlist_mutex);
5492 
5493         if (!--swhash->hlist_refcount)
5494                 swevent_hlist_release(swhash);
5495 
5496         mutex_unlock(&swhash->hlist_mutex);
5497 }
5498 
5499 static void swevent_hlist_put(struct perf_event *event)
5500 {
5501         int cpu;
5502 
5503         if (event->cpu != -1) {
5504                 swevent_hlist_put_cpu(event, event->cpu);
5505                 return;
5506         }
5507 
5508         for_each_possible_cpu(cpu)
5509                 swevent_hlist_put_cpu(event, cpu);
5510 }
5511 
5512 static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
5513 {
5514         struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5515         int err = 0;
5516 
5517         mutex_lock(&swhash->hlist_mutex);
5518 
5519         if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
5520                 struct swevent_hlist *hlist;
5521 
5522                 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
5523                 if (!hlist) {
5524                         err = -ENOMEM;
5525                         goto exit;
5526                 }
5527                 rcu_assign_pointer(swhash->swevent_hlist, hlist);
5528         }
5529         swhash->hlist_refcount++;
5530 exit:
5531         mutex_unlock(&swhash->hlist_mutex);
5532 
5533         return err;
5534 }
5535 
5536 static int swevent_hlist_get(struct perf_event *event)
5537 {
5538         int err;
5539         int cpu, failed_cpu;
5540 
5541         if (event->cpu != -1)
5542                 return swevent_hlist_get_cpu(event, event->cpu);
5543 
5544         get_online_cpus();
5545         for_each_possible_cpu(cpu) {
5546                 err = swevent_hlist_get_cpu(event, cpu);
5547                 if (err) {
5548                         failed_cpu = cpu;
5549                         goto fail;
5550                 }
5551         }
5552         put_online_cpus();
5553 
5554         return 0;
5555 fail:
5556         for_each_possible_cpu(cpu) {
5557                 if (cpu == failed_cpu)
5558                         break;
5559                 swevent_hlist_put_cpu(event, cpu);
5560         }
5561 
5562         put_online_cpus();
5563         return err;
5564 }
5565 
5566 struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
5567 
5568 static void sw_perf_event_destroy(struct perf_event *event)
5569 {
5570         u64 event_id = event->attr.config;
5571 
5572         WARN_ON(event->parent);
5573 
5574         static_key_slow_dec(&perf_swevent_enabled[event_id]);
5575         swevent_hlist_put(event);
5576 }
5577 
5578 static int perf_swevent_init(struct perf_event *event)
5579 {
5580         u64 event_id = event->attr.config;
5581 
5582         if (event->attr.type != PERF_TYPE_SOFTWARE)
5583                 return -ENOENT;
5584 
5585         /*
5586          * no branch sampling for software events
5587          */
5588         if (has_branch_stack(event))
5589                 return -EOPNOTSUPP;
5590 
5591         switch (event_id) {
5592         case PERF_COUNT_SW_CPU_CLOCK:
5593         case PERF_COUNT_SW_TASK_CLOCK:
5594                 return -ENOENT;
5595 
5596         default:
5597                 break;
5598         }
5599 
5600         if (event_id >= PERF_COUNT_SW_MAX)
5601                 return -ENOENT;
5602 
5603         if (!event->parent) {
5604                 int err;
5605 
5606                 err = swevent_hlist_get(event);
5607                 if (err)
5608                         return err;
5609 
5610                 static_key_slow_inc(&perf_swevent_enabled[event_id]);
5611                 event->destroy = sw_perf_event_destroy;
5612         }
5613 
5614         return 0;
5615 }
5616 
5617 static int perf_swevent_event_idx(struct perf_event *event)
5618 {
5619         return 0;
5620 }
5621 
5622 static struct pmu perf_swevent = {
5623         .task_ctx_nr    = perf_sw_context,
5624 
5625         .event_init     = perf_swevent_init,
5626         .add            = perf_swevent_add,
5627         .del            = perf_swevent_del,
5628         .start          = perf_swevent_start,
5629         .stop           = perf_swevent_stop,
5630         .read           = perf_swevent_read,
5631 
5632         .event_idx      = perf_swevent_event_idx,
5633 };
5634 
5635 #ifdef CONFIG_EVENT_TRACING
5636 
5637 static int perf_tp_filter_match(struct perf_event *event,
5638                                 struct perf_sample_data *data)
5639 {
5640         void *record = data->raw->data;
5641 
5642         /* only top level events have filters set */
5643         if (event->parent)
5644                 event = event->parent;
5645 
5646         if (likely(!event->filter) || filter_match_preds(event->filter, record))
5647                 return 1;
5648         return 0;
5649 }
5650 
5651 static int perf_tp_event_match(struct perf_event *event,
5652                                 struct perf_sample_data *data,
5653                                 struct pt_regs *regs)
5654 {
5655         if (event->hw.state & PERF_HES_STOPPED)
5656                 return 0;
5657         /*
5658          * All tracepoints are from kernel-space.
5659          */
5660         if (event->attr.exclude_kernel)
5661                 return 0;
5662 
5663         if (!perf_tp_filter_match(event, data))
5664                 return 0;
5665 
5666         return 1;
5667 }
5668 
5669 void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
5670                    struct pt_regs *regs, struct hlist_head *head, int rctx,
5671                    struct task_struct *task)
5672 {
5673         struct perf_sample_data data;
5674         struct perf_event *event;
5675 
5676         struct perf_raw_record raw = {
5677                 .size = entry_size,
5678                 .data = record,
5679         };
5680 
5681         perf_sample_data_init(&data, addr, 0);
5682         data.raw = &raw;
5683 
5684         hlist_for_each_entry_rcu(event, head, hlist_entry) {
5685                 if (perf_tp_event_match(event, &data, regs))
5686                         perf_swevent_event(event, count, &data, regs);
5687         }
5688 
5689         /*
5690          * If we got specified a target task, also iterate its context and
5691          * deliver this event there too.
5692          */
5693         if (task && task != current) {
5694                 struct perf_event_context *ctx;
5695                 struct trace_entry *entry = record;
5696 
5697                 rcu_read_lock();
5698                 ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]);
5699                 if (!ctx)
5700                         goto unlock;
5701 
5702                 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
5703                         if (event->attr.type != PERF_TYPE_TRACEPOINT)
5704                                 continue;
5705                         if (event->attr.config != entry->type)
5706                                 continue;
5707                         if (perf_tp_event_match(event, &data,