~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/events/core.c

Version: ~ [ linux-5.2-rc1 ] ~ [ linux-5.1.2 ] ~ [ linux-5.0.16 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.43 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.119 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.176 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.179 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.139 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.67 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Performance events core code:
  3  *
  4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  5  *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
  6  *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
  7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  8  *
  9  * For licensing details see kernel-base/COPYING
 10  */
 11 
 12 #include <linux/fs.h>
 13 #include <linux/mm.h>
 14 #include <linux/cpu.h>
 15 #include <linux/smp.h>
 16 #include <linux/idr.h>
 17 #include <linux/file.h>
 18 #include <linux/poll.h>
 19 #include <linux/slab.h>
 20 #include <linux/hash.h>
 21 #include <linux/tick.h>
 22 #include <linux/sysfs.h>
 23 #include <linux/dcache.h>
 24 #include <linux/percpu.h>
 25 #include <linux/ptrace.h>
 26 #include <linux/reboot.h>
 27 #include <linux/vmstat.h>
 28 #include <linux/device.h>
 29 #include <linux/export.h>
 30 #include <linux/vmalloc.h>
 31 #include <linux/hardirq.h>
 32 #include <linux/rculist.h>
 33 #include <linux/uaccess.h>
 34 #include <linux/syscalls.h>
 35 #include <linux/anon_inodes.h>
 36 #include <linux/kernel_stat.h>
 37 #include <linux/cgroup.h>
 38 #include <linux/perf_event.h>
 39 #include <linux/trace_events.h>
 40 #include <linux/hw_breakpoint.h>
 41 #include <linux/mm_types.h>
 42 #include <linux/module.h>
 43 #include <linux/mman.h>
 44 #include <linux/compat.h>
 45 #include <linux/bpf.h>
 46 #include <linux/filter.h>
 47 
 48 #include "internal.h"
 49 
 50 #include <asm/irq_regs.h>
 51 
 52 static struct workqueue_struct *perf_wq;
 53 
 54 typedef int (*remote_function_f)(void *);
 55 
 56 struct remote_function_call {
 57         struct task_struct      *p;
 58         remote_function_f       func;
 59         void                    *info;
 60         int                     ret;
 61 };
 62 
 63 static void remote_function(void *data)
 64 {
 65         struct remote_function_call *tfc = data;
 66         struct task_struct *p = tfc->p;
 67 
 68         if (p) {
 69                 tfc->ret = -EAGAIN;
 70                 if (task_cpu(p) != smp_processor_id() || !task_curr(p))
 71                         return;
 72         }
 73 
 74         tfc->ret = tfc->func(tfc->info);
 75 }
 76 
 77 /**
 78  * task_function_call - call a function on the cpu on which a task runs
 79  * @p:          the task to evaluate
 80  * @func:       the function to be called
 81  * @info:       the function call argument
 82  *
 83  * Calls the function @func when the task is currently running. This might
 84  * be on the current CPU, which just calls the function directly
 85  *
 86  * returns: @func return value, or
 87  *          -ESRCH  - when the process isn't running
 88  *          -EAGAIN - when the process moved away
 89  */
 90 static int
 91 task_function_call(struct task_struct *p, remote_function_f func, void *info)
 92 {
 93         struct remote_function_call data = {
 94                 .p      = p,
 95                 .func   = func,
 96                 .info   = info,
 97                 .ret    = -ESRCH, /* No such (running) process */
 98         };
 99 
100         if (task_curr(p))
101                 smp_call_function_single(task_cpu(p), remote_function, &data, 1);
102 
103         return data.ret;
104 }
105 
106 /**
107  * cpu_function_call - call a function on the cpu
108  * @func:       the function to be called
109  * @info:       the function call argument
110  *
111  * Calls the function @func on the remote cpu.
112  *
113  * returns: @func return value or -ENXIO when the cpu is offline
114  */
115 static int cpu_function_call(int cpu, remote_function_f func, void *info)
116 {
117         struct remote_function_call data = {
118                 .p      = NULL,
119                 .func   = func,
120                 .info   = info,
121                 .ret    = -ENXIO, /* No such CPU */
122         };
123 
124         smp_call_function_single(cpu, remote_function, &data, 1);
125 
126         return data.ret;
127 }
128 
129 #define EVENT_OWNER_KERNEL ((void *) -1)
130 
131 static bool is_kernel_event(struct perf_event *event)
132 {
133         return event->owner == EVENT_OWNER_KERNEL;
134 }
135 
136 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
137                        PERF_FLAG_FD_OUTPUT  |\
138                        PERF_FLAG_PID_CGROUP |\
139                        PERF_FLAG_FD_CLOEXEC)
140 
141 /*
142  * branch priv levels that need permission checks
143  */
144 #define PERF_SAMPLE_BRANCH_PERM_PLM \
145         (PERF_SAMPLE_BRANCH_KERNEL |\
146          PERF_SAMPLE_BRANCH_HV)
147 
148 enum event_type_t {
149         EVENT_FLEXIBLE = 0x1,
150         EVENT_PINNED = 0x2,
151         EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
152 };
153 
154 /*
155  * perf_sched_events : >0 events exist
156  * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
157  */
158 struct static_key_deferred perf_sched_events __read_mostly;
159 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
160 static DEFINE_PER_CPU(int, perf_sched_cb_usages);
161 
162 static atomic_t nr_mmap_events __read_mostly;
163 static atomic_t nr_comm_events __read_mostly;
164 static atomic_t nr_task_events __read_mostly;
165 static atomic_t nr_freq_events __read_mostly;
166 static atomic_t nr_switch_events __read_mostly;
167 
168 static LIST_HEAD(pmus);
169 static DEFINE_MUTEX(pmus_lock);
170 static struct srcu_struct pmus_srcu;
171 
172 /*
173  * perf event paranoia level:
174  *  -1 - not paranoid at all
175  *   0 - disallow raw tracepoint access for unpriv
176  *   1 - disallow cpu events for unpriv
177  *   2 - disallow kernel profiling for unpriv
178  */
179 int sysctl_perf_event_paranoid __read_mostly = 1;
180 
181 /* Minimum for 512 kiB + 1 user control page */
182 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
183 
184 /*
185  * max perf event sample rate
186  */
187 #define DEFAULT_MAX_SAMPLE_RATE         100000
188 #define DEFAULT_SAMPLE_PERIOD_NS        (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)
189 #define DEFAULT_CPU_TIME_MAX_PERCENT    25
190 
191 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
192 
193 static int max_samples_per_tick __read_mostly   = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
194 static int perf_sample_period_ns __read_mostly  = DEFAULT_SAMPLE_PERIOD_NS;
195 
196 static int perf_sample_allowed_ns __read_mostly =
197         DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100;
198 
199 void update_perf_cpu_limits(void)
200 {
201         u64 tmp = perf_sample_period_ns;
202 
203         tmp *= sysctl_perf_cpu_time_max_percent;
204         do_div(tmp, 100);
205         ACCESS_ONCE(perf_sample_allowed_ns) = tmp;
206 }
207 
208 static int perf_rotate_context(struct perf_cpu_context *cpuctx);
209 
210 int perf_proc_update_handler(struct ctl_table *table, int write,
211                 void __user *buffer, size_t *lenp,
212                 loff_t *ppos)
213 {
214         int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
215 
216         if (ret || !write)
217                 return ret;
218 
219         max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
220         perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
221         update_perf_cpu_limits();
222 
223         return 0;
224 }
225 
226 int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT;
227 
228 int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
229                                 void __user *buffer, size_t *lenp,
230                                 loff_t *ppos)
231 {
232         int ret = proc_dointvec(table, write, buffer, lenp, ppos);
233 
234         if (ret || !write)
235                 return ret;
236 
237         update_perf_cpu_limits();
238 
239         return 0;
240 }
241 
242 /*
243  * perf samples are done in some very critical code paths (NMIs).
244  * If they take too much CPU time, the system can lock up and not
245  * get any real work done.  This will drop the sample rate when
246  * we detect that events are taking too long.
247  */
248 #define NR_ACCUMULATED_SAMPLES 128
249 static DEFINE_PER_CPU(u64, running_sample_length);
250 
251 static void perf_duration_warn(struct irq_work *w)
252 {
253         u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns);
254         u64 avg_local_sample_len;
255         u64 local_samples_len;
256 
257         local_samples_len = __this_cpu_read(running_sample_length);
258         avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES;
259 
260         printk_ratelimited(KERN_WARNING
261                         "perf interrupt took too long (%lld > %lld), lowering "
262                         "kernel.perf_event_max_sample_rate to %d\n",
263                         avg_local_sample_len, allowed_ns >> 1,
264                         sysctl_perf_event_sample_rate);
265 }
266 
267 static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn);
268 
269 void perf_sample_event_took(u64 sample_len_ns)
270 {
271         u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns);
272         u64 avg_local_sample_len;
273         u64 local_samples_len;
274 
275         if (allowed_ns == 0)
276                 return;
277 
278         /* decay the counter by 1 average sample */
279         local_samples_len = __this_cpu_read(running_sample_length);
280         local_samples_len -= local_samples_len/NR_ACCUMULATED_SAMPLES;
281         local_samples_len += sample_len_ns;
282         __this_cpu_write(running_sample_length, local_samples_len);
283 
284         /*
285          * note: this will be biased artifically low until we have
286          * seen NR_ACCUMULATED_SAMPLES.  Doing it this way keeps us
287          * from having to maintain a count.
288          */
289         avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES;
290 
291         if (avg_local_sample_len <= allowed_ns)
292                 return;
293 
294         if (max_samples_per_tick <= 1)
295                 return;
296 
297         max_samples_per_tick = DIV_ROUND_UP(max_samples_per_tick, 2);
298         sysctl_perf_event_sample_rate = max_samples_per_tick * HZ;
299         perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
300 
301         update_perf_cpu_limits();
302 
303         if (!irq_work_queue(&perf_duration_work)) {
304                 early_printk("perf interrupt took too long (%lld > %lld), lowering "
305                              "kernel.perf_event_max_sample_rate to %d\n",
306                              avg_local_sample_len, allowed_ns >> 1,
307                              sysctl_perf_event_sample_rate);
308         }
309 }
310 
311 static atomic64_t perf_event_id;
312 
313 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
314                               enum event_type_t event_type);
315 
316 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
317                              enum event_type_t event_type,
318                              struct task_struct *task);
319 
320 static void update_context_time(struct perf_event_context *ctx);
321 static u64 perf_event_time(struct perf_event *event);
322 
323 void __weak perf_event_print_debug(void)        { }
324 
325 extern __weak const char *perf_pmu_name(void)
326 {
327         return "pmu";
328 }
329 
330 static inline u64 perf_clock(void)
331 {
332         return local_clock();
333 }
334 
335 static inline u64 perf_event_clock(struct perf_event *event)
336 {
337         return event->clock();
338 }
339 
340 static inline struct perf_cpu_context *
341 __get_cpu_context(struct perf_event_context *ctx)
342 {
343         return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
344 }
345 
346 static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
347                           struct perf_event_context *ctx)
348 {
349         raw_spin_lock(&cpuctx->ctx.lock);
350         if (ctx)
351                 raw_spin_lock(&ctx->lock);
352 }
353 
354 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
355                             struct perf_event_context *ctx)
356 {
357         if (ctx)
358                 raw_spin_unlock(&ctx->lock);
359         raw_spin_unlock(&cpuctx->ctx.lock);
360 }
361 
362 #ifdef CONFIG_CGROUP_PERF
363 
364 static inline bool
365 perf_cgroup_match(struct perf_event *event)
366 {
367         struct perf_event_context *ctx = event->ctx;
368         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
369 
370         /* @event doesn't care about cgroup */
371         if (!event->cgrp)
372                 return true;
373 
374         /* wants specific cgroup scope but @cpuctx isn't associated with any */
375         if (!cpuctx->cgrp)
376                 return false;
377 
378         /*
379          * Cgroup scoping is recursive.  An event enabled for a cgroup is
380          * also enabled for all its descendant cgroups.  If @cpuctx's
381          * cgroup is a descendant of @event's (the test covers identity
382          * case), it's a match.
383          */
384         return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,
385                                     event->cgrp->css.cgroup);
386 }
387 
388 static inline void perf_detach_cgroup(struct perf_event *event)
389 {
390         css_put(&event->cgrp->css);
391         event->cgrp = NULL;
392 }
393 
394 static inline int is_cgroup_event(struct perf_event *event)
395 {
396         return event->cgrp != NULL;
397 }
398 
399 static inline u64 perf_cgroup_event_time(struct perf_event *event)
400 {
401         struct perf_cgroup_info *t;
402 
403         t = per_cpu_ptr(event->cgrp->info, event->cpu);
404         return t->time;
405 }
406 
407 static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
408 {
409         struct perf_cgroup_info *info;
410         u64 now;
411 
412         now = perf_clock();
413 
414         info = this_cpu_ptr(cgrp->info);
415 
416         info->time += now - info->timestamp;
417         info->timestamp = now;
418 }
419 
420 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
421 {
422         struct perf_cgroup *cgrp_out = cpuctx->cgrp;
423         if (cgrp_out)
424                 __update_cgrp_time(cgrp_out);
425 }
426 
427 static inline void update_cgrp_time_from_event(struct perf_event *event)
428 {
429         struct perf_cgroup *cgrp;
430 
431         /*
432          * ensure we access cgroup data only when needed and
433          * when we know the cgroup is pinned (css_get)
434          */
435         if (!is_cgroup_event(event))
436                 return;
437 
438         cgrp = perf_cgroup_from_task(current);
439         /*
440          * Do not update time when cgroup is not active
441          */
442         if (cgrp == event->cgrp)
443                 __update_cgrp_time(event->cgrp);
444 }
445 
446 static inline void
447 perf_cgroup_set_timestamp(struct task_struct *task,
448                           struct perf_event_context *ctx)
449 {
450         struct perf_cgroup *cgrp;
451         struct perf_cgroup_info *info;
452 
453         /*
454          * ctx->lock held by caller
455          * ensure we do not access cgroup data
456          * unless we have the cgroup pinned (css_get)
457          */
458         if (!task || !ctx->nr_cgroups)
459                 return;
460 
461         cgrp = perf_cgroup_from_task(task);
462         info = this_cpu_ptr(cgrp->info);
463         info->timestamp = ctx->timestamp;
464 }
465 
466 #define PERF_CGROUP_SWOUT       0x1 /* cgroup switch out every event */
467 #define PERF_CGROUP_SWIN        0x2 /* cgroup switch in events based on task */
468 
469 /*
470  * reschedule events based on the cgroup constraint of task.
471  *
472  * mode SWOUT : schedule out everything
473  * mode SWIN : schedule in based on cgroup for next
474  */
475 void perf_cgroup_switch(struct task_struct *task, int mode)
476 {
477         struct perf_cpu_context *cpuctx;
478         struct pmu *pmu;
479         unsigned long flags;
480 
481         /*
482          * disable interrupts to avoid geting nr_cgroup
483          * changes via __perf_event_disable(). Also
484          * avoids preemption.
485          */
486         local_irq_save(flags);
487 
488         /*
489          * we reschedule only in the presence of cgroup
490          * constrained events.
491          */
492         rcu_read_lock();
493 
494         list_for_each_entry_rcu(pmu, &pmus, entry) {
495                 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
496                 if (cpuctx->unique_pmu != pmu)
497                         continue; /* ensure we process each cpuctx once */
498 
499                 /*
500                  * perf_cgroup_events says at least one
501                  * context on this CPU has cgroup events.
502                  *
503                  * ctx->nr_cgroups reports the number of cgroup
504                  * events for a context.
505                  */
506                 if (cpuctx->ctx.nr_cgroups > 0) {
507                         perf_ctx_lock(cpuctx, cpuctx->task_ctx);
508                         perf_pmu_disable(cpuctx->ctx.pmu);
509 
510                         if (mode & PERF_CGROUP_SWOUT) {
511                                 cpu_ctx_sched_out(cpuctx, EVENT_ALL);
512                                 /*
513                                  * must not be done before ctxswout due
514                                  * to event_filter_match() in event_sched_out()
515                                  */
516                                 cpuctx->cgrp = NULL;
517                         }
518 
519                         if (mode & PERF_CGROUP_SWIN) {
520                                 WARN_ON_ONCE(cpuctx->cgrp);
521                                 /*
522                                  * set cgrp before ctxsw in to allow
523                                  * event_filter_match() to not have to pass
524                                  * task around
525                                  */
526                                 cpuctx->cgrp = perf_cgroup_from_task(task);
527                                 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
528                         }
529                         perf_pmu_enable(cpuctx->ctx.pmu);
530                         perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
531                 }
532         }
533 
534         rcu_read_unlock();
535 
536         local_irq_restore(flags);
537 }
538 
539 static inline void perf_cgroup_sched_out(struct task_struct *task,
540                                          struct task_struct *next)
541 {
542         struct perf_cgroup *cgrp1;
543         struct perf_cgroup *cgrp2 = NULL;
544 
545         /*
546          * we come here when we know perf_cgroup_events > 0
547          */
548         cgrp1 = perf_cgroup_from_task(task);
549 
550         /*
551          * next is NULL when called from perf_event_enable_on_exec()
552          * that will systematically cause a cgroup_switch()
553          */
554         if (next)
555                 cgrp2 = perf_cgroup_from_task(next);
556 
557         /*
558          * only schedule out current cgroup events if we know
559          * that we are switching to a different cgroup. Otherwise,
560          * do no touch the cgroup events.
561          */
562         if (cgrp1 != cgrp2)
563                 perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
564 }
565 
566 static inline void perf_cgroup_sched_in(struct task_struct *prev,
567                                         struct task_struct *task)
568 {
569         struct perf_cgroup *cgrp1;
570         struct perf_cgroup *cgrp2 = NULL;
571 
572         /*
573          * we come here when we know perf_cgroup_events > 0
574          */
575         cgrp1 = perf_cgroup_from_task(task);
576 
577         /* prev can never be NULL */
578         cgrp2 = perf_cgroup_from_task(prev);
579 
580         /*
581          * only need to schedule in cgroup events if we are changing
582          * cgroup during ctxsw. Cgroup events were not scheduled
583          * out of ctxsw out if that was not the case.
584          */
585         if (cgrp1 != cgrp2)
586                 perf_cgroup_switch(task, PERF_CGROUP_SWIN);
587 }
588 
589 static inline int perf_cgroup_connect(int fd, struct perf_event *event,
590                                       struct perf_event_attr *attr,
591                                       struct perf_event *group_leader)
592 {
593         struct perf_cgroup *cgrp;
594         struct cgroup_subsys_state *css;
595         struct fd f = fdget(fd);
596         int ret = 0;
597 
598         if (!f.file)
599                 return -EBADF;
600 
601         css = css_tryget_online_from_dir(f.file->f_path.dentry,
602                                          &perf_event_cgrp_subsys);
603         if (IS_ERR(css)) {
604                 ret = PTR_ERR(css);
605                 goto out;
606         }
607 
608         cgrp = container_of(css, struct perf_cgroup, css);
609         event->cgrp = cgrp;
610 
611         /*
612          * all events in a group must monitor
613          * the same cgroup because a task belongs
614          * to only one perf cgroup at a time
615          */
616         if (group_leader && group_leader->cgrp != cgrp) {
617                 perf_detach_cgroup(event);
618                 ret = -EINVAL;
619         }
620 out:
621         fdput(f);
622         return ret;
623 }
624 
625 static inline void
626 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
627 {
628         struct perf_cgroup_info *t;
629         t = per_cpu_ptr(event->cgrp->info, event->cpu);
630         event->shadow_ctx_time = now - t->timestamp;
631 }
632 
633 static inline void
634 perf_cgroup_defer_enabled(struct perf_event *event)
635 {
636         /*
637          * when the current task's perf cgroup does not match
638          * the event's, we need to remember to call the
639          * perf_mark_enable() function the first time a task with
640          * a matching perf cgroup is scheduled in.
641          */
642         if (is_cgroup_event(event) && !perf_cgroup_match(event))
643                 event->cgrp_defer_enabled = 1;
644 }
645 
646 static inline void
647 perf_cgroup_mark_enabled(struct perf_event *event,
648                          struct perf_event_context *ctx)
649 {
650         struct perf_event *sub;
651         u64 tstamp = perf_event_time(event);
652 
653         if (!event->cgrp_defer_enabled)
654                 return;
655 
656         event->cgrp_defer_enabled = 0;
657 
658         event->tstamp_enabled = tstamp - event->total_time_enabled;
659         list_for_each_entry(sub, &event->sibling_list, group_entry) {
660                 if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
661                         sub->tstamp_enabled = tstamp - sub->total_time_enabled;
662                         sub->cgrp_defer_enabled = 0;
663                 }
664         }
665 }
666 #else /* !CONFIG_CGROUP_PERF */
667 
668 static inline bool
669 perf_cgroup_match(struct perf_event *event)
670 {
671         return true;
672 }
673 
674 static inline void perf_detach_cgroup(struct perf_event *event)
675 {}
676 
677 static inline int is_cgroup_event(struct perf_event *event)
678 {
679         return 0;
680 }
681 
682 static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
683 {
684         return 0;
685 }
686 
687 static inline void update_cgrp_time_from_event(struct perf_event *event)
688 {
689 }
690 
691 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
692 {
693 }
694 
695 static inline void perf_cgroup_sched_out(struct task_struct *task,
696                                          struct task_struct *next)
697 {
698 }
699 
700 static inline void perf_cgroup_sched_in(struct task_struct *prev,
701                                         struct task_struct *task)
702 {
703 }
704 
705 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
706                                       struct perf_event_attr *attr,
707                                       struct perf_event *group_leader)
708 {
709         return -EINVAL;
710 }
711 
712 static inline void
713 perf_cgroup_set_timestamp(struct task_struct *task,
714                           struct perf_event_context *ctx)
715 {
716 }
717 
718 void
719 perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
720 {
721 }
722 
723 static inline void
724 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
725 {
726 }
727 
728 static inline u64 perf_cgroup_event_time(struct perf_event *event)
729 {
730         return 0;
731 }
732 
733 static inline void
734 perf_cgroup_defer_enabled(struct perf_event *event)
735 {
736 }
737 
738 static inline void
739 perf_cgroup_mark_enabled(struct perf_event *event,
740                          struct perf_event_context *ctx)
741 {
742 }
743 #endif
744 
745 /*
746  * set default to be dependent on timer tick just
747  * like original code
748  */
749 #define PERF_CPU_HRTIMER (1000 / HZ)
750 /*
751  * function must be called with interrupts disbled
752  */
753 static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr)
754 {
755         struct perf_cpu_context *cpuctx;
756         int rotations = 0;
757 
758         WARN_ON(!irqs_disabled());
759 
760         cpuctx = container_of(hr, struct perf_cpu_context, hrtimer);
761         rotations = perf_rotate_context(cpuctx);
762 
763         raw_spin_lock(&cpuctx->hrtimer_lock);
764         if (rotations)
765                 hrtimer_forward_now(hr, cpuctx->hrtimer_interval);
766         else
767                 cpuctx->hrtimer_active = 0;
768         raw_spin_unlock(&cpuctx->hrtimer_lock);
769 
770         return rotations ? HRTIMER_RESTART : HRTIMER_NORESTART;
771 }
772 
773 static void __perf_mux_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu)
774 {
775         struct hrtimer *timer = &cpuctx->hrtimer;
776         struct pmu *pmu = cpuctx->ctx.pmu;
777         u64 interval;
778 
779         /* no multiplexing needed for SW PMU */
780         if (pmu->task_ctx_nr == perf_sw_context)
781                 return;
782 
783         /*
784          * check default is sane, if not set then force to
785          * default interval (1/tick)
786          */
787         interval = pmu->hrtimer_interval_ms;
788         if (interval < 1)
789                 interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;
790 
791         cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);
792 
793         raw_spin_lock_init(&cpuctx->hrtimer_lock);
794         hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
795         timer->function = perf_mux_hrtimer_handler;
796 }
797 
798 static int perf_mux_hrtimer_restart(struct perf_cpu_context *cpuctx)
799 {
800         struct hrtimer *timer = &cpuctx->hrtimer;
801         struct pmu *pmu = cpuctx->ctx.pmu;
802         unsigned long flags;
803 
804         /* not for SW PMU */
805         if (pmu->task_ctx_nr == perf_sw_context)
806                 return 0;
807 
808         raw_spin_lock_irqsave(&cpuctx->hrtimer_lock, flags);
809         if (!cpuctx->hrtimer_active) {
810                 cpuctx->hrtimer_active = 1;
811                 hrtimer_forward_now(timer, cpuctx->hrtimer_interval);
812                 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
813         }
814         raw_spin_unlock_irqrestore(&cpuctx->hrtimer_lock, flags);
815 
816         return 0;
817 }
818 
819 void perf_pmu_disable(struct pmu *pmu)
820 {
821         int *count = this_cpu_ptr(pmu->pmu_disable_count);
822         if (!(*count)++)
823                 pmu->pmu_disable(pmu);
824 }
825 
826 void perf_pmu_enable(struct pmu *pmu)
827 {
828         int *count = this_cpu_ptr(pmu->pmu_disable_count);
829         if (!--(*count))
830                 pmu->pmu_enable(pmu);
831 }
832 
833 static DEFINE_PER_CPU(struct list_head, active_ctx_list);
834 
835 /*
836  * perf_event_ctx_activate(), perf_event_ctx_deactivate(), and
837  * perf_event_task_tick() are fully serialized because they're strictly cpu
838  * affine and perf_event_ctx{activate,deactivate} are called with IRQs
839  * disabled, while perf_event_task_tick is called from IRQ context.
840  */
841 static void perf_event_ctx_activate(struct perf_event_context *ctx)
842 {
843         struct list_head *head = this_cpu_ptr(&active_ctx_list);
844 
845         WARN_ON(!irqs_disabled());
846 
847         WARN_ON(!list_empty(&ctx->active_ctx_list));
848 
849         list_add(&ctx->active_ctx_list, head);
850 }
851 
852 static void perf_event_ctx_deactivate(struct perf_event_context *ctx)
853 {
854         WARN_ON(!irqs_disabled());
855 
856         WARN_ON(list_empty(&ctx->active_ctx_list));
857 
858         list_del_init(&ctx->active_ctx_list);
859 }
860 
861 static void get_ctx(struct perf_event_context *ctx)
862 {
863         WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
864 }
865 
866 static void free_ctx(struct rcu_head *head)
867 {
868         struct perf_event_context *ctx;
869 
870         ctx = container_of(head, struct perf_event_context, rcu_head);
871         kfree(ctx->task_ctx_data);
872         kfree(ctx);
873 }
874 
875 static void put_ctx(struct perf_event_context *ctx)
876 {
877         if (atomic_dec_and_test(&ctx->refcount)) {
878                 if (ctx->parent_ctx)
879                         put_ctx(ctx->parent_ctx);
880                 if (ctx->task)
881                         put_task_struct(ctx->task);
882                 call_rcu(&ctx->rcu_head, free_ctx);
883         }
884 }
885 
886 /*
887  * Because of perf_event::ctx migration in sys_perf_event_open::move_group and
888  * perf_pmu_migrate_context() we need some magic.
889  *
890  * Those places that change perf_event::ctx will hold both
891  * perf_event_ctx::mutex of the 'old' and 'new' ctx value.
892  *
893  * Lock ordering is by mutex address. There are two other sites where
894  * perf_event_context::mutex nests and those are:
895  *
896  *  - perf_event_exit_task_context()    [ child , 0 ]
897  *      __perf_event_exit_task()
898  *        sync_child_event()
899  *          put_event()                 [ parent, 1 ]
900  *
901  *  - perf_event_init_context()         [ parent, 0 ]
902  *      inherit_task_group()
903  *        inherit_group()
904  *          inherit_event()
905  *            perf_event_alloc()
906  *              perf_init_event()
907  *                perf_try_init_event() [ child , 1 ]
908  *
909  * While it appears there is an obvious deadlock here -- the parent and child
910  * nesting levels are inverted between the two. This is in fact safe because
911  * life-time rules separate them. That is an exiting task cannot fork, and a
912  * spawning task cannot (yet) exit.
913  *
914  * But remember that that these are parent<->child context relations, and
915  * migration does not affect children, therefore these two orderings should not
916  * interact.
917  *
918  * The change in perf_event::ctx does not affect children (as claimed above)
919  * because the sys_perf_event_open() case will install a new event and break
920  * the ctx parent<->child relation, and perf_pmu_migrate_context() is only
921  * concerned with cpuctx and that doesn't have children.
922  *
923  * The places that change perf_event::ctx will issue:
924  *
925  *   perf_remove_from_context();
926  *   synchronize_rcu();
927  *   perf_install_in_context();
928  *
929  * to affect the change. The remove_from_context() + synchronize_rcu() should
930  * quiesce the event, after which we can install it in the new location. This
931  * means that only external vectors (perf_fops, prctl) can perturb the event
932  * while in transit. Therefore all such accessors should also acquire
933  * perf_event_context::mutex to serialize against this.
934  *
935  * However; because event->ctx can change while we're waiting to acquire
936  * ctx->mutex we must be careful and use the below perf_event_ctx_lock()
937  * function.
938  *
939  * Lock order:
940  *      task_struct::perf_event_mutex
941  *        perf_event_context::mutex
942  *          perf_event_context::lock
943  *          perf_event::child_mutex;
944  *          perf_event::mmap_mutex
945  *          mmap_sem
946  */
947 static struct perf_event_context *
948 perf_event_ctx_lock_nested(struct perf_event *event, int nesting)
949 {
950         struct perf_event_context *ctx;
951 
952 again:
953         rcu_read_lock();
954         ctx = ACCESS_ONCE(event->ctx);
955         if (!atomic_inc_not_zero(&ctx->refcount)) {
956                 rcu_read_unlock();
957                 goto again;
958         }
959         rcu_read_unlock();
960 
961         mutex_lock_nested(&ctx->mutex, nesting);
962         if (event->ctx != ctx) {
963                 mutex_unlock(&ctx->mutex);
964                 put_ctx(ctx);
965                 goto again;
966         }
967 
968         return ctx;
969 }
970 
971 static inline struct perf_event_context *
972 perf_event_ctx_lock(struct perf_event *event)
973 {
974         return perf_event_ctx_lock_nested(event, 0);
975 }
976 
977 static void perf_event_ctx_unlock(struct perf_event *event,
978                                   struct perf_event_context *ctx)
979 {
980         mutex_unlock(&ctx->mutex);
981         put_ctx(ctx);
982 }
983 
984 /*
985  * This must be done under the ctx->lock, such as to serialize against
986  * context_equiv(), therefore we cannot call put_ctx() since that might end up
987  * calling scheduler related locks and ctx->lock nests inside those.
988  */
989 static __must_check struct perf_event_context *
990 unclone_ctx(struct perf_event_context *ctx)
991 {
992         struct perf_event_context *parent_ctx = ctx->parent_ctx;
993 
994         lockdep_assert_held(&ctx->lock);
995 
996         if (parent_ctx)
997                 ctx->parent_ctx = NULL;
998         ctx->generation++;
999 
1000         return parent_ctx;
1001 }
1002 
1003 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
1004 {
1005         /*
1006          * only top level events have the pid namespace they were created in
1007          */
1008         if (event->parent)
1009                 event = event->parent;
1010 
1011         return task_tgid_nr_ns(p, event->ns);
1012 }
1013 
1014 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
1015 {
1016         /*
1017          * only top level events have the pid namespace they were created in
1018          */
1019         if (event->parent)
1020                 event = event->parent;
1021 
1022         return task_pid_nr_ns(p, event->ns);
1023 }
1024 
1025 /*
1026  * If we inherit events we want to return the parent event id
1027  * to userspace.
1028  */
1029 static u64 primary_event_id(struct perf_event *event)
1030 {
1031         u64 id = event->id;
1032 
1033         if (event->parent)
1034                 id = event->parent->id;
1035 
1036         return id;
1037 }
1038 
1039 /*
1040  * Get the perf_event_context for a task and lock it.
1041  * This has to cope with with the fact that until it is locked,
1042  * the context could get moved to another task.
1043  */
1044 static struct perf_event_context *
1045 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
1046 {
1047         struct perf_event_context *ctx;
1048 
1049 retry:
1050         /*
1051          * One of the few rules of preemptible RCU is that one cannot do
1052          * rcu_read_unlock() while holding a scheduler (or nested) lock when
1053          * part of the read side critical section was preemptible -- see
1054          * rcu_read_unlock_special().
1055          *
1056          * Since ctx->lock nests under rq->lock we must ensure the entire read
1057          * side critical section is non-preemptible.
1058          */
1059         preempt_disable();
1060         rcu_read_lock();
1061         ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
1062         if (ctx) {
1063                 /*
1064                  * If this context is a clone of another, it might
1065                  * get swapped for another underneath us by
1066                  * perf_event_task_sched_out, though the
1067                  * rcu_read_lock() protects us from any context
1068                  * getting freed.  Lock the context and check if it
1069                  * got swapped before we could get the lock, and retry
1070                  * if so.  If we locked the right context, then it
1071                  * can't get swapped on us any more.
1072                  */
1073                 raw_spin_lock_irqsave(&ctx->lock, *flags);
1074                 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
1075                         raw_spin_unlock_irqrestore(&ctx->lock, *flags);
1076                         rcu_read_unlock();
1077                         preempt_enable();
1078                         goto retry;
1079                 }
1080 
1081                 if (!atomic_inc_not_zero(&ctx->refcount)) {
1082                         raw_spin_unlock_irqrestore(&ctx->lock, *flags);
1083                         ctx = NULL;
1084                 }
1085         }
1086         rcu_read_unlock();
1087         preempt_enable();
1088         return ctx;
1089 }
1090 
1091 /*
1092  * Get the context for a task and increment its pin_count so it
1093  * can't get swapped to another task.  This also increments its
1094  * reference count so that the context can't get freed.
1095  */
1096 static struct perf_event_context *
1097 perf_pin_task_context(struct task_struct *task, int ctxn)
1098 {
1099         struct perf_event_context *ctx;
1100         unsigned long flags;
1101 
1102         ctx = perf_lock_task_context(task, ctxn, &flags);
1103         if (ctx) {
1104                 ++ctx->pin_count;
1105                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
1106         }
1107         return ctx;
1108 }
1109 
1110 static void perf_unpin_context(struct perf_event_context *ctx)
1111 {
1112         unsigned long flags;
1113 
1114         raw_spin_lock_irqsave(&ctx->lock, flags);
1115         --ctx->pin_count;
1116         raw_spin_unlock_irqrestore(&ctx->lock, flags);
1117 }
1118 
1119 /*
1120  * Update the record of the current time in a context.
1121  */
1122 static void update_context_time(struct perf_event_context *ctx)
1123 {
1124         u64 now = perf_clock();
1125 
1126         ctx->time += now - ctx->timestamp;
1127         ctx->timestamp = now;
1128 }
1129 
1130 static u64 perf_event_time(struct perf_event *event)
1131 {
1132         struct perf_event_context *ctx = event->ctx;
1133 
1134         if (is_cgroup_event(event))
1135                 return perf_cgroup_event_time(event);
1136 
1137         return ctx ? ctx->time : 0;
1138 }
1139 
1140 /*
1141  * Update the total_time_enabled and total_time_running fields for a event.
1142  * The caller of this function needs to hold the ctx->lock.
1143  */
1144 static void update_event_times(struct perf_event *event)
1145 {
1146         struct perf_event_context *ctx = event->ctx;
1147         u64 run_end;
1148 
1149         if (event->state < PERF_EVENT_STATE_INACTIVE ||
1150             event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
1151                 return;
1152         /*
1153          * in cgroup mode, time_enabled represents
1154          * the time the event was enabled AND active
1155          * tasks were in the monitored cgroup. This is
1156          * independent of the activity of the context as
1157          * there may be a mix of cgroup and non-cgroup events.
1158          *
1159          * That is why we treat cgroup events differently
1160          * here.
1161          */
1162         if (is_cgroup_event(event))
1163                 run_end = perf_cgroup_event_time(event);
1164         else if (ctx->is_active)
1165                 run_end = ctx->time;
1166         else
1167                 run_end = event->tstamp_stopped;
1168 
1169         event->total_time_enabled = run_end - event->tstamp_enabled;
1170 
1171         if (event->state == PERF_EVENT_STATE_INACTIVE)
1172                 run_end = event->tstamp_stopped;
1173         else
1174                 run_end = perf_event_time(event);
1175 
1176         event->total_time_running = run_end - event->tstamp_running;
1177 
1178 }
1179 
1180 /*
1181  * Update total_time_enabled and total_time_running for all events in a group.
1182  */
1183 static void update_group_times(struct perf_event *leader)
1184 {
1185         struct perf_event *event;
1186 
1187         update_event_times(leader);
1188         list_for_each_entry(event, &leader->sibling_list, group_entry)
1189                 update_event_times(event);
1190 }
1191 
1192 static struct list_head *
1193 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
1194 {
1195         if (event->attr.pinned)
1196                 return &ctx->pinned_groups;
1197         else
1198                 return &ctx->flexible_groups;
1199 }
1200 
1201 /*
1202  * Add a event from the lists for its context.
1203  * Must be called with ctx->mutex and ctx->lock held.
1204  */
1205 static void
1206 list_add_event(struct perf_event *event, struct perf_event_context *ctx)
1207 {
1208         WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
1209         event->attach_state |= PERF_ATTACH_CONTEXT;
1210 
1211         /*
1212          * If we're a stand alone event or group leader, we go to the context
1213          * list, group events are kept attached to the group so that
1214          * perf_group_detach can, at all times, locate all siblings.
1215          */
1216         if (event->group_leader == event) {
1217                 struct list_head *list;
1218 
1219                 if (is_software_event(event))
1220                         event->group_flags |= PERF_GROUP_SOFTWARE;
1221 
1222                 list = ctx_group_list(event, ctx);
1223                 list_add_tail(&event->group_entry, list);
1224         }
1225 
1226         if (is_cgroup_event(event))
1227                 ctx->nr_cgroups++;
1228 
1229         list_add_rcu(&event->event_entry, &ctx->event_list);
1230         ctx->nr_events++;
1231         if (event->attr.inherit_stat)
1232                 ctx->nr_stat++;
1233 
1234         ctx->generation++;
1235 }
1236 
1237 /*
1238  * Initialize event state based on the perf_event_attr::disabled.
1239  */
1240 static inline void perf_event__state_init(struct perf_event *event)
1241 {
1242         event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
1243                                               PERF_EVENT_STATE_INACTIVE;
1244 }
1245 
1246 static void __perf_event_read_size(struct perf_event *event, int nr_siblings)
1247 {
1248         int entry = sizeof(u64); /* value */
1249         int size = 0;
1250         int nr = 1;
1251 
1252         if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1253                 size += sizeof(u64);
1254 
1255         if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1256                 size += sizeof(u64);
1257 
1258         if (event->attr.read_format & PERF_FORMAT_ID)
1259                 entry += sizeof(u64);
1260 
1261         if (event->attr.read_format & PERF_FORMAT_GROUP) {
1262                 nr += nr_siblings;
1263                 size += sizeof(u64);
1264         }
1265 
1266         size += entry * nr;
1267         event->read_size = size;
1268 }
1269 
1270 static void __perf_event_header_size(struct perf_event *event, u64 sample_type)
1271 {
1272         struct perf_sample_data *data;
1273         u16 size = 0;
1274 
1275         if (sample_type & PERF_SAMPLE_IP)
1276                 size += sizeof(data->ip);
1277 
1278         if (sample_type & PERF_SAMPLE_ADDR)
1279                 size += sizeof(data->addr);
1280 
1281         if (sample_type & PERF_SAMPLE_PERIOD)
1282                 size += sizeof(data->period);
1283 
1284         if (sample_type & PERF_SAMPLE_WEIGHT)
1285                 size += sizeof(data->weight);
1286 
1287         if (sample_type & PERF_SAMPLE_READ)
1288                 size += event->read_size;
1289 
1290         if (sample_type & PERF_SAMPLE_DATA_SRC)
1291                 size += sizeof(data->data_src.val);
1292 
1293         if (sample_type & PERF_SAMPLE_TRANSACTION)
1294                 size += sizeof(data->txn);
1295 
1296         event->header_size = size;
1297 }
1298 
1299 /*
1300  * Called at perf_event creation and when events are attached/detached from a
1301  * group.
1302  */
1303 static void perf_event__header_size(struct perf_event *event)
1304 {
1305         __perf_event_read_size(event,
1306                                event->group_leader->nr_siblings);
1307         __perf_event_header_size(event, event->attr.sample_type);
1308 }
1309 
1310 static void perf_event__id_header_size(struct perf_event *event)
1311 {
1312         struct perf_sample_data *data;
1313         u64 sample_type = event->attr.sample_type;
1314         u16 size = 0;
1315 
1316         if (sample_type & PERF_SAMPLE_TID)
1317                 size += sizeof(data->tid_entry);
1318 
1319         if (sample_type & PERF_SAMPLE_TIME)
1320                 size += sizeof(data->time);
1321 
1322         if (sample_type & PERF_SAMPLE_IDENTIFIER)
1323                 size += sizeof(data->id);
1324 
1325         if (sample_type & PERF_SAMPLE_ID)
1326                 size += sizeof(data->id);
1327 
1328         if (sample_type & PERF_SAMPLE_STREAM_ID)
1329                 size += sizeof(data->stream_id);
1330 
1331         if (sample_type & PERF_SAMPLE_CPU)
1332                 size += sizeof(data->cpu_entry);
1333 
1334         event->id_header_size = size;
1335 }
1336 
1337 static bool perf_event_validate_size(struct perf_event *event)
1338 {
1339         /*
1340          * The values computed here will be over-written when we actually
1341          * attach the event.
1342          */
1343         __perf_event_read_size(event, event->group_leader->nr_siblings + 1);
1344         __perf_event_header_size(event, event->attr.sample_type & ~PERF_SAMPLE_READ);
1345         perf_event__id_header_size(event);
1346 
1347         /*
1348          * Sum the lot; should not exceed the 64k limit we have on records.
1349          * Conservative limit to allow for callchains and other variable fields.
1350          */
1351         if (event->read_size + event->header_size +
1352             event->id_header_size + sizeof(struct perf_event_header) >= 16*1024)
1353                 return false;
1354 
1355         return true;
1356 }
1357 
1358 static void perf_group_attach(struct perf_event *event)
1359 {
1360         struct perf_event *group_leader = event->group_leader, *pos;
1361 
1362         /*
1363          * We can have double attach due to group movement in perf_event_open.
1364          */
1365         if (event->attach_state & PERF_ATTACH_GROUP)
1366                 return;
1367 
1368         event->attach_state |= PERF_ATTACH_GROUP;
1369 
1370         if (group_leader == event)
1371                 return;
1372 
1373         WARN_ON_ONCE(group_leader->ctx != event->ctx);
1374 
1375         if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
1376                         !is_software_event(event))
1377                 group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
1378 
1379         list_add_tail(&event->group_entry, &group_leader->sibling_list);
1380         group_leader->nr_siblings++;
1381 
1382         perf_event__header_size(group_leader);
1383 
1384         list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
1385                 perf_event__header_size(pos);
1386 }
1387 
1388 /*
1389  * Remove a event from the lists for its context.
1390  * Must be called with ctx->mutex and ctx->lock held.
1391  */
1392 static void
1393 list_del_event(struct perf_event *event, struct perf_event_context *ctx)
1394 {
1395         struct perf_cpu_context *cpuctx;
1396 
1397         WARN_ON_ONCE(event->ctx != ctx);
1398         lockdep_assert_held(&ctx->lock);
1399 
1400         /*
1401          * We can have double detach due to exit/hot-unplug + close.
1402          */
1403         if (!(event->attach_state & PERF_ATTACH_CONTEXT))
1404                 return;
1405 
1406         event->attach_state &= ~PERF_ATTACH_CONTEXT;
1407 
1408         if (is_cgroup_event(event)) {
1409                 ctx->nr_cgroups--;
1410                 cpuctx = __get_cpu_context(ctx);
1411                 /*
1412                  * if there are no more cgroup events
1413                  * then cler cgrp to avoid stale pointer
1414                  * in update_cgrp_time_from_cpuctx()
1415                  */
1416                 if (!ctx->nr_cgroups)
1417                         cpuctx->cgrp = NULL;
1418         }
1419 
1420         ctx->nr_events--;
1421         if (event->attr.inherit_stat)
1422                 ctx->nr_stat--;
1423 
1424         list_del_rcu(&event->event_entry);
1425 
1426         if (event->group_leader == event)
1427                 list_del_init(&event->group_entry);
1428 
1429         update_group_times(event);
1430 
1431         /*
1432          * If event was in error state, then keep it
1433          * that way, otherwise bogus counts will be
1434          * returned on read(). The only way to get out
1435          * of error state is by explicit re-enabling
1436          * of the event
1437          */
1438         if (event->state > PERF_EVENT_STATE_OFF)
1439                 event->state = PERF_EVENT_STATE_OFF;
1440 
1441         ctx->generation++;
1442 }
1443 
1444 static void perf_group_detach(struct perf_event *event)
1445 {
1446         struct perf_event *sibling, *tmp;
1447         struct list_head *list = NULL;
1448 
1449         /*
1450          * We can have double detach due to exit/hot-unplug + close.
1451          */
1452         if (!(event->attach_state & PERF_ATTACH_GROUP))
1453                 return;
1454 
1455         event->attach_state &= ~PERF_ATTACH_GROUP;
1456 
1457         /*
1458          * If this is a sibling, remove it from its group.
1459          */
1460         if (event->group_leader != event) {
1461                 list_del_init(&event->group_entry);
1462                 event->group_leader->nr_siblings--;
1463                 goto out;
1464         }
1465 
1466         if (!list_empty(&event->group_entry))
1467                 list = &event->group_entry;
1468 
1469         /*
1470          * If this was a group event with sibling events then
1471          * upgrade the siblings to singleton events by adding them
1472          * to whatever list we are on.
1473          */
1474         list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
1475                 if (list)
1476                         list_move_tail(&sibling->group_entry, list);
1477                 sibling->group_leader = sibling;
1478 
1479                 /* Inherit group flags from the previous leader */
1480                 sibling->group_flags = event->group_flags;
1481 
1482                 WARN_ON_ONCE(sibling->ctx != event->ctx);
1483         }
1484 
1485 out:
1486         perf_event__header_size(event->group_leader);
1487 
1488         list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
1489                 perf_event__header_size(tmp);
1490 }
1491 
1492 /*
1493  * User event without the task.
1494  */
1495 static bool is_orphaned_event(struct perf_event *event)
1496 {
1497         return event && !is_kernel_event(event) && !event->owner;
1498 }
1499 
1500 /*
1501  * Event has a parent but parent's task finished and it's
1502  * alive only because of children holding refference.
1503  */
1504 static bool is_orphaned_child(struct perf_event *event)
1505 {
1506         return is_orphaned_event(event->parent);
1507 }
1508 
1509 static void orphans_remove_work(struct work_struct *work);
1510 
1511 static void schedule_orphans_remove(struct perf_event_context *ctx)
1512 {
1513         if (!ctx->task || ctx->orphans_remove_sched || !perf_wq)
1514                 return;
1515 
1516         if (queue_delayed_work(perf_wq, &ctx->orphans_remove, 1)) {
1517                 get_ctx(ctx);
1518                 ctx->orphans_remove_sched = true;
1519         }
1520 }
1521 
1522 static int __init perf_workqueue_init(void)
1523 {
1524         perf_wq = create_singlethread_workqueue("perf");
1525         WARN(!perf_wq, "failed to create perf workqueue\n");
1526         return perf_wq ? 0 : -1;
1527 }
1528 
1529 core_initcall(perf_workqueue_init);
1530 
1531 static inline int pmu_filter_match(struct perf_event *event)
1532 {
1533         struct pmu *pmu = event->pmu;
1534         return pmu->filter_match ? pmu->filter_match(event) : 1;
1535 }
1536 
1537 static inline int
1538 event_filter_match(struct perf_event *event)
1539 {
1540         return (event->cpu == -1 || event->cpu == smp_processor_id())
1541             && perf_cgroup_match(event) && pmu_filter_match(event);
1542 }
1543 
1544 static void
1545 event_sched_out(struct perf_event *event,
1546                   struct perf_cpu_context *cpuctx,
1547                   struct perf_event_context *ctx)
1548 {
1549         u64 tstamp = perf_event_time(event);
1550         u64 delta;
1551 
1552         WARN_ON_ONCE(event->ctx != ctx);
1553         lockdep_assert_held(&ctx->lock);
1554 
1555         /*
1556          * An event which could not be activated because of
1557          * filter mismatch still needs to have its timings
1558          * maintained, otherwise bogus information is return
1559          * via read() for time_enabled, time_running:
1560          */
1561         if (event->state == PERF_EVENT_STATE_INACTIVE
1562             && !event_filter_match(event)) {
1563                 delta = tstamp - event->tstamp_stopped;
1564                 event->tstamp_running += delta;
1565                 event->tstamp_stopped = tstamp;
1566         }
1567 
1568         if (event->state != PERF_EVENT_STATE_ACTIVE)
1569                 return;
1570 
1571         perf_pmu_disable(event->pmu);
1572 
1573         event->state = PERF_EVENT_STATE_INACTIVE;
1574         if (event->pending_disable) {
1575                 event->pending_disable = 0;
1576                 event->state = PERF_EVENT_STATE_OFF;
1577         }
1578         event->tstamp_stopped = tstamp;
1579         event->pmu->del(event, 0);
1580         event->oncpu = -1;
1581 
1582         if (!is_software_event(event))
1583                 cpuctx->active_oncpu--;
1584         if (!--ctx->nr_active)
1585                 perf_event_ctx_deactivate(ctx);
1586         if (event->attr.freq && event->attr.sample_freq)
1587                 ctx->nr_freq--;
1588         if (event->attr.exclusive || !cpuctx->active_oncpu)
1589                 cpuctx->exclusive = 0;
1590 
1591         if (is_orphaned_child(event))
1592                 schedule_orphans_remove(ctx);
1593 
1594         perf_pmu_enable(event->pmu);
1595 }
1596 
1597 static void
1598 group_sched_out(struct perf_event *group_event,
1599                 struct perf_cpu_context *cpuctx,
1600                 struct perf_event_context *ctx)
1601 {
1602         struct perf_event *event;
1603         int state = group_event->state;
1604 
1605         event_sched_out(group_event, cpuctx, ctx);
1606 
1607         /*
1608          * Schedule out siblings (if any):
1609          */
1610         list_for_each_entry(event, &group_event->sibling_list, group_entry)
1611                 event_sched_out(event, cpuctx, ctx);
1612 
1613         if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
1614                 cpuctx->exclusive = 0;
1615 }
1616 
1617 struct remove_event {
1618         struct perf_event *event;
1619         bool detach_group;
1620 };
1621 
1622 /*
1623  * Cross CPU call to remove a performance event
1624  *
1625  * We disable the event on the hardware level first. After that we
1626  * remove it from the context list.
1627  */
1628 static int __perf_remove_from_context(void *info)
1629 {
1630         struct remove_event *re = info;
1631         struct perf_event *event = re->event;
1632         struct perf_event_context *ctx = event->ctx;
1633         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1634 
1635         raw_spin_lock(&ctx->lock);
1636         event_sched_out(event, cpuctx, ctx);
1637         if (re->detach_group)
1638                 perf_group_detach(event);
1639         list_del_event(event, ctx);
1640         if (!ctx->nr_events && cpuctx->task_ctx == ctx) {
1641                 ctx->is_active = 0;
1642                 cpuctx->task_ctx = NULL;
1643         }
1644         raw_spin_unlock(&ctx->lock);
1645 
1646         return 0;
1647 }
1648 
1649 
1650 /*
1651  * Remove the event from a task's (or a CPU's) list of events.
1652  *
1653  * CPU events are removed with a smp call. For task events we only
1654  * call when the task is on a CPU.
1655  *
1656  * If event->ctx is a cloned context, callers must make sure that
1657  * every task struct that event->ctx->task could possibly point to
1658  * remains valid.  This is OK when called from perf_release since
1659  * that only calls us on the top-level context, which can't be a clone.
1660  * When called from perf_event_exit_task, it's OK because the
1661  * context has been detached from its task.
1662  */
1663 static void perf_remove_from_context(struct perf_event *event, bool detach_group)
1664 {
1665         struct perf_event_context *ctx = event->ctx;
1666         struct task_struct *task = ctx->task;
1667         struct remove_event re = {
1668                 .event = event,
1669                 .detach_group = detach_group,
1670         };
1671 
1672         lockdep_assert_held(&ctx->mutex);
1673 
1674         if (!task) {
1675                 /*
1676                  * Per cpu events are removed via an smp call. The removal can
1677                  * fail if the CPU is currently offline, but in that case we
1678                  * already called __perf_remove_from_context from
1679                  * perf_event_exit_cpu.
1680                  */
1681                 cpu_function_call(event->cpu, __perf_remove_from_context, &re);
1682                 return;
1683         }
1684 
1685 retry:
1686         if (!task_function_call(task, __perf_remove_from_context, &re))
1687                 return;
1688 
1689         raw_spin_lock_irq(&ctx->lock);
1690         /*
1691          * If we failed to find a running task, but find the context active now
1692          * that we've acquired the ctx->lock, retry.
1693          */
1694         if (ctx->is_active) {
1695                 raw_spin_unlock_irq(&ctx->lock);
1696                 /*
1697                  * Reload the task pointer, it might have been changed by
1698                  * a concurrent perf_event_context_sched_out().
1699                  */
1700                 task = ctx->task;
1701                 goto retry;
1702         }
1703 
1704         /*
1705          * Since the task isn't running, its safe to remove the event, us
1706          * holding the ctx->lock ensures the task won't get scheduled in.
1707          */
1708         if (detach_group)
1709                 perf_group_detach(event);
1710         list_del_event(event, ctx);
1711         raw_spin_unlock_irq(&ctx->lock);
1712 }
1713 
1714 /*
1715  * Cross CPU call to disable a performance event
1716  */
1717 int __perf_event_disable(void *info)
1718 {
1719         struct perf_event *event = info;
1720         struct perf_event_context *ctx = event->ctx;
1721         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1722 
1723         /*
1724          * If this is a per-task event, need to check whether this
1725          * event's task is the current task on this cpu.
1726          *
1727          * Can trigger due to concurrent perf_event_context_sched_out()
1728          * flipping contexts around.
1729          */
1730         if (ctx->task && cpuctx->task_ctx != ctx)
1731                 return -EINVAL;
1732 
1733         raw_spin_lock(&ctx->lock);
1734 
1735         /*
1736          * If the event is on, turn it off.
1737          * If it is in error state, leave it in error state.
1738          */
1739         if (event->state >= PERF_EVENT_STATE_INACTIVE) {
1740                 update_context_time(ctx);
1741                 update_cgrp_time_from_event(event);
1742                 update_group_times(event);
1743                 if (event == event->group_leader)
1744                         group_sched_out(event, cpuctx, ctx);
1745                 else
1746                         event_sched_out(event, cpuctx, ctx);
1747                 event->state = PERF_EVENT_STATE_OFF;
1748         }
1749 
1750         raw_spin_unlock(&ctx->lock);
1751 
1752         return 0;
1753 }
1754 
1755 /*
1756  * Disable a event.
1757  *
1758  * If event->ctx is a cloned context, callers must make sure that
1759  * every task struct that event->ctx->task could possibly point to
1760  * remains valid.  This condition is satisifed when called through
1761  * perf_event_for_each_child or perf_event_for_each because they
1762  * hold the top-level event's child_mutex, so any descendant that
1763  * goes to exit will block in sync_child_event.
1764  * When called from perf_pending_event it's OK because event->ctx
1765  * is the current context on this CPU and preemption is disabled,
1766  * hence we can't get into perf_event_task_sched_out for this context.
1767  */
1768 static void _perf_event_disable(struct perf_event *event)
1769 {
1770         struct perf_event_context *ctx = event->ctx;
1771         struct task_struct *task = ctx->task;
1772 
1773         if (!task) {
1774                 /*
1775                  * Disable the event on the cpu that it's on
1776                  */
1777                 cpu_function_call(event->cpu, __perf_event_disable, event);
1778                 return;
1779         }
1780 
1781 retry:
1782         if (!task_function_call(task, __perf_event_disable, event))
1783                 return;
1784 
1785         raw_spin_lock_irq(&ctx->lock);
1786         /*
1787          * If the event is still active, we need to retry the cross-call.
1788          */
1789         if (event->state == PERF_EVENT_STATE_ACTIVE) {
1790                 raw_spin_unlock_irq(&ctx->lock);
1791                 /*
1792                  * Reload the task pointer, it might have been changed by
1793                  * a concurrent perf_event_context_sched_out().
1794                  */
1795                 task = ctx->task;
1796                 goto retry;
1797         }
1798 
1799         /*
1800          * Since we have the lock this context can't be scheduled
1801          * in, so we can change the state safely.
1802          */
1803         if (event->state == PERF_EVENT_STATE_INACTIVE) {
1804                 update_group_times(event);
1805                 event->state = PERF_EVENT_STATE_OFF;
1806         }
1807         raw_spin_unlock_irq(&ctx->lock);
1808 }
1809 
1810 /*
1811  * Strictly speaking kernel users cannot create groups and therefore this
1812  * interface does not need the perf_event_ctx_lock() magic.
1813  */
1814 void perf_event_disable(struct perf_event *event)
1815 {
1816         struct perf_event_context *ctx;
1817 
1818         ctx = perf_event_ctx_lock(event);
1819         _perf_event_disable(event);
1820         perf_event_ctx_unlock(event, ctx);
1821 }
1822 EXPORT_SYMBOL_GPL(perf_event_disable);
1823 
1824 static void perf_set_shadow_time(struct perf_event *event,
1825                                  struct perf_event_context *ctx,
1826                                  u64 tstamp)
1827 {
1828         /*
1829          * use the correct time source for the time snapshot
1830          *
1831          * We could get by without this by leveraging the
1832          * fact that to get to this function, the caller
1833          * has most likely already called update_context_time()
1834          * and update_cgrp_time_xx() and thus both timestamp
1835          * are identical (or very close). Given that tstamp is,
1836          * already adjusted for cgroup, we could say that:
1837          *    tstamp - ctx->timestamp
1838          * is equivalent to
1839          *    tstamp - cgrp->timestamp.
1840          *
1841          * Then, in perf_output_read(), the calculation would
1842          * work with no changes because:
1843          * - event is guaranteed scheduled in
1844          * - no scheduled out in between
1845          * - thus the timestamp would be the same
1846          *
1847          * But this is a bit hairy.
1848          *
1849          * So instead, we have an explicit cgroup call to remain
1850          * within the time time source all along. We believe it
1851          * is cleaner and simpler to understand.
1852          */
1853         if (is_cgroup_event(event))
1854                 perf_cgroup_set_shadow_time(event, tstamp);
1855         else
1856                 event->shadow_ctx_time = tstamp - ctx->timestamp;
1857 }
1858 
1859 #define MAX_INTERRUPTS (~0ULL)
1860 
1861 static void perf_log_throttle(struct perf_event *event, int enable);
1862 static void perf_log_itrace_start(struct perf_event *event);
1863 
1864 static int
1865 event_sched_in(struct perf_event *event,
1866                  struct perf_cpu_context *cpuctx,
1867                  struct perf_event_context *ctx)
1868 {
1869         u64 tstamp = perf_event_time(event);
1870         int ret = 0;
1871 
1872         lockdep_assert_held(&ctx->lock);
1873 
1874         if (event->state <= PERF_EVENT_STATE_OFF)
1875                 return 0;
1876 
1877         event->state = PERF_EVENT_STATE_ACTIVE;
1878         event->oncpu = smp_processor_id();
1879 
1880         /*
1881          * Unthrottle events, since we scheduled we might have missed several
1882          * ticks already, also for a heavily scheduling task there is little
1883          * guarantee it'll get a tick in a timely manner.
1884          */
1885         if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
1886                 perf_log_throttle(event, 1);
1887                 event->hw.interrupts = 0;
1888         }
1889 
1890         /*
1891          * The new state must be visible before we turn it on in the hardware:
1892          */
1893         smp_wmb();
1894 
1895         perf_pmu_disable(event->pmu);
1896 
1897         perf_set_shadow_time(event, ctx, tstamp);
1898 
1899         perf_log_itrace_start(event);
1900 
1901         if (event->pmu->add(event, PERF_EF_START)) {
1902                 event->state = PERF_EVENT_STATE_INACTIVE;
1903                 event->oncpu = -1;
1904                 ret = -EAGAIN;
1905                 goto out;
1906         }
1907 
1908         event->tstamp_running += tstamp - event->tstamp_stopped;
1909 
1910         if (!is_software_event(event))
1911                 cpuctx->active_oncpu++;
1912         if (!ctx->nr_active++)
1913                 perf_event_ctx_activate(ctx);
1914         if (event->attr.freq && event->attr.sample_freq)
1915                 ctx->nr_freq++;
1916 
1917         if (event->attr.exclusive)
1918                 cpuctx->exclusive = 1;
1919 
1920         if (is_orphaned_child(event))
1921                 schedule_orphans_remove(ctx);
1922 
1923 out:
1924         perf_pmu_enable(event->pmu);
1925 
1926         return ret;
1927 }
1928 
1929 static int
1930 group_sched_in(struct perf_event *group_event,
1931                struct perf_cpu_context *cpuctx,
1932                struct perf_event_context *ctx)
1933 {
1934         struct perf_event *event, *partial_group = NULL;
1935         struct pmu *pmu = ctx->pmu;
1936         u64 now = ctx->time;
1937         bool simulate = false;
1938 
1939         if (group_event->state == PERF_EVENT_STATE_OFF)
1940                 return 0;
1941 
1942         pmu->start_txn(pmu);
1943 
1944         if (event_sched_in(group_event, cpuctx, ctx)) {
1945                 pmu->cancel_txn(pmu);
1946                 perf_mux_hrtimer_restart(cpuctx);
1947                 return -EAGAIN;
1948         }
1949 
1950         /*
1951          * Schedule in siblings as one group (if any):
1952          */
1953         list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1954                 if (event_sched_in(event, cpuctx, ctx)) {
1955                         partial_group = event;
1956                         goto group_error;
1957                 }
1958         }
1959 
1960         if (!pmu->commit_txn(pmu))
1961                 return 0;
1962 
1963 group_error:
1964         /*
1965          * Groups can be scheduled in as one unit only, so undo any
1966          * partial group before returning:
1967          * The events up to the failed event are scheduled out normally,
1968          * tstamp_stopped will be updated.
1969          *
1970          * The failed events and the remaining siblings need to have
1971          * their timings updated as if they had gone thru event_sched_in()
1972          * and event_sched_out(). This is required to get consistent timings
1973          * across the group. This also takes care of the case where the group
1974          * could never be scheduled by ensuring tstamp_stopped is set to mark
1975          * the time the event was actually stopped, such that time delta
1976          * calculation in update_event_times() is correct.
1977          */
1978         list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1979                 if (event == partial_group)
1980                         simulate = true;
1981 
1982                 if (simulate) {
1983                         event->tstamp_running += now - event->tstamp_stopped;
1984                         event->tstamp_stopped = now;
1985                 } else {
1986                         event_sched_out(event, cpuctx, ctx);
1987                 }
1988         }
1989         event_sched_out(group_event, cpuctx, ctx);
1990 
1991         pmu->cancel_txn(pmu);
1992 
1993         perf_mux_hrtimer_restart(cpuctx);
1994 
1995         return -EAGAIN;
1996 }
1997 
1998 /*
1999  * Work out whether we can put this event group on the CPU now.
2000  */
2001 static int group_can_go_on(struct perf_event *event,
2002                            struct perf_cpu_context *cpuctx,
2003                            int can_add_hw)
2004 {
2005         /*
2006          * Groups consisting entirely of software events can always go on.
2007          */
2008         if (event->group_flags & PERF_GROUP_SOFTWARE)
2009                 return 1;
2010         /*
2011          * If an exclusive group is already on, no other hardware
2012          * events can go on.
2013          */
2014         if (cpuctx->exclusive)
2015                 return 0;
2016         /*
2017          * If this group is exclusive and there are already
2018          * events on the CPU, it can't go on.
2019          */
2020         if (event->attr.exclusive && cpuctx->active_oncpu)
2021                 return 0;
2022         /*
2023          * Otherwise, try to add it if all previous groups were able
2024          * to go on.
2025          */
2026         return can_add_hw;
2027 }
2028 
2029 static void add_event_to_ctx(struct perf_event *event,
2030                                struct perf_event_context *ctx)
2031 {
2032         u64 tstamp = perf_event_time(event);
2033 
2034         list_add_event(event, ctx);
2035         perf_group_attach(event);
2036         event->tstamp_enabled = tstamp;
2037         event->tstamp_running = tstamp;
2038         event->tstamp_stopped = tstamp;
2039 }
2040 
2041 static void task_ctx_sched_out(struct perf_event_context *ctx);
2042 static void
2043 ctx_sched_in(struct perf_event_context *ctx,
2044              struct perf_cpu_context *cpuctx,
2045              enum event_type_t event_type,
2046              struct task_struct *task);
2047 
2048 static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
2049                                 struct perf_event_context *ctx,
2050                                 struct task_struct *task)
2051 {
2052         cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
2053         if (ctx)
2054                 ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
2055         cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
2056         if (ctx)
2057                 ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
2058 }
2059 
2060 /*
2061  * Cross CPU call to install and enable a performance event
2062  *
2063  * Must be called with ctx->mutex held
2064  */
2065 static int  __perf_install_in_context(void *info)
2066 {
2067         struct perf_event *event = info;
2068         struct perf_event_context *ctx = event->ctx;
2069         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2070         struct perf_event_context *task_ctx = cpuctx->task_ctx;
2071         struct task_struct *task = current;
2072 
2073         perf_ctx_lock(cpuctx, task_ctx);
2074         perf_pmu_disable(cpuctx->ctx.pmu);
2075 
2076         /*
2077          * If there was an active task_ctx schedule it out.
2078          */
2079         if (task_ctx)
2080                 task_ctx_sched_out(task_ctx);
2081 
2082         /*
2083          * If the context we're installing events in is not the
2084          * active task_ctx, flip them.
2085          */
2086         if (ctx->task && task_ctx != ctx) {
2087                 if (task_ctx)
2088                         raw_spin_unlock(&task_ctx->lock);
2089                 raw_spin_lock(&ctx->lock);
2090                 task_ctx = ctx;
2091         }
2092 
2093         if (task_ctx) {
2094                 cpuctx->task_ctx = task_ctx;
2095                 task = task_ctx->task;
2096         }
2097 
2098         cpu_ctx_sched_out(cpuctx, EVENT_ALL);
2099 
2100         update_context_time(ctx);
2101         /*
2102          * update cgrp time only if current cgrp
2103          * matches event->cgrp. Must be done before
2104          * calling add_event_to_ctx()
2105          */
2106         update_cgrp_time_from_event(event);
2107 
2108         add_event_to_ctx(event, ctx);
2109 
2110         /*
2111          * Schedule everything back in
2112          */
2113         perf_event_sched_in(cpuctx, task_ctx, task);
2114 
2115         perf_pmu_enable(cpuctx->ctx.pmu);
2116         perf_ctx_unlock(cpuctx, task_ctx);
2117 
2118         return 0;
2119 }
2120 
2121 /*
2122  * Attach a performance event to a context
2123  *
2124  * First we add the event to the list with the hardware enable bit
2125  * in event->hw_config cleared.
2126  *
2127  * If the event is attached to a task which is on a CPU we use a smp
2128  * call to enable it in the task context. The task might have been
2129  * scheduled away, but we check this in the smp call again.
2130  */
2131 static void
2132 perf_install_in_context(struct perf_event_context *ctx,
2133                         struct perf_event *event,
2134                         int cpu)
2135 {
2136         struct task_struct *task = ctx->task;
2137 
2138         lockdep_assert_held(&ctx->mutex);
2139 
2140         event->ctx = ctx;
2141         if (event->cpu != -1)
2142                 event->cpu = cpu;
2143 
2144         if (!task) {
2145                 /*
2146                  * Per cpu events are installed via an smp call and
2147                  * the install is always successful.
2148                  */
2149                 cpu_function_call(cpu, __perf_install_in_context, event);
2150                 return;
2151         }
2152 
2153 retry:
2154         if (!task_function_call(task, __perf_install_in_context, event))
2155                 return;
2156 
2157         raw_spin_lock_irq(&ctx->lock);
2158         /*
2159          * If we failed to find a running task, but find the context active now
2160          * that we've acquired the ctx->lock, retry.
2161          */
2162         if (ctx->is_active) {
2163                 raw_spin_unlock_irq(&ctx->lock);
2164                 /*
2165                  * Reload the task pointer, it might have been changed by
2166                  * a concurrent perf_event_context_sched_out().
2167                  */
2168                 task = ctx->task;
2169                 goto retry;
2170         }
2171 
2172         /*
2173          * Since the task isn't running, its safe to add the event, us holding
2174          * the ctx->lock ensures the task won't get scheduled in.
2175          */
2176         add_event_to_ctx(event, ctx);
2177         raw_spin_unlock_irq(&ctx->lock);
2178 }
2179 
2180 /*
2181  * Put a event into inactive state and update time fields.
2182  * Enabling the leader of a group effectively enables all
2183  * the group members that aren't explicitly disabled, so we
2184  * have to update their ->tstamp_enabled also.
2185  * Note: this works for group members as well as group leaders
2186  * since the non-leader members' sibling_lists will be empty.
2187  */
2188 static void __perf_event_mark_enabled(struct perf_event *event)
2189 {
2190         struct perf_event *sub;
2191         u64 tstamp = perf_event_time(event);
2192 
2193         event->state = PERF_EVENT_STATE_INACTIVE;
2194         event->tstamp_enabled = tstamp - event->total_time_enabled;
2195         list_for_each_entry(sub, &event->sibling_list, group_entry) {
2196                 if (sub->state >= PERF_EVENT_STATE_INACTIVE)
2197                         sub->tstamp_enabled = tstamp - sub->total_time_enabled;
2198         }
2199 }
2200 
2201 /*
2202  * Cross CPU call to enable a performance event
2203  */
2204 static int __perf_event_enable(void *info)
2205 {
2206         struct perf_event *event = info;
2207         struct perf_event_context *ctx = event->ctx;
2208         struct perf_event *leader = event->group_leader;
2209         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2210         int err;
2211 
2212         /*
2213          * There's a time window between 'ctx->is_active' check
2214          * in perf_event_enable function and this place having:
2215          *   - IRQs on
2216          *   - ctx->lock unlocked
2217          *
2218          * where the task could be killed and 'ctx' deactivated
2219          * by perf_event_exit_task.
2220          */
2221         if (!ctx->is_active)
2222                 return -EINVAL;
2223 
2224         raw_spin_lock(&ctx->lock);
2225         update_context_time(ctx);
2226 
2227         if (event->state >= PERF_EVENT_STATE_INACTIVE)
2228                 goto unlock;
2229 
2230         /*
2231          * set current task's cgroup time reference point
2232          */
2233         perf_cgroup_set_timestamp(current, ctx);
2234 
2235         __perf_event_mark_enabled(event);
2236 
2237         if (!event_filter_match(event)) {
2238                 if (is_cgroup_event(event))
2239                         perf_cgroup_defer_enabled(event);
2240                 goto unlock;
2241         }
2242 
2243         /*
2244          * If the event is in a group and isn't the group leader,
2245          * then don't put it on unless the group is on.
2246          */
2247         if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
2248                 goto unlock;
2249 
2250         if (!group_can_go_on(event, cpuctx, 1)) {
2251                 err = -EEXIST;
2252         } else {
2253                 if (event == leader)
2254                         err = group_sched_in(event, cpuctx, ctx);
2255                 else
2256                         err = event_sched_in(event, cpuctx, ctx);
2257         }
2258 
2259         if (err) {
2260                 /*
2261                  * If this event can't go on and it's part of a
2262                  * group, then the whole group has to come off.
2263                  */
2264                 if (leader != event) {
2265                         group_sched_out(leader, cpuctx, ctx);
2266                         perf_mux_hrtimer_restart(cpuctx);
2267                 }
2268                 if (leader->attr.pinned) {
2269                         update_group_times(leader);
2270                         leader->state = PERF_EVENT_STATE_ERROR;
2271                 }
2272         }
2273 
2274 unlock:
2275         raw_spin_unlock(&ctx->lock);
2276 
2277         return 0;
2278 }
2279 
2280 /*
2281  * Enable a event.
2282  *
2283  * If event->ctx is a cloned context, callers must make sure that
2284  * every task struct that event->ctx->task could possibly point to
2285  * remains valid.  This condition is satisfied when called through
2286  * perf_event_for_each_child or perf_event_for_each as described
2287  * for perf_event_disable.
2288  */
2289 static void _perf_event_enable(struct perf_event *event)
2290 {
2291         struct perf_event_context *ctx = event->ctx;
2292         struct task_struct *task = ctx->task;
2293 
2294         if (!task) {
2295                 /*
2296                  * Enable the event on the cpu that it's on
2297                  */
2298                 cpu_function_call(event->cpu, __perf_event_enable, event);
2299                 return;
2300         }
2301 
2302         raw_spin_lock_irq(&ctx->lock);
2303         if (event->state >= PERF_EVENT_STATE_INACTIVE)
2304                 goto out;
2305 
2306         /*
2307          * If the event is in error state, clear that first.
2308          * That way, if we see the event in error state below, we
2309          * know that it has gone back into error state, as distinct
2310          * from the task having been scheduled away before the
2311          * cross-call arrived.
2312          */
2313         if (event->state == PERF_EVENT_STATE_ERROR)
2314                 event->state = PERF_EVENT_STATE_OFF;
2315 
2316 retry:
2317         if (!ctx->is_active) {
2318                 __perf_event_mark_enabled(event);
2319                 goto out;
2320         }
2321 
2322         raw_spin_unlock_irq(&ctx->lock);
2323 
2324         if (!task_function_call(task, __perf_event_enable, event))
2325                 return;
2326 
2327         raw_spin_lock_irq(&ctx->lock);
2328 
2329         /*
2330          * If the context is active and the event is still off,
2331          * we need to retry the cross-call.
2332          */
2333         if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
2334                 /*
2335                  * task could have been flipped by a concurrent
2336                  * perf_event_context_sched_out()
2337                  */
2338                 task = ctx->task;
2339                 goto retry;
2340         }
2341 
2342 out:
2343         raw_spin_unlock_irq(&ctx->lock);
2344 }
2345 
2346 /*
2347  * See perf_event_disable();
2348  */
2349 void perf_event_enable(struct perf_event *event)
2350 {
2351         struct perf_event_context *ctx;
2352 
2353         ctx = perf_event_ctx_lock(event);
2354         _perf_event_enable(event);
2355         perf_event_ctx_unlock(event, ctx);
2356 }
2357 EXPORT_SYMBOL_GPL(perf_event_enable);
2358 
2359 static int _perf_event_refresh(struct perf_event *event, int refresh)
2360 {
2361         /*
2362          * not supported on inherited events
2363          */
2364         if (event->attr.inherit || !is_sampling_event(event))
2365                 return -EINVAL;
2366 
2367         atomic_add(refresh, &event->event_limit);
2368         _perf_event_enable(event);
2369 
2370         return 0;
2371 }
2372 
2373 /*
2374  * See perf_event_disable()
2375  */
2376 int perf_event_refresh(struct perf_event *event, int refresh)
2377 {
2378         struct perf_event_context *ctx;
2379         int ret;
2380 
2381         ctx = perf_event_ctx_lock(event);
2382         ret = _perf_event_refresh(event, refresh);
2383         perf_event_ctx_unlock(event, ctx);
2384 
2385         return ret;
2386 }
2387 EXPORT_SYMBOL_GPL(perf_event_refresh);
2388 
2389 static void ctx_sched_out(struct perf_event_context *ctx,
2390                           struct perf_cpu_context *cpuctx,
2391                           enum event_type_t event_type)
2392 {
2393         struct perf_event *event;
2394         int is_active = ctx->is_active;
2395 
2396         ctx->is_active &= ~event_type;
2397         if (likely(!ctx->nr_events))
2398                 return;
2399 
2400         update_context_time(ctx);
2401         update_cgrp_time_from_cpuctx(cpuctx);
2402         if (!ctx->nr_active)
2403                 return;
2404 
2405         perf_pmu_disable(ctx->pmu);
2406         if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {
2407                 list_for_each_entry(event, &ctx->pinned_groups, group_entry)
2408                         group_sched_out(event, cpuctx, ctx);
2409         }
2410 
2411         if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {
2412                 list_for_each_entry(event, &ctx->flexible_groups, group_entry)
2413                         group_sched_out(event, cpuctx, ctx);
2414         }
2415         perf_pmu_enable(ctx->pmu);
2416 }
2417 
2418 /*
2419  * Test whether two contexts are equivalent, i.e. whether they have both been
2420  * cloned from the same version of the same context.
2421  *
2422  * Equivalence is measured using a generation number in the context that is
2423  * incremented on each modification to it; see unclone_ctx(), list_add_event()
2424  * and list_del_event().
2425  */
2426 static int context_equiv(struct perf_event_context *ctx1,
2427                          struct perf_event_context *ctx2)
2428 {
2429         lockdep_assert_held(&ctx1->lock);
2430         lockdep_assert_held(&ctx2->lock);
2431 
2432         /* Pinning disables the swap optimization */
2433         if (ctx1->pin_count || ctx2->pin_count)
2434                 return 0;
2435 
2436         /* If ctx1 is the parent of ctx2 */
2437         if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)
2438                 return 1;
2439 
2440         /* If ctx2 is the parent of ctx1 */
2441         if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)
2442                 return 1;
2443 
2444         /*
2445          * If ctx1 and ctx2 have the same parent; we flatten the parent
2446          * hierarchy, see perf_event_init_context().
2447          */
2448         if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&
2449                         ctx1->parent_gen == ctx2->parent_gen)
2450                 return 1;
2451 
2452         /* Unmatched */
2453         return 0;
2454 }
2455 
2456 static void __perf_event_sync_stat(struct perf_event *event,
2457                                      struct perf_event *next_event)
2458 {
2459         u64 value;
2460 
2461         if (!event->attr.inherit_stat)
2462                 return;
2463 
2464         /*
2465          * Update the event value, we cannot use perf_event_read()
2466          * because we're in the middle of a context switch and have IRQs
2467          * disabled, which upsets smp_call_function_single(), however
2468          * we know the event must be on the current CPU, therefore we
2469          * don't need to use it.
2470          */
2471         switch (event->state) {
2472         case PERF_EVENT_STATE_ACTIVE:
2473                 event->pmu->read(event);
2474                 /* fall-through */
2475 
2476         case PERF_EVENT_STATE_INACTIVE:
2477                 update_event_times(event);
2478                 break;
2479 
2480         default:
2481                 break;
2482         }
2483 
2484         /*
2485          * In order to keep per-task stats reliable we need to flip the event
2486          * values when we flip the contexts.
2487          */
2488         value = local64_read(&next_event->count);
2489         value = local64_xchg(&event->count, value);
2490         local64_set(&next_event->count, value);
2491 
2492         swap(event->total_time_enabled, next_event->total_time_enabled);
2493         swap(event->total_time_running, next_event->total_time_running);
2494 
2495         /*
2496          * Since we swizzled the values, update the user visible data too.
2497          */
2498         perf_event_update_userpage(event);
2499         perf_event_update_userpage(next_event);
2500 }
2501 
2502 static void perf_event_sync_stat(struct perf_event_context *ctx,
2503                                    struct perf_event_context *next_ctx)
2504 {
2505         struct perf_event *event, *next_event;
2506 
2507         if (!ctx->nr_stat)
2508                 return;
2509 
2510         update_context_time(ctx);
2511 
2512         event = list_first_entry(&ctx->event_list,
2513                                    struct perf_event, event_entry);
2514 
2515         next_event = list_first_entry(&next_ctx->event_list,
2516                                         struct perf_event, event_entry);
2517 
2518         while (&event->event_entry != &ctx->event_list &&
2519                &next_event->event_entry != &next_ctx->event_list) {
2520 
2521                 __perf_event_sync_stat(event, next_event);
2522 
2523                 event = list_next_entry(event, event_entry);
2524                 next_event = list_next_entry(next_event, event_entry);
2525         }
2526 }
2527 
2528 static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
2529                                          struct task_struct *next)
2530 {
2531         struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
2532         struct perf_event_context *next_ctx;
2533         struct perf_event_context *parent, *next_parent;
2534         struct perf_cpu_context *cpuctx;
2535         int do_switch = 1;
2536 
2537         if (likely(!ctx))
2538                 return;
2539 
2540         cpuctx = __get_cpu_context(ctx);
2541         if (!cpuctx->task_ctx)
2542                 return;
2543 
2544         rcu_read_lock();
2545         next_ctx = next->perf_event_ctxp[ctxn];
2546         if (!next_ctx)
2547                 goto unlock;
2548 
2549         parent = rcu_dereference(ctx->parent_ctx);
2550         next_parent = rcu_dereference(next_ctx->parent_ctx);
2551 
2552         /* If neither context have a parent context; they cannot be clones. */
2553         if (!parent && !next_parent)
2554                 goto unlock;
2555 
2556         if (next_parent == ctx || next_ctx == parent || next_parent == parent) {
2557                 /*
2558                  * Looks like the two contexts are clones, so we might be
2559                  * able to optimize the context switch.  We lock both
2560                  * contexts and check that they are clones under the
2561                  * lock (including re-checking that neither has been
2562                  * uncloned in the meantime).  It doesn't matter which
2563                  * order we take the locks because no other cpu could
2564                  * be trying to lock both of these tasks.
2565                  */
2566                 raw_spin_lock(&ctx->lock);
2567                 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
2568                 if (context_equiv(ctx, next_ctx)) {
2569                         /*
2570                          * XXX do we need a memory barrier of sorts
2571                          * wrt to rcu_dereference() of perf_event_ctxp
2572                          */
2573                         task->perf_event_ctxp[ctxn] = next_ctx;
2574                         next->perf_event_ctxp[ctxn] = ctx;
2575                         ctx->task = next;
2576                         next_ctx->task = task;
2577 
2578                         swap(ctx->task_ctx_data, next_ctx->task_ctx_data);
2579 
2580                         do_switch = 0;
2581 
2582                         perf_event_sync_stat(ctx, next_ctx);
2583                 }
2584                 raw_spin_unlock(&next_ctx->lock);
2585                 raw_spin_unlock(&ctx->lock);
2586         }
2587 unlock:
2588         rcu_read_unlock();
2589 
2590         if (do_switch) {
2591                 raw_spin_lock(&ctx->lock);
2592                 ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2593                 cpuctx->task_ctx = NULL;
2594                 raw_spin_unlock(&ctx->lock);
2595         }
2596 }
2597 
2598 void perf_sched_cb_dec(struct pmu *pmu)
2599 {
2600         this_cpu_dec(perf_sched_cb_usages);
2601 }
2602 
2603 void perf_sched_cb_inc(struct pmu *pmu)
2604 {
2605         this_cpu_inc(perf_sched_cb_usages);
2606 }
2607 
2608 /*
2609  * This function provides the context switch callback to the lower code
2610  * layer. It is invoked ONLY when the context switch callback is enabled.
2611  */
2612 static void perf_pmu_sched_task(struct task_struct *prev,
2613                                 struct task_struct *next,
2614                                 bool sched_in)
2615 {
2616         struct perf_cpu_context *cpuctx;
2617         struct pmu *pmu;
2618         unsigned long flags;
2619 
2620         if (prev == next)
2621                 return;
2622 
2623         local_irq_save(flags);
2624 
2625         rcu_read_lock();
2626 
2627         list_for_each_entry_rcu(pmu, &pmus, entry) {
2628                 if (pmu->sched_task) {
2629                         cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
2630 
2631                         perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2632 
2633                         perf_pmu_disable(pmu);
2634 
2635                         pmu->sched_task(cpuctx->task_ctx, sched_in);
2636 
2637                         perf_pmu_enable(pmu);
2638 
2639                         perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2640                 }
2641         }
2642 
2643         rcu_read_unlock();
2644 
2645         local_irq_restore(flags);
2646 }
2647 
2648 static void perf_event_switch(struct task_struct *task,
2649                               struct task_struct *next_prev, bool sched_in);
2650 
2651 #define for_each_task_context_nr(ctxn)                                  \
2652         for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
2653 
2654 /*
2655  * Called from scheduler to remove the events of the current task,
2656  * with interrupts disabled.
2657  *
2658  * We stop each event and update the event value in event->count.
2659  *
2660  * This does not protect us against NMI, but disable()
2661  * sets the disabled bit in the control field of event _before_
2662  * accessing the event control register. If a NMI hits, then it will
2663  * not restart the event.
2664  */
2665 void __perf_event_task_sched_out(struct task_struct *task,
2666                                  struct task_struct *next)
2667 {
2668         int ctxn;
2669 
2670         if (__this_cpu_read(perf_sched_cb_usages))
2671                 perf_pmu_sched_task(task, next, false);
2672 
2673         if (atomic_read(&nr_switch_events))
2674                 perf_event_switch(task, next, false);
2675 
2676         for_each_task_context_nr(ctxn)
2677                 perf_event_context_sched_out(task, ctxn, next);
2678 
2679         /*
2680          * if cgroup events exist on this CPU, then we need
2681          * to check if we have to switch out PMU state.
2682          * cgroup event are system-wide mode only
2683          */
2684         if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
2685                 perf_cgroup_sched_out(task, next);
2686 }
2687 
2688 static void task_ctx_sched_out(struct perf_event_context *ctx)
2689 {
2690         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2691 
2692         if (!cpuctx->task_ctx)
2693                 return;
2694 
2695         if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2696                 return;
2697 
2698         ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2699         cpuctx->task_ctx = NULL;
2700 }
2701 
2702 /*
2703  * Called with IRQs disabled
2704  */
2705 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
2706                               enum event_type_t event_type)
2707 {
2708         ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
2709 }
2710 
2711 static void
2712 ctx_pinned_sched_in(struct perf_event_context *ctx,
2713                     struct perf_cpu_context *cpuctx)
2714 {
2715         struct perf_event *event;
2716 
2717         list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
2718                 if (event->state <= PERF_EVENT_STATE_OFF)
2719                         continue;
2720                 if (!event_filter_match(event))
2721                         continue;
2722 
2723                 /* may need to reset tstamp_enabled */
2724                 if (is_cgroup_event(event))
2725                         perf_cgroup_mark_enabled(event, ctx);
2726 
2727                 if (group_can_go_on(event, cpuctx, 1))
2728                         group_sched_in(event, cpuctx, ctx);
2729 
2730                 /*
2731                  * If this pinned group hasn't been scheduled,
2732                  * put it in error state.
2733                  */
2734                 if (event->state == PERF_EVENT_STATE_INACTIVE) {
2735                         update_group_times(event);
2736                         event->state = PERF_EVENT_STATE_ERROR;
2737                 }
2738         }
2739 }
2740 
2741 static void
2742 ctx_flexible_sched_in(struct perf_event_context *ctx,
2743                       struct perf_cpu_context *cpuctx)
2744 {
2745         struct perf_event *event;
2746         int can_add_hw = 1;
2747 
2748         list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
2749                 /* Ignore events in OFF or ERROR state */
2750                 if (event->state <= PERF_EVENT_STATE_OFF)
2751                         continue;
2752                 /*
2753                  * Listen to the 'cpu' scheduling filter constraint
2754                  * of events:
2755                  */
2756                 if (!event_filter_match(event))
2757                         continue;
2758 
2759                 /* may need to reset tstamp_enabled */
2760                 if (is_cgroup_event(event))
2761                         perf_cgroup_mark_enabled(event, ctx);
2762 
2763                 if (group_can_go_on(event, cpuctx, can_add_hw)) {
2764                         if (group_sched_in(event, cpuctx, ctx))
2765                                 can_add_hw = 0;
2766                 }
2767         }
2768 }
2769 
2770 static void
2771 ctx_sched_in(struct perf_event_context *ctx,
2772              struct perf_cpu_context *cpuctx,
2773              enum event_type_t event_type,
2774              struct task_struct *task)
2775 {
2776         u64 now;
2777         int is_active = ctx->is_active;
2778 
2779         ctx->is_active |= event_type;
2780         if (likely(!ctx->nr_events))
2781                 return;
2782 
2783         now = perf_clock();
2784         ctx->timestamp = now;
2785         perf_cgroup_set_timestamp(task, ctx);
2786         /*
2787          * First go through the list and put on any pinned groups
2788          * in order to give them the best chance of going on.
2789          */
2790         if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))
2791                 ctx_pinned_sched_in(ctx, cpuctx);
2792 
2793         /* Then walk through the lower prio flexible groups */
2794         if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))
2795                 ctx_flexible_sched_in(ctx, cpuctx);
2796 }
2797 
2798 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
2799                              enum event_type_t event_type,
2800                              struct task_struct *task)
2801 {
2802         struct perf_event_context *ctx = &cpuctx->ctx;
2803 
2804         ctx_sched_in(ctx, cpuctx, event_type, task);
2805 }
2806 
2807 static void perf_event_context_sched_in(struct perf_event_context *ctx,
2808                                         struct task_struct *task)
2809 {
2810         struct perf_cpu_context *cpuctx;
2811 
2812         cpuctx = __get_cpu_context(ctx);
2813         if (cpuctx->task_ctx == ctx)
2814                 return;
2815 
2816         perf_ctx_lock(cpuctx, ctx);
2817         perf_pmu_disable(ctx->pmu);
2818         /*
2819          * We want to keep the following priority order:
2820          * cpu pinned (that don't need to move), task pinned,
2821          * cpu flexible, task flexible.
2822          */
2823         cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2824 
2825         if (ctx->nr_events)
2826                 cpuctx->task_ctx = ctx;
2827 
2828         perf_event_sched_in(cpuctx, cpuctx->task_ctx, task);
2829 
2830         perf_pmu_enable(ctx->pmu);
2831         perf_ctx_unlock(cpuctx, ctx);
2832 }
2833 
2834 /*
2835  * Called from scheduler to add the events of the current task
2836  * with interrupts disabled.
2837  *
2838  * We restore the event value and then enable it.
2839  *
2840  * This does not protect us against NMI, but enable()
2841  * sets the enabled bit in the control field of event _before_
2842  * accessing the event control register. If a NMI hits, then it will
2843  * keep the event running.
2844  */
2845 void __perf_event_task_sched_in(struct task_struct *prev,
2846                                 struct task_struct *task)
2847 {
2848         struct perf_event_context *ctx;
2849         int ctxn;
2850 
2851         for_each_task_context_nr(ctxn) {
2852                 ctx = task->perf_event_ctxp[ctxn];
2853                 if (likely(!ctx))
2854                         continue;
2855 
2856                 perf_event_context_sched_in(ctx, task);
2857         }
2858         /*
2859          * if cgroup events exist on this CPU, then we need
2860          * to check if we have to switch in PMU state.
2861          * cgroup event are system-wide mode only
2862          */
2863         if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
2864                 perf_cgroup_sched_in(prev, task);
2865 
2866         if (atomic_read(&nr_switch_events))
2867                 perf_event_switch(task, prev, true);
2868 
2869         if (__this_cpu_read(perf_sched_cb_usages))
2870                 perf_pmu_sched_task(prev, task, true);
2871 }
2872 
2873 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
2874 {
2875         u64 frequency = event->attr.sample_freq;
2876         u64 sec = NSEC_PER_SEC;
2877         u64 divisor, dividend;
2878 
2879         int count_fls, nsec_fls, frequency_fls, sec_fls;
2880 
2881         count_fls = fls64(count);
2882         nsec_fls = fls64(nsec);
2883         frequency_fls = fls64(frequency);
2884         sec_fls = 30;
2885 
2886         /*
2887          * We got @count in @nsec, with a target of sample_freq HZ
2888          * the target period becomes:
2889          *
2890          *             @count * 10^9
2891          * period = -------------------
2892          *          @nsec * sample_freq
2893          *
2894          */
2895 
2896         /*
2897          * Reduce accuracy by one bit such that @a and @b converge
2898          * to a similar magnitude.
2899          */
2900 #define REDUCE_FLS(a, b)                \
2901 do {                                    \
2902         if (a##_fls > b##_fls) {        \
2903                 a >>= 1;                \
2904                 a##_fls--;              \
2905         } else {                        \
2906                 b >>= 1;                \
2907                 b##_fls--;              \
2908         }                               \
2909 } while (0)
2910 
2911         /*
2912          * Reduce accuracy until either term fits in a u64, then proceed with
2913          * the other, so that finally we can do a u64/u64 division.
2914          */
2915         while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
2916                 REDUCE_FLS(nsec, frequency);
2917                 REDUCE_FLS(sec, count);
2918         }
2919 
2920         if (count_fls + sec_fls > 64) {
2921                 divisor = nsec * frequency;
2922 
2923                 while (count_fls + sec_fls > 64) {
2924                         REDUCE_FLS(count, sec);
2925                         divisor >>= 1;
2926                 }
2927 
2928                 dividend = count * sec;
2929         } else {
2930                 dividend = count * sec;
2931 
2932                 while (nsec_fls + frequency_fls > 64) {
2933                         REDUCE_FLS(nsec, frequency);
2934                         dividend >>= 1;
2935                 }
2936 
2937                 divisor = nsec * frequency;
2938         }
2939 
2940         if (!divisor)
2941                 return dividend;
2942 
2943         return div64_u64(dividend, divisor);
2944 }
2945 
2946 static DEFINE_PER_CPU(int, perf_throttled_count);
2947 static DEFINE_PER_CPU(u64, perf_throttled_seq);
2948 
2949 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
2950 {
2951         struct hw_perf_event *hwc = &event->hw;
2952         s64 period, sample_period;
2953         s64 delta;
2954 
2955         period = perf_calculate_period(event, nsec, count);
2956 
2957         delta = (s64)(period - hwc->sample_period);
2958         delta = (delta + 7) / 8; /* low pass filter */
2959 
2960         sample_period = hwc->sample_period + delta;
2961 
2962         if (!sample_period)
2963                 sample_period = 1;
2964 
2965         hwc->sample_period = sample_period;
2966 
2967         if (local64_read(&hwc->period_left) > 8*sample_period) {
2968                 if (disable)
2969                         event->pmu->stop(event, PERF_EF_UPDATE);
2970 
2971                 local64_set(&hwc->period_left, 0);
2972 
2973                 if (disable)
2974                         event->pmu->start(event, PERF_EF_RELOAD);
2975         }
2976 }
2977 
2978 /*
2979  * combine freq adjustment with unthrottling to avoid two passes over the
2980  * events. At the same time, make sure, having freq events does not change
2981  * the rate of unthrottling as that would introduce bias.
2982  */
2983 static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
2984                                            int needs_unthr)
2985 {
2986         struct perf_event *event;
2987         struct hw_perf_event *hwc;
2988         u64 now, period = TICK_NSEC;
2989         s64 delta;
2990 
2991         /*
2992          * only need to iterate over all events iff:
2993          * - context have events in frequency mode (needs freq adjust)
2994          * - there are events to unthrottle on this cpu
2995          */
2996         if (!(ctx->nr_freq || needs_unthr))
2997                 return;
2998 
2999         raw_spin_lock(&ctx->lock);
3000         perf_pmu_disable(ctx->pmu);
3001 
3002         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
3003                 if (event->state != PERF_EVENT_STATE_ACTIVE)
3004                         continue;
3005 
3006                 if (!event_filter_match(event))
3007                         continue;
3008 
3009                 perf_pmu_disable(event->pmu);
3010 
3011                 hwc = &event->hw;
3012 
3013                 if (hwc->interrupts == MAX_INTERRUPTS) {
3014                         hwc->interrupts = 0;
3015                         perf_log_throttle(event, 1);
3016                         event->pmu->start(event, 0);
3017                 }
3018 
3019                 if (!event->attr.freq || !event->attr.sample_freq)
3020                         goto next;
3021 
3022                 /*
3023                  * stop the event and update event->count
3024                  */
3025                 event->pmu->stop(event, PERF_EF_UPDATE);
3026 
3027                 now = local64_read(&event->count);
3028                 delta = now - hwc->freq_count_stamp;
3029                 hwc->freq_count_stamp = now;
3030 
3031                 /*
3032                  * restart the event
3033                  * reload only if value has changed
3034                  * we have stopped the event so tell that
3035                  * to perf_adjust_period() to avoid stopping it
3036                  * twice.
3037                  */
3038                 if (delta > 0)
3039                         perf_adjust_period(event, period, delta, false);
3040 
3041                 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
3042         next:
3043                 perf_pmu_enable(event->pmu);
3044         }
3045 
3046         perf_pmu_enable(ctx->pmu);
3047         raw_spin_unlock(&ctx->lock);
3048 }
3049 
3050 /*
3051  * Round-robin a context's events:
3052  */
3053 static void rotate_ctx(struct perf_event_context *ctx)
3054 {
3055         /*
3056          * Rotate the first entry last of non-pinned groups. Rotation might be
3057          * disabled by the inheritance code.
3058          */
3059         if (!ctx->rotate_disable)
3060                 list_rotate_left(&ctx->flexible_groups);
3061 }
3062 
3063 static int perf_rotate_context(struct perf_cpu_context *cpuctx)
3064 {
3065         struct perf_event_context *ctx = NULL;
3066         int rotate = 0;
3067 
3068         if (cpuctx->ctx.nr_events) {
3069                 if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
3070                         rotate = 1;
3071         }
3072 
3073         ctx = cpuctx->task_ctx;
3074         if (ctx && ctx->nr_events) {
3075                 if (ctx->nr_events != ctx->nr_active)
3076                         rotate = 1;
3077         }
3078 
3079         if (!rotate)
3080                 goto done;
3081 
3082         perf_ctx_lock(cpuctx, cpuctx->task_ctx);
3083         perf_pmu_disable(cpuctx->ctx.pmu);
3084 
3085         cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
3086         if (ctx)
3087                 ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
3088 
3089         rotate_ctx(&cpuctx->ctx);
3090         if (ctx)
3091                 rotate_ctx(ctx);
3092 
3093         perf_event_sched_in(cpuctx, ctx, current);
3094 
3095         perf_pmu_enable(cpuctx->ctx.pmu);
3096         perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
3097 done:
3098 
3099         return rotate;
3100 }
3101 
3102 #ifdef CONFIG_NO_HZ_FULL
3103 bool perf_event_can_stop_tick(void)
3104 {
3105         if (atomic_read(&nr_freq_events) ||
3106             __this_cpu_read(perf_throttled_count))
3107                 return false;
3108         else
3109                 return true;
3110 }
3111 #endif
3112 
3113 void perf_event_task_tick(void)
3114 {
3115         struct list_head *head = this_cpu_ptr(&active_ctx_list);
3116         struct perf_event_context *ctx, *tmp;
3117         int throttled;
3118 
3119         WARN_ON(!irqs_disabled());
3120 
3121         __this_cpu_inc(perf_throttled_seq);
3122         throttled = __this_cpu_xchg(perf_throttled_count, 0);
3123 
3124         list_for_each_entry_safe(ctx, tmp, head, active_ctx_list)
3125                 perf_adjust_freq_unthr_context(ctx, throttled);
3126 }
3127 
3128 static int event_enable_on_exec(struct perf_event *event,
3129                                 struct perf_event_context *ctx)
3130 {
3131         if (!event->attr.enable_on_exec)
3132                 return 0;
3133 
3134         event->attr.enable_on_exec = 0;
3135         if (event->state >= PERF_EVENT_STATE_INACTIVE)
3136                 return 0;
3137 
3138         __perf_event_mark_enabled(event);
3139 
3140         return 1;
3141 }
3142 
3143 /*
3144  * Enable all of a task's events that have been marked enable-on-exec.
3145  * This expects task == current.
3146  */
3147 static void perf_event_enable_on_exec(struct perf_event_context *ctx)
3148 {
3149         struct perf_event_context *clone_ctx = NULL;
3150         struct perf_event *event;
3151         unsigned long flags;
3152         int enabled = 0;
3153         int ret;
3154 
3155         local_irq_save(flags);
3156         if (!ctx || !ctx->nr_events)
3157                 goto out;
3158 
3159         /*
3160          * We must ctxsw out cgroup events to avoid conflict
3161          * when invoking perf_task_event_sched_in() later on
3162          * in this function. Otherwise we end up trying to
3163          * ctxswin cgroup events which are already scheduled
3164          * in.
3165          */
3166         perf_cgroup_sched_out(current, NULL);
3167 
3168         raw_spin_lock(&ctx->lock);
3169         task_ctx_sched_out(ctx);
3170 
3171         list_for_each_entry(event, &ctx->event_list, event_entry) {
3172                 ret = event_enable_on_exec(event, ctx);
3173                 if (ret)
3174                         enabled = 1;
3175         }
3176 
3177         /*
3178          * Unclone this context if we enabled any event.
3179          */
3180         if (enabled)
3181                 clone_ctx = unclone_ctx(ctx);
3182 
3183         raw_spin_unlock(&ctx->lock);
3184 
3185         /*
3186          * Also calls ctxswin for cgroup events, if any:
3187          */
3188         perf_event_context_sched_in(ctx, ctx->task);
3189 out:
3190         local_irq_restore(flags);
3191 
3192         if (clone_ctx)
3193                 put_ctx(clone_ctx);
3194 }
3195 
3196 void perf_event_exec(void)
3197 {
3198         struct perf_event_context *ctx;
3199         int ctxn;
3200 
3201         rcu_read_lock();
3202         for_each_task_context_nr(ctxn) {
3203                 ctx = current->perf_event_ctxp[ctxn];
3204                 if (!ctx)
3205                         continue;
3206 
3207                 perf_event_enable_on_exec(ctx);
3208         }
3209         rcu_read_unlock();
3210 }
3211 
3212 /*
3213  * Cross CPU call to read the hardware event
3214  */
3215 static void __perf_event_read(void *info)
3216 {
3217         struct perf_event *event = info;
3218         struct perf_event_context *ctx = event->ctx;
3219         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
3220 
3221         /*
3222          * If this is a task context, we need to check whether it is
3223          * the current task context of this cpu.  If not it has been
3224          * scheduled out before the smp call arrived.  In that case
3225          * event->count would have been updated to a recent sample
3226          * when the event was scheduled out.
3227          */
3228         if (ctx->task && cpuctx->task_ctx != ctx)
3229                 return;
3230 
3231         raw_spin_lock(&ctx->lock);
3232         if (ctx->is_active) {
3233                 update_context_time(ctx);
3234                 update_cgrp_time_from_event(event);
3235         }
3236         update_event_times(event);
3237         if (event->state == PERF_EVENT_STATE_ACTIVE)
3238                 event->pmu->read(event);
3239         raw_spin_unlock(&ctx->lock);
3240 }
3241 
3242 static inline u64 perf_event_count(struct perf_event *event)
3243 {
3244         if (event->pmu->count)
3245                 return event->pmu->count(event);
3246 
3247         return __perf_event_count(event);
3248 }
3249 
3250 /*
3251  * NMI-safe method to read a local event, that is an event that
3252  * is:
3253  *   - either for the current task, or for this CPU
3254  *   - does not have inherit set, for inherited task events
3255  *     will not be local and we cannot read them atomically
3256  *   - must not have a pmu::count method
3257  */
3258 u64 perf_event_read_local(struct perf_event *event)
3259 {
3260         unsigned long flags;
3261         u64 val;
3262 
3263         /*
3264          * Disabling interrupts avoids all counter scheduling (context
3265          * switches, timer based rotation and IPIs).
3266          */
3267         local_irq_save(flags);
3268 
3269         /* If this is a per-task event, it must be for current */
3270         WARN_ON_ONCE((event->attach_state & PERF_ATTACH_TASK) &&
3271                      event->hw.target != current);
3272 
3273         /* If this is a per-CPU event, it must be for this CPU */
3274         WARN_ON_ONCE(!(event->attach_state & PERF_ATTACH_TASK) &&
3275                      event->cpu != smp_processor_id());
3276 
3277         /*
3278          * It must not be an event with inherit set, we cannot read
3279          * all child counters from atomic context.
3280          */
3281         WARN_ON_ONCE(event->attr.inherit);
3282 
3283         /*
3284          * It must not have a pmu::count method, those are not
3285          * NMI safe.
3286          */
3287         WARN_ON_ONCE(event->pmu->count);
3288 
3289         /*
3290          * If the event is currently on this CPU, its either a per-task event,
3291          * or local to this CPU. Furthermore it means its ACTIVE (otherwise
3292          * oncpu == -1).
3293          */
3294         if (event->oncpu == smp_processor_id())
3295                 event->pmu->read(event);
3296 
3297         val = local64_read(&event->count);
3298         local_irq_restore(flags);
3299 
3300         return val;
3301 }
3302 
3303 static u64 perf_event_read(struct perf_event *event)
3304 {
3305         /*
3306          * If event is enabled and currently active on a CPU, update the
3307          * value in the event structure:
3308          */
3309         if (event->state == PERF_EVENT_STATE_ACTIVE) {
3310                 smp_call_function_single(event->oncpu,
3311                                          __perf_event_read, event, 1);
3312         } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
3313                 struct perf_event_context *ctx = event->ctx;
3314                 unsigned long flags;
3315 
3316                 raw_spin_lock_irqsave(&ctx->lock, flags);
3317                 /*
3318                  * may read while context is not active
3319                  * (e.g., thread is blocked), in that case
3320                  * we cannot update context time
3321                  */
3322                 if (ctx->is_active) {
3323                         update_context_time(ctx);
3324                         update_cgrp_time_from_event(event);
3325                 }
3326                 update_event_times(event);
3327                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
3328         }
3329 
3330         return perf_event_count(event);
3331 }
3332 
3333 /*
3334  * Initialize the perf_event context in a task_struct:
3335  */
3336 static void __perf_event_init_context(struct perf_event_context *ctx)
3337 {
3338         raw_spin_lock_init(&ctx->lock);
3339         mutex_init(&ctx->mutex);
3340         INIT_LIST_HEAD(&ctx->active_ctx_list);
3341         INIT_LIST_HEAD(&ctx->pinned_groups);
3342         INIT_LIST_HEAD(&ctx->flexible_groups);
3343         INIT_LIST_HEAD(&ctx->event_list);
3344         atomic_set(&ctx->refcount, 1);
3345         INIT_DELAYED_WORK(&ctx->orphans_remove, orphans_remove_work);
3346 }
3347 
3348 static struct perf_event_context *
3349 alloc_perf_context(struct pmu *pmu, struct task_struct *task)
3350 {
3351         struct perf_event_context *ctx;
3352 
3353         ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
3354         if (!ctx)
3355                 return NULL;
3356 
3357         __perf_event_init_context(ctx);
3358         if (task) {
3359                 ctx->task = task;
3360                 get_task_struct(task);
3361         }
3362         ctx->pmu = pmu;
3363 
3364         return ctx;
3365 }
3366 
3367 static struct task_struct *
3368 find_lively_task_by_vpid(pid_t vpid)
3369 {
3370         struct task_struct *task;
3371         int err;
3372 
3373         rcu_read_lock();
3374         if (!vpid)
3375                 task = current;
3376         else
3377                 task = find_task_by_vpid(vpid);
3378         if (task)
3379                 get_task_struct(task);
3380         rcu_read_unlock();
3381 
3382         if (!task)
3383                 return ERR_PTR(-ESRCH);
3384 
3385         /* Reuse ptrace permission checks for now. */
3386         err = -EACCES;
3387         if (!ptrace_may_access(task, PTRACE_MODE_READ))
3388                 goto errout;
3389 
3390         return task;
3391 errout:
3392         put_task_struct(task);
3393         return ERR_PTR(err);
3394 
3395 }
3396 
3397 /*
3398  * Returns a matching context with refcount and pincount.
3399  */
3400 static struct perf_event_context *
3401 find_get_context(struct pmu *pmu, struct task_struct *task,
3402                 struct perf_event *event)
3403 {
3404         struct perf_event_context *ctx, *clone_ctx = NULL;
3405         struct perf_cpu_context *cpuctx;
3406         void *task_ctx_data = NULL;
3407         unsigned long flags;
3408         int ctxn, err;
3409         int cpu = event->cpu;
3410 
3411         if (!task) {
3412                 /* Must be root to operate on a CPU event: */
3413                 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
3414                         return ERR_PTR(-EACCES);
3415 
3416                 /*
3417                  * We could be clever and allow to attach a event to an
3418                  * offline CPU and activate it when the CPU comes up, but
3419                  * that's for later.
3420                  */
3421                 if (!cpu_online(cpu))
3422                         return ERR_PTR(-ENODEV);
3423 
3424                 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
3425                 ctx = &cpuctx->ctx;
3426                 get_ctx(ctx);
3427                 ++ctx->pin_count;
3428 
3429                 return ctx;
3430         }
3431 
3432         err = -EINVAL;
3433         ctxn = pmu->task_ctx_nr;
3434         if (ctxn < 0)
3435                 goto errout;
3436 
3437         if (event->attach_state & PERF_ATTACH_TASK_DATA) {
3438                 task_ctx_data = kzalloc(pmu->task_ctx_size, GFP_KERNEL);
3439                 if (!task_ctx_data) {
3440                         err = -ENOMEM;
3441                         goto errout;
3442                 }
3443         }
3444 
3445 retry:
3446         ctx = perf_lock_task_context(task, ctxn, &flags);
3447         if (ctx) {
3448                 clone_ctx = unclone_ctx(ctx);
3449                 ++ctx->pin_count;
3450 
3451                 if (task_ctx_data && !ctx->task_ctx_data) {
3452                         ctx->task_ctx_data = task_ctx_data;
3453                         task_ctx_data = NULL;
3454                 }
3455                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
3456 
3457                 if (clone_ctx)
3458                         put_ctx(clone_ctx);
3459         } else {
3460                 ctx = alloc_perf_context(pmu, task);
3461                 err = -ENOMEM;
3462                 if (!ctx)
3463                         goto errout;
3464 
3465                 if (task_ctx_data) {
3466                         ctx->task_ctx_data = task_ctx_data;
3467                         task_ctx_data = NULL;
3468                 }
3469 
3470                 err = 0;
3471                 mutex_lock(&task->perf_event_mutex);
3472                 /*
3473                  * If it has already passed perf_event_exit_task().
3474                  * we must see PF_EXITING, it takes this mutex too.
3475                  */
3476                 if (task->flags & PF_EXITING)
3477                         err = -ESRCH;
3478                 else if (task->perf_event_ctxp[ctxn])
3479                         err = -EAGAIN;
3480                 else {
3481                         get_ctx(ctx);
3482                         ++ctx->pin_count;
3483                         rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
3484                 }
3485                 mutex_unlock(&task->perf_event_mutex);
3486 
3487                 if (unlikely(err)) {
3488                         put_ctx(ctx);
3489 
3490                         if (err == -EAGAIN)
3491                                 goto retry;
3492                         goto errout;
3493                 }
3494         }
3495 
3496         kfree(task_ctx_data);
3497         return ctx;
3498 
3499 errout:
3500         kfree(task_ctx_data);
3501         return ERR_PTR(err);
3502 }
3503 
3504 static void perf_event_free_filter(struct perf_event *event);
3505 static void perf_event_free_bpf_prog(struct perf_event *event);
3506 
3507 static void free_event_rcu(struct rcu_head *head)
3508 {
3509         struct perf_event *event;
3510 
3511         event = container_of(head, struct perf_event, rcu_head);
3512         if (event->ns)
3513                 put_pid_ns(event->ns);
3514         perf_event_free_filter(event);
3515         kfree(event);
3516 }
3517 
3518 static void ring_buffer_attach(struct perf_event *event,
3519                                struct ring_buffer *rb);
3520 
3521 static void unaccount_event_cpu(struct perf_event *event, int cpu)
3522 {
3523         if (event->parent)
3524                 return;
3525 
3526         if (is_cgroup_event(event))
3527                 atomic_dec(&per_cpu(perf_cgroup_events, cpu));
3528 }
3529 
3530 static void unaccount_event(struct perf_event *event)
3531 {
3532         if (event->parent)
3533                 return;
3534 
3535         if (event->attach_state & PERF_ATTACH_TASK)
3536                 static_key_slow_dec_deferred(&perf_sched_events);
3537         if (event->attr.mmap || event->attr.mmap_data)
3538                 atomic_dec(&nr_mmap_events);
3539         if (event->attr.comm)
3540                 atomic_dec(&nr_comm_events);
3541         if (event->attr.task)
3542                 atomic_dec(&nr_task_events);
3543         if (event->attr.freq)
3544                 atomic_dec(&nr_freq_events);
3545         if (event->attr.context_switch) {
3546                 static_key_slow_dec_deferred(&perf_sched_events);
3547                 atomic_dec(&nr_switch_events);
3548         }
3549         if (is_cgroup_event(event))
3550                 static_key_slow_dec_deferred(&perf_sched_events);
3551         if (has_branch_stack(event))
3552                 static_key_slow_dec_deferred(&perf_sched_events);
3553 
3554         unaccount_event_cpu(event, event->cpu);
3555 }
3556 
3557 /*
3558  * The following implement mutual exclusion of events on "exclusive" pmus
3559  * (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled
3560  * at a time, so we disallow creating events that might conflict, namely:
3561  *
3562  *  1) cpu-wide events in the presence of per-task events,
3563  *  2) per-task events in the presence of cpu-wide events,
3564  *  3) two matching events on the same context.
3565  *
3566  * The former two cases are handled in the allocation path (perf_event_alloc(),
3567  * __free_event()), the latter -- before the first perf_install_in_context().
3568  */
3569 static int exclusive_event_init(struct perf_event *event)
3570 {
3571         struct pmu *pmu = event->pmu;
3572 
3573         if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE))
3574                 return 0;
3575 
3576         /*
3577          * Prevent co-existence of per-task and cpu-wide events on the
3578          * same exclusive pmu.
3579          *
3580          * Negative pmu::exclusive_cnt means there are cpu-wide
3581          * events on this "exclusive" pmu, positive means there are
3582          * per-task events.
3583          *
3584          * Since this is called in perf_event_alloc() path, event::ctx
3585          * doesn't exist yet; it is, however, safe to use PERF_ATTACH_TASK
3586          * to mean "per-task event", because unlike other attach states it
3587          * never gets cleared.
3588          */
3589         if (event->attach_state & PERF_ATTACH_TASK) {
3590                 if (!atomic_inc_unless_negative(&pmu->exclusive_cnt))
3591                         return -EBUSY;
3592         } else {
3593                 if (!atomic_dec_unless_positive(&pmu->exclusive_cnt))
3594                         return -EBUSY;
3595         }
3596 
3597         return 0;
3598 }
3599 
3600 static void exclusive_event_destroy(struct perf_event *event)
3601 {
3602         struct pmu *pmu = event->pmu;
3603 
3604         if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE))
3605                 return;
3606 
3607         /* see comment in exclusive_event_init() */
3608         if (event->attach_state & PERF_ATTACH_TASK)
3609                 atomic_dec(&pmu->exclusive_cnt);
3610         else
3611                 atomic_inc(&pmu->exclusive_cnt);
3612 }
3613 
3614 static bool exclusive_event_match(struct perf_event *e1, struct perf_event *e2)
3615 {
3616         if ((e1->pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE) &&
3617             (e1->cpu == e2->cpu ||
3618              e1->cpu == -1 ||
3619              e2->cpu == -1))
3620                 return true;
3621         return false;
3622 }
3623 
3624 /* Called under the same ctx::mutex as perf_install_in_context() */
3625 static bool exclusive_event_installable(struct perf_event *event,
3626                                         struct perf_event_context *ctx)
3627 {
3628         struct perf_event *iter_event;
3629         struct pmu *pmu = event->pmu;
3630 
3631         if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE))
3632                 return true;
3633 
3634         list_for_each_entry(iter_event, &ctx->event_list, event_entry) {
3635                 if (exclusive_event_match(iter_event, event))
3636                         return false;
3637         }
3638 
3639         return true;
3640 }
3641 
3642 static void __free_event(struct perf_event *event)
3643 {
3644         if (!event->parent) {
3645                 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
3646                         put_callchain_buffers();
3647         }
3648 
3649         perf_event_free_bpf_prog(event);
3650 
3651         if (event->destroy)
3652                 event->destroy(event);
3653 
3654         if (event->ctx)
3655                 put_ctx(event->ctx);
3656 
3657         if (event->pmu) {
3658                 exclusive_event_destroy(event);
3659                 module_put(event->pmu->module);
3660         }
3661 
3662         call_rcu(&event->rcu_head, free_event_rcu);
3663 }
3664 
3665 static void _free_event(struct perf_event *event)
3666 {
3667         irq_work_sync(&event->pending);
3668 
3669         unaccount_event(event);
3670 
3671         if (event->rb) {
3672                 /*
3673                  * Can happen when we close an event with re-directed output.
3674                  *
3675                  * Since we have a 0 refcount, perf_mmap_close() will skip
3676                  * over us; possibly making our ring_buffer_put() the last.
3677                  */
3678                 mutex_lock(&event->mmap_mutex);
3679                 ring_buffer_attach(event, NULL);
3680                 mutex_unlock(&event->mmap_mutex);
3681         }
3682 
3683         if (is_cgroup_event(event))
3684                 perf_detach_cgroup(event);
3685 
3686         __free_event(event);
3687 }
3688 
3689 /*
3690  * Used to free events which have a known refcount of 1, such as in error paths
3691  * where the event isn't exposed yet and inherited events.
3692  */
3693 static void free_event(struct perf_event *event)
3694 {
3695         if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,
3696                                 "unexpected event refcount: %ld; ptr=%p\n",
3697                                 atomic_long_read(&event->refcount), event)) {
3698                 /* leak to avoid use-after-free */
3699                 return;
3700         }
3701 
3702         _free_event(event);
3703 }
3704 
3705 /*
3706  * Remove user event from the owner task.
3707  */
3708 static void perf_remove_from_owner(struct perf_event *event)
3709 {
3710         struct task_struct *owner;
3711 
3712         rcu_read_lock();
3713         owner = ACCESS_ONCE(event->owner);
3714         /*
3715          * Matches the smp_wmb() in perf_event_exit_task(). If we observe
3716          * !owner it means the list deletion is complete and we can indeed
3717          * free this event, otherwise we need to serialize on
3718          * owner->perf_event_mutex.
3719          */
3720         smp_read_barrier_depends();
3721         if (owner) {
3722                 /*
3723                  * Since delayed_put_task_struct() also drops the last
3724                  * task reference we can safely take a new reference
3725                  * while holding the rcu_read_lock().
3726                  */
3727                 get_task_struct(owner);
3728         }
3729         rcu_read_unlock();
3730 
3731         if (owner) {
3732                 /*
3733                  * If we're here through perf_event_exit_task() we're already
3734                  * holding ctx->mutex which would be an inversion wrt. the
3735                  * normal lock order.
3736                  *
3737                  * However we can safely take this lock because its the child
3738                  * ctx->mutex.
3739                  */
3740                 mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING);
3741 
3742                 /*
3743                  * We have to re-check the event->owner field, if it is cleared
3744                  * we raced with perf_event_exit_task(), acquiring the mutex
3745                  * ensured they're done, and we can proceed with freeing the
3746                  * event.
3747                  */
3748                 if (event->owner)
3749                         list_del_init(&event->owner_entry);
3750                 mutex_unlock(&owner->perf_event_mutex);
3751                 put_task_struct(owner);
3752         }
3753 }
3754 
3755 static void put_event(struct perf_event *event)
3756 {
3757         struct perf_event_context *ctx;
3758 
3759         if (!atomic_long_dec_and_test(&event->refcount))
3760                 return;
3761 
3762         if (!is_kernel_event(event))
3763                 perf_remove_from_owner(event);
3764 
3765         /*
3766          * There are two ways this annotation is useful:
3767          *
3768          *  1) there is a lock recursion from perf_event_exit_task
3769          *     see the comment there.
3770          *
3771          *  2) there is a lock-inversion with mmap_sem through
3772          *     perf_event_read_group(), which takes faults while
3773          *     holding ctx->mutex, however this is called after
3774          *     the last filedesc died, so there is no possibility
3775          *     to trigger the AB-BA case.
3776          */
3777         ctx = perf_event_ctx_lock_nested(event, SINGLE_DEPTH_NESTING);
3778         WARN_ON_ONCE(ctx->parent_ctx);
3779         perf_remove_from_context(event, true);
3780         perf_event_ctx_unlock(event, ctx);
3781 
3782         _free_event(event);
3783 }
3784 
3785 int perf_event_release_kernel(struct perf_event *event)
3786 {
3787         put_event(event);
3788         return 0;
3789 }
3790 EXPORT_SYMBOL_GPL(perf_event_release_kernel);
3791 
3792 /*
3793  * Called when the last reference to the file is gone.
3794  */
3795 static int perf_release(struct inode *inode, struct file *file)
3796 {
3797         put_event(file->private_data);
3798         return 0;
3799 }
3800 
3801 /*
3802  * Remove all orphanes events from the context.
3803  */
3804 static void orphans_remove_work(struct work_struct *work)
3805 {
3806         struct perf_event_context *ctx;
3807         struct perf_event *event, *tmp;
3808 
3809         ctx = container_of(work, struct perf_event_context,
3810                            orphans_remove.work);
3811 
3812         mutex_lock(&ctx->mutex);
3813         list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry) {
3814                 struct perf_event *parent_event = event->parent;
3815 
3816                 if (!is_orphaned_child(event))
3817                         continue;
3818 
3819                 perf_remove_from_context(event, true);
3820 
3821                 mutex_lock(&parent_event->child_mutex);
3822                 list_del_init(&event->child_list);
3823                 mutex_unlock(&parent_event->child_mutex);
3824 
3825                 free_event(event);
3826                 put_event(parent_event);
3827         }
3828 
3829         raw_spin_lock_irq(&ctx->lock);
3830         ctx->orphans_remove_sched = false;
3831         raw_spin_unlock_irq(&ctx->lock);
3832         mutex_unlock(&ctx->mutex);
3833 
3834         put_ctx(ctx);
3835 }
3836 
3837 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
3838 {
3839         struct perf_event *child;
3840         u64 total = 0;
3841 
3842         *enabled = 0;
3843         *running = 0;
3844 
3845         mutex_lock(&event->child_mutex);
3846         total += perf_event_read(event);
3847         *enabled += event->total_time_enabled +
3848                         atomic64_read(&event->child_total_time_enabled);
3849         *running += event->total_time_running +
3850                         atomic64_read(&event->child_total_time_running);
3851 
3852         list_for_each_entry(child, &event->child_list, child_list) {
3853                 total += perf_event_read(child);
3854                 *enabled += child->total_time_enabled;
3855                 *running += child->total_time_running;
3856         }
3857         mutex_unlock(&event->child_mutex);
3858 
3859         return total;
3860 }
3861 EXPORT_SYMBOL_GPL(perf_event_read_value);
3862 
3863 static int perf_event_read_group(struct perf_event *event,
3864                                    u64 read_format, char __user *buf)
3865 {
3866         struct perf_event *leader = event->group_leader, *sub;
3867         struct perf_event_context *ctx = leader->ctx;
3868         int n = 0, size = 0, ret;
3869         u64 count, enabled, running;
3870         u64 values[5];
3871 
3872         lockdep_assert_held(&ctx->mutex);
3873 
3874         count = perf_event_read_value(leader, &enabled, &running);
3875 
3876         values[n++] = 1 + leader->nr_siblings;
3877         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3878                 values[n++] = enabled;
3879         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3880                 values[n++] = running;
3881         values[n++] = count;
3882         if (read_format & PERF_FORMAT_ID)
3883                 values[n++] = primary_event_id(leader);
3884 
3885         size = n * sizeof(u64);
3886 
3887         if (copy_to_user(buf, values, size))
3888                 return -EFAULT;
3889 
3890         ret = size;
3891 
3892         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3893                 n = 0;
3894 
3895                 values[n++] = perf_event_read_value(sub, &enabled, &running);
3896                 if (read_format & PERF_FORMAT_ID)
3897                         values[n++] = primary_event_id(sub);
3898 
3899                 size = n * sizeof(u64);
3900 
3901                 if (copy_to_user(buf + ret, values, size)) {
3902                         return -EFAULT;
3903                 }
3904 
3905                 ret += size;
3906         }
3907 
3908         return ret;
3909 }
3910 
3911 static int perf_event_read_one(struct perf_event *event,
3912                                  u64 read_format, char __user *buf)
3913 {
3914         u64 enabled, running;
3915         u64 values[4];
3916         int n = 0;
3917 
3918         values[n++] = perf_event_read_value(event, &enabled, &running);
3919         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3920                 values[n++] = enabled;
3921         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3922                 values[n++] = running;
3923         if (read_format & PERF_FORMAT_ID)
3924                 values[n++] = primary_event_id(event);
3925 
3926         if (copy_to_user(buf, values, n * sizeof(u64)))
3927                 return -EFAULT;
3928 
3929         return n * sizeof(u64);
3930 }
3931 
3932 static bool is_event_hup(struct perf_event *event)
3933 {
3934         bool no_children;
3935 
3936         if (event->state != PERF_EVENT_STATE_EXIT)
3937                 return false;
3938 
3939         mutex_lock(&event->child_mutex);
3940         no_children = list_empty(&event->child_list);
3941         mutex_unlock(&event->child_mutex);
3942         return no_children;
3943 }
3944 
3945 /*
3946  * Read the performance event - simple non blocking version for now
3947  */
3948 static ssize_t
3949 perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
3950 {
3951         u64 read_format = event->attr.read_format;
3952         int ret;
3953 
3954         /*
3955          * Return end-of-file for a read on a event that is in
3956          * error state (i.e. because it was pinned but it couldn't be
3957          * scheduled on to the CPU at some point).
3958          */
3959         if (event->state == PERF_EVENT_STATE_ERROR)
3960                 return 0;
3961 
3962         if (count < event->read_size)
3963                 return -ENOSPC;
3964 
3965         WARN_ON_ONCE(event->ctx->parent_ctx);
3966         if (read_format & PERF_FORMAT_GROUP)
3967                 ret = perf_event_read_group(event, read_format, buf);
3968         else
3969                 ret = perf_event_read_one(event, read_format, buf);
3970 
3971         return ret;
3972 }
3973 
3974 static ssize_t
3975 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
3976 {
3977         struct perf_event *event = file->private_data;
3978         struct perf_event_context *ctx;
3979         int ret;
3980 
3981         ctx = perf_event_ctx_lock(event);
3982         ret = perf_read_hw(event, buf, count);
3983         perf_event_ctx_unlock(event, ctx);
3984 
3985         return ret;
3986 }
3987 
3988 static unsigned int perf_poll(struct file *file, poll_table *wait)
3989 {
3990         struct perf_event *event = file->private_data;
3991         struct ring_buffer *rb;
3992         unsigned int events = POLLHUP;
3993 
3994         poll_wait(file, &event->waitq, wait);
3995 
3996         if (is_event_hup(event))
3997                 return events;
3998 
3999         /*
4000          * Pin the event->rb by taking event->mmap_mutex; otherwise
4001          * perf_event_set_output() can swizzle our rb and make us miss wakeups.
4002          */
4003         mutex_lock(&event->mmap_mutex);
4004         rb = event->rb;
4005         if (rb)
4006                 events = atomic_xchg(&rb->poll, 0);
4007         mutex_unlock(&event->mmap_mutex);
4008         return events;
4009 }
4010 
4011 static void _perf_event_reset(struct perf_event *event)
4012 {
4013         (void)perf_event_read(event);
4014         local64_set(&event->count, 0);
4015         perf_event_update_userpage(event);
4016 }
4017 
4018 /*
4019  * Holding the top-level event's child_mutex means that any
4020  * descendant process that has inherited this event will block
4021  * in sync_child_event if it goes to exit, thus satisfying the
4022  * task existence requirements of perf_event_enable/disable.
4023  */
4024 static void perf_event_for_each_child(struct perf_event *event,
4025                                         void (*func)(struct perf_event *))
4026 {
4027         struct perf_event *child;
4028 
4029         WARN_ON_ONCE(event->ctx->parent_ctx);
4030 
4031         mutex_lock(&event->child_mutex);
4032         func(event);
4033         list_for_each_entry(child, &event->child_list, child_list)
4034                 func(child);
4035         mutex_unlock(&event->child_mutex);
4036 }
4037 
4038 static void perf_event_for_each(struct perf_event *event,
4039                                   void (*func)(struct perf_event *))
4040 {
4041         struct perf_event_context *ctx = event->ctx;
4042         struct perf_event *sibling;
4043 
4044         lockdep_assert_held(&ctx->mutex);
4045 
4046         event = event->group_leader;
4047 
4048         perf_event_for_each_child(event, func);
4049         list_for_each_entry(sibling, &event->sibling_list, group_entry)
4050                 perf_event_for_each_child(sibling, func);
4051 }
4052 
4053 struct period_event {
4054         struct perf_event *event;
4055         u64 value;
4056 };
4057 
4058 static int __perf_event_period(void *info)
4059 {
4060         struct period_event *pe = info;
4061         struct perf_event *event = pe->event;
4062         struct perf_event_context *ctx = event->ctx;
4063         u64 value = pe->value;
4064         bool active;
4065 
4066         raw_spin_lock(&ctx->lock);
4067         if (event->attr.freq) {
4068                 event->attr.sample_freq = value;
4069         } else {
4070                 event->attr.sample_period = value;
4071                 event->hw.sample_period = value;
4072         }
4073 
4074         active = (event->state == PERF_EVENT_STATE_ACTIVE);
4075         if (active) {
4076                 perf_pmu_disable(ctx->pmu);
4077                 event->pmu->stop(event, PERF_EF_UPDATE);
4078         }
4079 
4080         local64_set(&event->hw.period_left, 0);
4081 
4082         if (active) {
4083                 event->pmu->start(event, PERF_EF_RELOAD);
4084                 perf_pmu_enable(ctx->pmu);
4085         }
4086         raw_spin_unlock(&ctx->lock);
4087 
4088         return 0;
4089 }
4090 
4091 static int perf_event_period(struct perf_event *event, u64 __user *arg)
4092 {
4093         struct period_event pe = { .event = event, };
4094         struct perf_event_context *ctx = event->ctx;
4095         struct task_struct *task;
4096         u64 value;
4097 
4098         if (!is_sampling_event(event))
4099                 return -EINVAL;
4100 
4101         if (copy_from_user(&value, arg, sizeof(value)))
4102                 return -EFAULT;
4103 
4104         if (!value)
4105                 return -EINVAL;
4106 
4107         if (event->attr.freq && value > sysctl_perf_event_sample_rate)
4108                 return -EINVAL;
4109 
4110         task = ctx->task;
4111         pe.value = value;
4112 
4113         if (!task) {
4114                 cpu_function_call(event->cpu, __perf_event_period, &pe);
4115                 return 0;
4116         }
4117 
4118 retry:
4119         if (!task_function_call(task, __perf_event_period, &pe))
4120                 return 0;
4121 
4122         raw_spin_lock_irq(&ctx->lock);
4123         if (ctx->is_active) {
4124                 raw_spin_unlock_irq(&ctx->lock);
4125                 task = ctx->task;
4126                 goto retry;
4127         }
4128 
4129         __perf_event_period(&pe);
4130         raw_spin_unlock_irq(&ctx->lock);
4131 
4132         return 0;
4133 }
4134 
4135 static const struct file_operations perf_fops;
4136 
4137 static inline int perf_fget_light(int fd, struct fd *p)
4138 {
4139         struct fd f = fdget(fd);
4140         if (!f.file)
4141                 return -EBADF;
4142 
4143         if (f.file->f_op != &perf_fops) {
4144                 fdput(f);
4145                 return -EBADF;
4146         }
4147         *p = f;
4148         return 0;
4149 }
4150 
4151 static int perf_event_set_output(struct perf_event *event,
4152                                  struct perf_event *output_event);
4153 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
4154 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd);
4155 
4156 static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)
4157 {
4158         void (*func)(struct perf_event *);
4159         u32 flags = arg;
4160 
4161         switch (cmd) {
4162         case PERF_EVENT_IOC_ENABLE:
4163                 func = _perf_event_enable;
4164                 break;
4165         case PERF_EVENT_IOC_DISABLE:
4166                 func = _perf_event_disable;
4167                 break;
4168         case PERF_EVENT_IOC_RESET:
4169                 func = _perf_event_reset;
4170                 break;
4171 
4172         case PERF_EVENT_IOC_REFRESH:
4173                 return _perf_event_refresh(event, arg);
4174 
4175         case PERF_EVENT_IOC_PERIOD:
4176                 return perf_event_period(event, (u64 __user *)arg);
4177 
4178         case PERF_EVENT_IOC_ID:
4179         {
4180                 u64 id = primary_event_id(event);
4181 
4182                 if (copy_to_user((void __user *)arg, &id, sizeof(id)))
4183                         return -EFAULT;
4184                 return 0;
4185         }
4186 
4187         case PERF_EVENT_IOC_SET_OUTPUT:
4188         {
4189                 int ret;
4190                 if (arg != -1) {
4191                         struct perf_event *output_event;
4192                         struct fd output;
4193                         ret = perf_fget_light(arg, &output);
4194                         if (ret)
4195                                 return ret;
4196                         output_event = output.file->private_data;
4197                         ret = perf_event_set_output(event, output_event);
4198                         fdput(output);
4199                 } else {
4200                         ret = perf_event_set_output(event, NULL);
4201                 }
4202                 return ret;
4203         }
4204 
4205         case PERF_EVENT_IOC_SET_FILTER:
4206                 return perf_event_set_filter(event, (void __user *)arg);
4207 
4208         case PERF_EVENT_IOC_SET_BPF:
4209                 return perf_event_set_bpf_prog(event, arg);
4210 
4211         default:
4212                 return -ENOTTY;
4213         }
4214 
4215         if (flags & PERF_IOC_FLAG_GROUP)
4216                 perf_event_for_each(event, func);
4217         else
4218                 perf_event_for_each_child(event, func);
4219 
4220         return 0;
4221 }
4222 
4223 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
4224 {
4225         struct perf_event *event = file->private_data;
4226         struct perf_event_context *ctx;
4227         long ret;
4228 
4229         ctx = perf_event_ctx_lock(event);
4230         ret = _perf_ioctl(event, cmd, arg);
4231         perf_event_ctx_unlock(event, ctx);
4232 
4233         return ret;
4234 }
4235 
4236 #ifdef CONFIG_COMPAT
4237 static long perf_compat_ioctl(struct file *file, unsigned int cmd,
4238                                 unsigned long arg)
4239 {
4240         switch (_IOC_NR(cmd)) {
4241         case _IOC_NR(PERF_EVENT_IOC_SET_FILTER):
4242         case _IOC_NR(PERF_EVENT_IOC_ID):
4243                 /* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */
4244                 if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) {
4245                         cmd &= ~IOCSIZE_MASK;
4246                         cmd |= sizeof(void *) << IOCSIZE_SHIFT;
4247                 }
4248                 break;
4249         }
4250         return perf_ioctl(file, cmd, arg);
4251 }
4252 #else
4253 # define perf_compat_ioctl NULL
4254 #endif
4255 
4256 int perf_event_task_enable(void)
4257 {
4258         struct perf_event_context *ctx;
4259         struct perf_event *event;
4260 
4261         mutex_lock(&current->perf_event_mutex);
4262         list_for_each_entry(event, &current->perf_event_list, owner_entry) {
4263                 ctx = perf_event_ctx_lock(event);
4264                 perf_event_for_each_child(event, _perf_event_enable);
4265                 perf_event_ctx_unlock(event, ctx);
4266         }
4267         mutex_unlock(&current->perf_event_mutex);
4268 
4269         return 0;
4270 }
4271 
4272 int perf_event_task_disable(void)
4273 {
4274         struct perf_event_context *ctx;
4275         struct perf_event *event;
4276 
4277         mutex_lock(&current->perf_event_mutex);
4278         list_for_each_entry(event, &current->perf_event_list, owner_entry) {
4279                 ctx = perf_event_ctx_lock(event);
4280                 perf_event_for_each_child(event, _perf_event_disable);
4281                 perf_event_ctx_unlock(event, ctx);
4282         }
4283         mutex_unlock(&current->perf_event_mutex);
4284 
4285         return 0;
4286 }
4287 
4288 static int perf_event_index(struct perf_event *event)
4289 {
4290         if (event->hw.state & PERF_HES_STOPPED)
4291                 return 0;
4292 
4293         if (event->state != PERF_EVENT_STATE_ACTIVE)
4294                 return 0;
4295 
4296         return event->pmu->event_idx(event);
4297 }
4298 
4299 static void calc_timer_values(struct perf_event *event,
4300                                 u64 *now,
4301                                 u64 *enabled,
4302                                 u64 *running)
4303 {
4304         u64 ctx_time;
4305 
4306         *now = perf_clock();
4307         ctx_time = event->shadow_ctx_time + *now;
4308         *enabled = ctx_time - event->tstamp_enabled;
4309         *running = ctx_time - event->tstamp_running;
4310 }
4311 
4312 static void perf_event_init_userpage(struct perf_event *event)
4313 {
4314         struct perf_event_mmap_page *userpg;
4315         struct ring_buffer *rb;
4316 
4317         rcu_read_lock();
4318         rb = rcu_dereference(event->rb);
4319         if (!rb)
4320                 goto unlock;
4321 
4322         userpg = rb->user_page;
4323 
4324         /* Allow new userspace to detect that bit 0 is deprecated */
4325         userpg->cap_bit0_is_deprecated = 1;
4326         userpg->size = offsetof(struct perf_event_mmap_page, __reserved);
4327         userpg->data_offset = PAGE_SIZE;
4328         userpg->data_size = perf_data_size(rb);
4329 
4330 unlock:
4331         rcu_read_unlock();
4332 }
4333 
4334 void __weak arch_perf_update_userpage(
4335         struct perf_event *event, struct perf_event_mmap_page *userpg, u64 now)
4336 {
4337 }
4338 
4339 /*
4340  * Callers need to ensure there can be no nesting of this function, otherwise
4341  * the seqlock logic goes bad. We can not serialize this because the arch
4342  * code calls this from NMI context.
4343  */
4344 void perf_event_update_userpage(struct perf_event *event)
4345 {
4346         struct perf_event_mmap_page *userpg;
4347         struct ring_buffer *rb;
4348         u64 enabled, running, now;
4349 
4350         rcu_read_lock();
4351         rb = rcu_dereference(event->rb);
4352         if (!rb)
4353                 goto unlock;
4354 
4355         /*
4356          * compute total_time_enabled, total_time_running
4357          * based on snapshot values taken when the event
4358          * was last scheduled in.
4359          *
4360          * we cannot simply called update_context_time()
4361          * because of locking issue as we can be called in
4362          * NMI context
4363          */
4364         calc_timer_values(event, &now, &enabled, &running);
4365 
4366         userpg = rb->user_page;
4367         /*
4368          * Disable preemption so as to not let the corresponding user-space
4369          * spin too long if we get preempted.
4370          */
4371         preempt_disable();
4372         ++userpg->lock;
4373         barrier();
4374         userpg->index = perf_event_index(event);
4375         userpg->offset = perf_event_count(event);
4376         if (userpg->index)
4377                 userpg->offset -= local64_read(&event->hw.prev_count);
4378 
4379         userpg->time_enabled = enabled +
4380                         atomic64_read(&event->child_total_time_enabled);
4381 
4382         userpg->time_running = running +
4383                         atomic64_read(&event->child_total_time_running);
4384 
4385         arch_perf_update_userpage(event, userpg, now);
4386 
4387         barrier();
4388         ++userpg->lock;
4389         preempt_enable();
4390 unlock:
4391         rcu_read_unlock();
4392 }
4393 
4394 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
4395 {
4396         struct perf_event *event = vma->vm_file->private_data;
4397         struct ring_buffer *rb;
4398         int ret = VM_FAULT_SIGBUS;
4399 
4400         if (vmf->flags & FAULT_FLAG_MKWRITE) {
4401                 if (vmf->pgoff == 0)
4402                         ret = 0;
4403                 return ret;
4404         }
4405 
4406         rcu_read_lock();
4407         rb = rcu_dereference(event->rb);
4408         if (!rb)
4409                 goto unlock;
4410 
4411         if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
4412                 goto unlock;
4413 
4414         vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
4415         if (!vmf->page)
4416                 goto unlock;
4417 
4418         get_page(vmf->page);
4419         vmf->page->mapping = vma->vm_file->f_mapping;
4420         vmf->page->index   = vmf->pgoff;
4421 
4422         ret = 0;
4423 unlock:
4424         rcu_read_unlock();
4425 
4426         return ret;
4427 }
4428 
4429 static void ring_buffer_attach(struct perf_event *event,
4430                                struct ring_buffer *rb)
4431 {
4432         struct ring_buffer *old_rb = NULL;
4433         unsigned long flags;
4434 
4435         if (event->rb) {
4436                 /*
4437                  * Should be impossible, we set this when removing
4438                  * event->rb_entry and wait/clear when adding event->rb_entry.
4439                  */
4440                 WARN_ON_ONCE(event->rcu_pending);
4441 
4442                 old_rb = event->rb;
4443                 spin_lock_irqsave(&old_rb->event_lock, flags);
4444                 list_del_rcu(&event->rb_entry);
4445                 spin_unlock_irqrestore(&old_rb->event_lock, flags);
4446 
4447                 event->rcu_batches = get_state_synchronize_rcu();
4448                 event->rcu_pending = 1;
4449         }
4450 
4451         if (rb) {
4452                 if (event->rcu_pending) {
4453                         cond_synchronize_rcu(event->rcu_batches);
4454                         event->rcu_pending = 0;
4455                 }
4456 
4457                 spin_lock_irqsave(&rb->event_lock, flags);
4458                 list_add_rcu(&event->rb_entry, &rb->event_list);
4459                 spin_unlock_irqrestore(&rb->event_lock, flags);
4460         }
4461 
4462         rcu_assign_pointer(event->rb, rb);
4463 
4464         if (old_rb) {
4465                 ring_buffer_put(old_rb);
4466                 /*
4467                  * Since we detached before setting the new rb, so that we
4468                  * could attach the new rb, we could have missed a wakeup.
4469                  * Provide it now.
4470                  */
4471                 wake_up_all(&event->waitq);
4472         }
4473 }
4474 
4475 static void ring_buffer_wakeup(struct perf_event *event)
4476 {
4477         struct ring_buffer *rb;
4478 
4479         rcu_read_lock();
4480         rb = rcu_dereference(event->rb);
4481         if (rb) {
4482                 list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
4483                         wake_up_all(&event->waitq);
4484         }
4485         rcu_read_unlock();
4486 }
4487 
4488 struct ring_buffer *ring_buffer_get(struct perf_event *event)
4489 {
4490         struct ring_buffer *rb;
4491 
4492         rcu_read_lock();
4493         rb = rcu_dereference(event->rb);
4494         if (rb) {
4495                 if (!atomic_inc_not_zero(&rb->refcount))
4496                         rb = NULL;
4497         }
4498         rcu_read_unlock();
4499 
4500         return rb;
4501 }
4502 
4503 void ring_buffer_put(struct ring_buffer *rb)
4504 {
4505         if (!atomic_dec_and_test(&rb->refcount))
4506                 return;
4507 
4508         WARN_ON_ONCE(!list_empty(&rb->event_list));
4509 
4510         call_rcu(&rb->rcu_head, rb_free_rcu);
4511 }
4512 
4513 static void perf_mmap_open(struct vm_area_struct *vma)
4514 {
4515         struct perf_event *event = vma->vm_file->private_data;
4516 
4517         atomic_inc(&event->mmap_count);
4518         atomic_inc(&event->rb->mmap_count);
4519 
4520         if (vma->vm_pgoff)
4521                 atomic_inc(&event->rb->aux_mmap_count);
4522 
4523         if (event->pmu->event_mapped)
4524                 event->pmu->event_mapped(event);
4525 }
4526 
4527 /*
4528  * A buffer can be mmap()ed multiple times; either directly through the same
4529  * event, or through other events by use of perf_event_set_output().
4530  *
4531  * In order to undo the VM accounting done by perf_mmap() we need to destroy
4532  * the buffer here, where we still have a VM context. This means we need
4533  * to detach all events redirecting to us.
4534  */
4535 static void perf_mmap_close(struct vm_area_struct *vma)
4536 {
4537         struct perf_event *event = vma->vm_file->private_data;
4538 
4539         struct ring_buffer *rb = ring_buffer_get(event);
4540         struct user_struct *mmap_user = rb->mmap_user;
4541         int mmap_locked = rb->mmap_locked;
4542         unsigned long size = perf_data_size(rb);
4543 
4544         if (event->pmu->event_unmapped)
4545                 event->pmu->event_unmapped(event);
4546 
4547         /*
4548          * rb->aux_mmap_count will always drop before rb->mmap_count and
4549          * event->mmap_count, so it is ok to use event->mmap_mutex to
4550          * serialize with perf_mmap here.
4551          */
4552         if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff &&
4553             atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &event->mmap_mutex)) {
4554                 atomic_long_sub(rb->aux_nr_pages, &mmap_user->locked_vm);
4555                 vma->vm_mm->pinned_vm -= rb->aux_mmap_locked;
4556 
4557                 rb_free_aux(rb);
4558                 mutex_unlock(&event->mmap_mutex);
4559         }
4560 
4561         atomic_dec(&rb->mmap_count);
4562 
4563         if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
4564                 goto out_put;
4565 
4566         ring_buffer_attach(event, NULL);
4567         mutex_unlock(&event->mmap_mutex);
4568 
4569         /* If there's still other mmap()s of this buffer, we're done. */
4570         if (atomic_read(&rb->mmap_count))
4571                 goto out_put;
4572 
4573         /*
4574          * No other mmap()s, detach from all other events that might redirect
4575          * into the now unreachable buffer. Somewhat complicated by the
4576          * fact that rb::event_lock otherwise nests inside mmap_mutex.
4577          */
4578 again:
4579         rcu_read_lock();
4580         list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
4581                 if (!atomic_long_inc_not_zero(&event->refcount)) {
4582                         /*
4583                          * This event is en-route to free_event() which will
4584                          * detach it and remove it from the list.
4585                          */
4586                         continue;
4587                 }
4588                 rcu_read_unlock();
4589 
4590                 mutex_lock(&event->mmap_mutex);
4591                 /*
4592                  * Check we didn't race with perf_event_set_output() which can
4593                  * swizzle the rb from under us while we were waiting to
4594                  * acquire mmap_mutex.
4595                  *
4596                  * If we find a different rb; ignore this event, a next
4597                  * iteration will no longer find it on the list. We have to
4598                  * still restart the iteration to make sure we're not now
4599                  * iterating the wrong list.
4600                  */
4601                 if (event->rb == rb)
4602                         ring_buffer_attach(event, NULL);
4603 
4604                 mutex_unlock(&event->mmap_mutex);
4605                 put_event(event);
4606 
4607                 /*
4608                  * Restart the iteration; either we're on the wrong list or
4609                  * destroyed its integrity by doing a deletion.
4610                  */
4611                 goto again;
4612         }
4613         rcu_read_unlock();
4614 
4615         /*
4616          * It could be there's still a few 0-ref events on the list; they'll
4617          * get cleaned up by free_event() -- they'll also still have their
4618          * ref on the rb and will free it whenever they are done with it.
4619          *
4620          * Aside from that, this buffer is 'fully' detached and unmapped,
4621          * undo the VM accounting.
4622          */
4623 
4624         atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm);
4625         vma->vm_mm->pinned_vm -= mmap_locked;
4626         free_uid(mmap_user);
4627 
4628 out_put:
4629         ring_buffer_put(rb); /* could be last */
4630 }
4631 
4632 static const struct vm_operations_struct perf_mmap_vmops = {
4633         .open           = perf_mmap_open,
4634         .close          = perf_mmap_close, /* non mergable */
4635         .fault          = perf_mmap_fault,
4636         .page_mkwrite   = perf_mmap_fault,
4637 };
4638 
4639 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
4640 {
4641         struct perf_event *event = file->private_data;
4642         unsigned long user_locked, user_lock_limit;
4643         struct user_struct *user = current_user();
4644         unsigned long locked, lock_limit;
4645         struct ring_buffer *rb = NULL;
4646         unsigned long vma_size;
4647         unsigned long nr_pages;
4648         long user_extra = 0, extra = 0;
4649         int ret = 0, flags = 0;
4650 
4651         /*
4652          * Don't allow mmap() of inherited per-task counters. This would
4653          * create a performance issue due to all children writing to the
4654          * same rb.
4655          */
4656         if (event->cpu == -1 && event->attr.inherit)
4657                 return -EINVAL;
4658 
4659         if (!(vma->vm_flags & VM_SHARED))
4660                 return -EINVAL;
4661 
4662         vma_size = vma->vm_end - vma->vm_start;
4663 
4664         if (vma->vm_pgoff == 0) {
4665                 nr_pages = (vma_size / PAGE_SIZE) - 1;
4666         } else {
4667                 /*
4668                  * AUX area mapping: if rb->aux_nr_pages != 0, it's already
4669                  * mapped, all subsequent mappings should have the same size
4670                  * and offset. Must be above the normal perf buffer.
4671                  */
4672                 u64 aux_offset, aux_size;
4673 
4674                 if (!event->rb)
4675                         return -EINVAL;
4676 
4677                 nr_pages = vma_size / PAGE_SIZE;
4678 
4679                 mutex_lock(&event->mmap_mutex);
4680                 ret = -EINVAL;
4681 
4682                 rb = event->rb;
4683                 if (!rb)
4684                         goto aux_unlock;
4685 
4686                 aux_offset = ACCESS_ONCE(rb->user_page->aux_offset);
4687                 aux_size = ACCESS_ONCE(rb->user_page->aux_size);
4688 
4689                 if (aux_offset < perf_data_size(rb) + PAGE_SIZE)
4690                         goto aux_unlock;
4691 
4692                 if (aux_offset != vma->vm_pgoff << PAGE_SHIFT)
4693                         goto aux_unlock;
4694 
4695                 /* already mapped with a different offset */
4696                 if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff)
4697                         goto aux_unlock;
4698 
4699                 if (aux_size != vma_size || aux_size != nr_pages * PAGE_SIZE)
4700                         goto aux_unlock;
4701 
4702                 /* already mapped with a different size */
4703                 if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages)
4704                         goto aux_unlock;
4705 
4706                 if (!is_power_of_2(nr_pages))
4707                         goto aux_unlock;
4708 
4709                 if (!atomic_inc_not_zero(&rb->mmap_count))
4710                         goto aux_unlock;
4711 
4712                 if (rb_has_aux(rb)) {
4713                         atomic_inc(&rb->aux_mmap_count);
4714                         ret = 0;
4715                         goto unlock;
4716                 }
4717 
4718                 atomic_set(&rb->aux_mmap_count, 1);
4719                 user_extra = nr_pages;
4720 
4721                 goto accounting;
4722         }
4723 
4724         /*
4725          * If we have rb pages ensure they're a power-of-two number, so we
4726          * can do bitmasks instead of modulo.
4727          */
4728         if (nr_pages != 0 && !is_power_of_2(nr_pages))
4729                 return -EINVAL;
4730 
4731         if (vma_size != PAGE_SIZE * (1 + nr_pages))
4732                 return -EINVAL;
4733 
4734         WARN_ON_ONCE(event->ctx->parent_ctx);
4735 again:
4736         mutex_lock(&event->mmap_mutex);
4737         if (event->rb) {
4738                 if (event->rb->nr_pages != nr_pages) {
4739                         ret = -EINVAL;
4740                         goto unlock;
4741                 }
4742 
4743                 if (!atomic_inc_not_zero(&event->rb->mmap_count)) {
4744                         /*
4745                          * Raced against perf_mmap_close() through
4746                          * perf_event_set_output(). Try again, hope for better
4747                          * luck.
4748                          */
4749                         mutex_unlock(&event->mmap_mutex);
4750                         goto again;
4751                 }
4752 
4753                 goto unlock;
4754         }
4755 
4756         user_extra = nr_pages + 1;
4757 
4758 accounting:
4759         user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
4760 
4761         /*
4762          * Increase the limit linearly with more CPUs:
4763          */
4764         user_lock_limit *= num_online_cpus();
4765 
4766         user_locked = atomic_long_read(&user->locked_vm) + user_extra;
4767 
4768         if (user_locked > user_lock_limit)
4769                 extra = user_locked - user_lock_limit;
4770 
4771         lock_limit = rlimit(RLIMIT_MEMLOCK);
4772         lock_limit >>= PAGE_SHIFT;
4773         locked = vma->vm_mm->pinned_vm + extra;
4774 
4775         if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
4776                 !capable(CAP_IPC_LOCK)) {
4777                 ret = -EPERM;
4778                 goto unlock;
4779         }
4780 
4781         WARN_ON(!rb && event->rb);
4782 
4783         if (vma->vm_flags & VM_WRITE)
4784                 flags |= RING_BUFFER_WRITABLE;
4785 
4786         if (!rb) {
4787                 rb = rb_alloc(nr_pages,
4788                               event->attr.watermark ? event->attr.wakeup_watermark : 0,
4789                               event->cpu, flags);
4790 
4791                 if (!rb) {
4792                         ret = -ENOMEM;
4793                         goto unlock;
4794                 }
4795 
4796                 atomic_set(&rb->mmap_count, 1);
4797                 rb->mmap_user = get_current_user();
4798                 rb->mmap_locked = extra;
4799 
4800                 ring_buffer_attach(event, rb);
4801 
4802                 perf_event_init_userpage(event);
4803                 perf_event_update_userpage(event);
4804         } else {
4805                 ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages,
4806                                    event->attr.aux_watermark, flags);
4807                 if (!ret)
4808                         rb->aux_mmap_locked = extra;
4809         }
4810 
4811 unlock:
4812         if (!ret) {
4813                 atomic_long_add(user_extra, &user->locked_vm);
4814                 vma->vm_mm->pinned_vm += extra;
4815 
4816                 atomic_inc(&event->mmap_count);
4817         } else if (rb) {
4818                 atomic_dec(&rb->mmap_count);
4819         }
4820 aux_unlock:
4821         mutex_unlock(&event->mmap_mutex);
4822 
4823         /*
4824          * Since pinned accounting is per vm we cannot allow fork() to copy our
4825          * vma.
4826          */
4827         vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;
4828         vma->vm_ops = &perf_mmap_vmops;
4829 
4830         if (event->pmu->event_mapped)
4831                 event->pmu->event_mapped(event);
4832 
4833         return ret;
4834 }
4835 
4836 static int perf_fasync(int fd, struct file *filp, int on)
4837 {
4838         struct inode *inode = file_inode(filp);
4839         struct perf_event *event = filp->private_data;
4840         int retval;
4841 
4842         mutex_lock(&inode->i_mutex);
4843         retval = fasync_helper(fd, filp, on, &event->fasync);
4844         mutex_unlock(&inode->i_mutex);
4845 
4846         if (retval < 0)
4847                 return retval;
4848 
4849         return 0;
4850 }
4851 
4852 static const struct file_operations perf_fops = {
4853         .llseek                 = no_llseek,
4854         .release                = perf_release,
4855         .read                   = perf_read,
4856         .poll                   = perf_poll,
4857         .unlocked_ioctl         = perf_ioctl,
4858         .compat_ioctl           = perf_compat_ioctl,
4859         .mmap                   = perf_mmap,
4860         .fasync                 = perf_fasync,
4861 };
4862 
4863 /*
4864  * Perf event wakeup
4865  *
4866  * If there's data, ensure we set the poll() state and publish everything
4867  * to user-space before waking everybody up.
4868  */
4869 
4870 static inline struct fasync_struct **perf_event_fasync(struct perf_event *event)
4871 {
4872         /* only the parent has fasync state */
4873         if (event->parent)
4874                 event = event->parent;
4875         return &event->fasync;
4876 }
4877 
4878 void perf_event_wakeup(struct perf_event *event)
4879 {
4880         ring_buffer_wakeup(event);
4881 
4882         if (event->pending_kill) {
4883                 kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill);
4884                 event->pending_kill = 0;
4885         }
4886 }
4887 
4888 static void perf_pending_event(struct irq_work *entry)
4889 {
4890         struct perf_event *event = container_of(entry,
4891                         struct perf_event, pending);
4892         int rctx;
4893 
4894         rctx = perf_swevent_get_recursion_context();
4895         /*
4896          * If we 'fail' here, that's OK, it means recursion is already disabled
4897          * and we won't recurse 'further'.
4898          */
4899 
4900         if (event->pending_disable) {
4901                 event->pending_disable = 0;
4902                 __perf_event_disable(event);
4903         }
4904 
4905         if (event->pending_wakeup) {
4906                 event->pending_wakeup = 0;
4907                 perf_event_wakeup(event);
4908         }
4909 
4910         if (rctx >= 0)
4911                 perf_swevent_put_recursion_context(rctx);
4912 }
4913 
4914 /*
4915  * We assume there is only KVM supporting the callbacks.
4916  * Later on, we might change it to a list if there is
4917  * another virtualization implementation supporting the callbacks.
4918  */
4919 struct perf_guest_info_callbacks *perf_guest_cbs;
4920 
4921 int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
4922 {
4923         perf_guest_cbs = cbs;
4924         return 0;
4925 }
4926 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
4927 
4928 int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
4929 {
4930         perf_guest_cbs = NULL;
4931         return 0;
4932 }
4933 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
4934 
4935 static void
4936 perf_output_sample_regs(struct perf_output_handle *handle,
4937                         struct pt_regs *regs, u64 mask)
4938 {
4939         int bit;
4940 
4941         for_each_set_bit(bit, (const unsigned long *) &mask,
4942                          sizeof(mask) * BITS_PER_BYTE) {
4943                 u64 val;
4944 
4945                 val = perf_reg_value(regs, bit);
4946                 perf_output_put(handle, val);
4947         }
4948 }
4949 
4950 static void perf_sample_regs_user(struct perf_regs *regs_user,
4951                                   struct pt_regs *regs,
4952                                   struct pt_regs *regs_user_copy)
4953 {
4954         if (user_mode(regs)) {
4955                 regs_user->abi = perf_reg_abi(current);
4956                 regs_user->regs = regs;
4957         } else if (current->mm) {
4958                 perf_get_regs_user(regs_user, regs, regs_user_copy);
4959         } else {
4960                 regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
4961                 regs_user->regs = NULL;
4962         }
4963 }
4964 
4965 static void perf_sample_regs_intr(struct perf_regs *regs_intr,
4966                                   struct pt_regs *regs)
4967 {
4968         regs_intr->regs = regs;
4969         regs_intr->abi  = perf_reg_abi(current);
4970 }
4971 
4972 
4973 /*
4974  * Get remaining task size from user stack pointer.
4975  *
4976  * It'd be better to take stack vma map and limit this more
4977  * precisly, but there's no way to get it safely under interrupt,
4978  * so using TASK_SIZE as limit.
4979  */
4980 static u64 perf_ustack_task_size(struct pt_regs *regs)
4981 {
4982         unsigned long addr = perf_user_stack_pointer(regs);
4983 
4984         if (!addr || addr >= TASK_SIZE)
4985                 return 0;
4986 
4987         return TASK_SIZE - addr;
4988 }
4989 
4990 static u16
4991 perf_sample_ustack_size(u16 stack_size, u16 header_size,
4992                         struct pt_regs *regs)
4993 {
4994         u64 task_size;
4995 
4996         /* No regs, no stack pointer, no dump. */
4997         if (!regs)
4998                 return 0;
4999 
5000         /*
5001          * Check if we fit in with the requested stack size into the:
5002          * - TASK_SIZE
5003          *   If we don't, we limit the size to the TASK_SIZE.
5004          *
5005          * - remaining sample size
5006          *   If we don't, we customize the stack size to
5007          *   fit in to the remaining sample size.
5008          */
5009 
5010         task_size  = min((u64) USHRT_MAX, perf_ustack_task_size(regs));
5011         stack_size = min(stack_size, (u16) task_size);
5012 
5013         /* Current header size plus static size and dynamic size. */
5014         header_size += 2 * sizeof(u64);
5015 
5016         /* Do we fit in with the current stack dump size? */
5017         if ((u16) (header_size + stack_size) < header_size) {
5018                 /*
5019                  * If we overflow the maximum size for the sample,
5020                  * we customize the stack dump size to fit in.
5021                  */
5022                 stack_size = USHRT_MAX - header_size - sizeof(u64);
5023                 stack_size = round_up(stack_size, sizeof(u64));
5024         }
5025 
5026         return stack_size;
5027 }
5028 
5029 static void
5030 perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size,
5031                           struct pt_regs *regs)
5032 {
5033         /* Case of a kernel thread, nothing to dump */
5034         if (!regs) {
5035                 u64 size = 0;
5036                 perf_output_put(handle, size);
5037         } else {
5038                 unsigned long sp;
5039                 unsigned int rem;
5040                 u64 dyn_size;
5041 
5042                 /*
5043                  * We dump:
5044                  * static size
5045                  *   - the size requested by user or the best one we can fit
5046                  *     in to the sample max size
5047                  * data
5048                  *   - user stack dump data
5049                  * dynamic size
5050                  *   - the actual dumped size
5051                  */
5052 
5053                 /* Static size. */
5054                 perf_output_put(handle, dump_size);
5055 
5056                 /* Data. */
5057                 sp = perf_user_stack_pointer(regs);
5058                 rem = __output_copy_user(handle, (void *) sp, dump_size);
5059                 dyn_size = dump_size - rem;
5060 
5061                 perf_output_skip(handle, rem);
5062 
5063                 /* Dynamic size. */
5064                 perf_output_put(handle, dyn_size);
5065         }
5066 }
5067 
5068 static void __perf_event_header__init_id(struct perf_event_header *header,
5069                                          struct perf_sample_data *data,
5070                                          struct perf_event *event)
5071 {
5072         u64 sample_type = event->attr.sample_type;
5073 
5074         data->type = sample_type;
5075         header->size += event->id_header_size;
5076 
5077         if (sample_type & PERF_SAMPLE_TID) {
5078                 /* namespace issues */
5079                 data->tid_entry.pid = perf_event_pid(event, current);
5080                 data->tid_entry.tid = perf_event_tid(event, current);
5081         }
5082 
5083         if (sample_type & PERF_SAMPLE_TIME)
5084                 data->time = perf_event_clock(event);
5085 
5086         if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
5087                 data->id = primary_event_id(event);
5088 
5089         if (sample_type & PERF_SAMPLE_STREAM_ID)
5090                 data->stream_id = event->id;
5091 
5092         if (sample_type & PERF_SAMPLE_CPU) {
5093                 data->cpu_entry.cpu      = raw_smp_processor_id();
5094                 data->cpu_entry.reserved = 0;
5095         }
5096 }
5097 
5098 void perf_event_header__init_id(struct perf_event_header *header,
5099                                 struct perf_sample_data *data,
5100                                 struct perf_event *event)
5101 {
5102         if (event->attr.sample_id_all)
5103                 __perf_event_header__init_id(header, data, event);
5104 }
5105 
5106 static void __perf_event__output_id_sample(struct perf_output_handle *handle,
5107                                            struct perf_sample_data *data)
5108 {
5109         u64 sample_type = data->type;
5110 
5111         if (sample_type & PERF_SAMPLE_TID)
5112                 perf_output_put(handle, data->tid_entry);
5113 
5114         if (sample_type & PERF_SAMPLE_TIME)
5115                 perf_output_put(handle, data->time);
5116 
5117         if (sample_type & PERF_SAMPLE_ID)
5118                 perf_output_put(handle, data->id);
5119 
5120         if (sample_type & PERF_SAMPLE_STREAM_ID)
5121                 perf_output_put(handle, data->stream_id);
5122 
5123         if (sample_type & PERF_SAMPLE_CPU)
5124                 perf_output_put(handle, data->cpu_entry);
5125 
5126         if (sample_type & PERF_SAMPLE_IDENTIFIER)
5127                 perf_output_put(handle, data->id);
5128 }
5129 
5130 void perf_event__output_id_sample(struct perf_event *event,
5131                                   struct perf_output_handle *handle,
5132                                   struct perf_sample_data *sample)
5133 {
5134         if (event->attr.sample_id_all)
5135                 __perf_event__output_id_sample(handle, sample);
5136 }
5137 
5138 static void perf_output_read_one(struct perf_output_handle *handle,
5139                                  struct perf_event *event,
5140                                  u64 enabled, u64 running)
5141 {
5142         u64 read_format = event->attr.read_format;
5143         u64 values[4];
5144         int n = 0;
5145 
5146         values[n++] = perf_event_count(event);
5147         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
5148                 values[n++] = enabled +
5149                         atomic64_read(&event->child_total_time_enabled);
5150         }
5151         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
5152                 values[n++] = running +
5153                         atomic64_read(&event->child_total_time_running);
5154         }
5155         if (read_format & PERF_FORMAT_ID)
5156                 values[n++] = primary_event_id(event);
5157 
5158         __output_copy(handle, values, n * sizeof(u64));
5159 }
5160 
5161 /*
5162  * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
5163  */
5164 static void perf_output_read_group(struct perf_output_handle *handle,
5165                             struct perf_event *event,
5166                             u64 enabled, u64 running)
5167 {
5168         struct perf_event *leader = event->group_leader, *sub;
5169         u64 read_format = event->attr.read_format;
5170         u64 values[5];
5171         int n = 0;
5172 
5173         values[n++] = 1 + leader->nr_siblings;
5174 
5175         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
5176                 values[n++] = enabled;
5177 
5178         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
5179                 values[n++] = running;
5180 
5181         if (leader != event)
5182                 leader->pmu->read(leader);
5183 
5184         values[n++] = perf_event_count(leader);
5185         if (read_format & PERF_FORMAT_ID)
5186                 values[n++] = primary_event_id(leader);
5187 
5188         __output_copy(handle, values, n * sizeof(u64));
5189 
5190         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
5191                 n = 0;
5192 
5193                 if ((sub != event) &&
5194                     (sub->state == PERF_EVENT_STATE_ACTIVE))
5195                         sub->pmu->read(sub);
5196 
5197                 values[n++] = perf_event_count(sub);
5198                 if (read_format & PERF_FORMAT_ID)
5199                         values[n++] = primary_event_id(sub);
5200 
5201                 __output_copy(handle, values, n * sizeof(u64));
5202         }
5203 }
5204 
5205 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
5206                                  PERF_FORMAT_TOTAL_TIME_RUNNING)
5207 
5208 static void perf_output_read(struct perf_output_handle *handle,
5209                              struct perf_event *event)
5210 {
5211         u64 enabled = 0, running = 0, now;
5212         u64 read_format = event->attr.read_format;
5213 
5214         /*
5215          * compute total_time_enabled, total_time_running
5216          * based on snapshot values taken when the event
5217          * was last scheduled in.
5218          *
5219          * we cannot simply called update_context_time()
5220          * because of locking issue as we are called in
5221          * NMI context
5222          */
5223         if (read_format & PERF_FORMAT_TOTAL_TIMES)
5224                 calc_timer_values(event, &now, &enabled, &running);
5225 
5226         if (event->attr.read_format & PERF_FORMAT_GROUP)
5227                 perf_output_read_group(handle, event, enabled, running);
5228         else
5229                 perf_output_read_one(handle, event, enabled, running);
5230 }
5231 
5232 void perf_output_sample(struct perf_output_handle *handle,
5233                         struct perf_event_header *header,
5234                         struct perf_sample_data *data,
5235                         struct perf_event *event)
5236 {
5237         u64 sample_type = data->type;
5238 
5239         perf_output_put(handle, *header);
5240 
5241         if (sample_type & PERF_SAMPLE_IDENTIFIER)
5242                 perf_output_put(handle, data->id);
5243 
5244         if (sample_type & PERF_SAMPLE_IP)
5245                 perf_output_put(handle, data->ip);
5246 
5247         if (sample_type & PERF_SAMPLE_TID)
5248                 perf_output_put(handle, data->tid_entry);
5249 
5250         if (sample_type & PERF_SAMPLE_TIME)
5251                 perf_output_put(handle, data->time);
5252 
5253         if (sample_type & PERF_SAMPLE_ADDR)
5254                 perf_output_put(handle, data->addr);
5255 
5256         if (sample_type & PERF_SAMPLE_ID)
5257                 perf_output_put(handle, data->id);
5258 
5259         if (sample_type & PERF_SAMPLE_STREAM_ID)
5260                 perf_output_put(handle, data->stream_id);
5261 
5262         if (sample_type & PERF_SAMPLE_CPU)
5263                 perf_output_put(handle, data->cpu_entry);
5264 
5265         if (sample_type & PERF_SAMPLE_PERIOD)
5266                 perf_output_put(handle, data->period);
5267 
5268         if (sample_type & PERF_SAMPLE_READ)
5269                 perf_output_read(handle, event);
5270 
5271         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
5272                 if (data->callchain) {
5273                         int size = 1;
5274 
5275                         if (data->callchain)
5276                                 size += data->callchain->nr;
5277 
5278                         size *= sizeof(u64);
5279 
5280                         __output_copy(handle, data->callchain, size);
5281                 } else {
5282                         u64 nr = 0;
5283                         perf_output_put(handle, nr);
5284                 }
5285         }
5286 
5287         if (sample_type & PERF_SAMPLE_RAW) {
5288                 if (data->raw) {
5289                         perf_output_put(handle, data->raw->size);
5290                         __output_copy(handle, data->raw->data,
5291                                            data->raw->size);
5292                 } else {
5293                         struct {
5294                                 u32     size;
5295                                 u32     data;
5296                         } raw = {
5297                                 .size = sizeof(u32),
5298                                 .data = 0,
5299                         };
5300                         perf_output_put(handle, raw);
5301                 }
5302         }
5303 
5304         if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
5305                 if (data->br_stack) {
5306                         size_t size;
5307 
5308                         size = data->br_stack->nr
5309                              * sizeof(struct perf_branch_entry);
5310 
5311                         perf_output_put(handle, data->br_stack->nr);
5312                         perf_output_copy(handle, data->br_stack->entries, size);
5313                 } else {
5314                         /*
5315                          * we always store at least the value of nr
5316                          */
5317                         u64 nr = 0;
5318                         perf_output_put(handle, nr);
5319                 }
5320         }
5321 
5322         if (sample_type & PERF_SAMPLE_REGS_USER) {
5323                 u64 abi = data->regs_user.abi;
5324 
5325                 /*
5326                  * If there are no regs to dump, notice it through
5327                  * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
5328                  */
5329                 perf_output_put(handle, abi);
5330 
5331                 if (abi) {
5332                         u64 mask = event->attr.sample_regs_user;
5333                         perf_output_sample_regs(handle,
5334                                                 data->regs_user.regs,
5335                                                 mask);
5336                 }
5337         }
5338 
5339         if (sample_type & PERF_SAMPLE_STACK_USER) {
5340                 perf_output_sample_ustack(handle,
5341                                           data->stack_user_size,
5342                                           data->regs_user.regs);
5343         }
5344 
5345         if (sample_type & PERF_SAMPLE_WEIGHT)
5346                 perf_output_put(handle, data->weight);
5347 
5348         if (sample_type & PERF_SAMPLE_DATA_SRC)
5349                 perf_output_put(handle, data->data_src.val);
5350 
5351         if (sample_type & PERF_SAMPLE_TRANSACTION)
5352                 perf_output_put(handle, data->txn);
5353 
5354         if (sample_type & PERF_SAMPLE_REGS_INTR) {
5355                 u64 abi = data->regs_intr.abi;
5356                 /*
5357                  * If there are no regs to dump, notice it through
5358                  * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
5359                  */
5360                 perf_output_put(handle, abi);
5361 
5362                 if (abi) {
5363                         u64 mask = event->attr.sample_regs_intr;
5364 
5365                         perf_output_sample_regs(handle,
5366                                                 data->regs_intr.regs,
5367                                                 mask);
5368                 }
5369         }
5370 
5371         if (!event->attr.watermark) {
5372                 int wakeup_events = event->attr.wakeup_events;
5373 
5374                 if (wakeup_events) {
5375                         struct ring_buffer *rb = handle->rb;
5376                         int events = local_inc_return(&rb->events);
5377 
5378                         if (events >= wakeup_events) {
5379                                 local_sub(wakeup_events, &rb->events);
5380                                 local_inc(&rb->wakeup);
5381                         }
5382                 }
5383         }
5384 }
5385 
5386 void perf_prepare_sample(struct perf_event_header *header,
5387                          struct perf_sample_data *data,
5388                          struct perf_event *event,
5389                          struct pt_regs *regs)
5390 {
5391         u64 sample_type = event->attr.sample_type;
5392 
5393         header->type = PERF_RECORD_SAMPLE;
5394         header->size = sizeof(*header) + event->header_size;
5395 
5396         header->misc = 0;
5397         header->misc |= perf_misc_flags(regs);
5398 
5399         __perf_event_header__init_id(header, data, event);
5400 
5401         if (sample_type & PERF_SAMPLE_IP)
5402                 data->ip = perf_instruction_pointer(regs);
5403 
5404         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
5405                 int size = 1;
5406 
5407                 data->callchain = perf_callchain(event, regs);
5408 
5409                 if (data->callchain)
5410                         size += data->callchain->nr;
5411 
5412                 header->size += size * sizeof(u64);
5413         }
5414 
5415         if (sample_type & PERF_SAMPLE_RAW) {
5416                 int size = sizeof(u32);
5417 
5418                 if (data->raw)
5419                         size += data->raw->size;
5420                 else
5421                         size += sizeof(u32);
5422 
5423                 WARN_ON_ONCE(size & (sizeof(u64)-1));
5424                 header->size += size;
5425         }
5426 
5427         if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
5428                 int size = sizeof(u64); /* nr */
5429                 if (data->br_stack) {
5430                         size += data->br_stack->nr
5431                               * sizeof(struct perf_branch_entry);
5432                 }
5433                 header->size += size;
5434         }
5435 
5436         if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER))
5437                 perf_sample_regs_user(&data->regs_user, regs,
5438                                       &data->regs_user_copy);
5439 
5440         if (sample_type & PERF_SAMPLE_REGS_USER) {
5441                 /* regs dump ABI info */
5442                 int size = sizeof(u64);
5443 
5444                 if (data->regs_user.regs) {
5445                         u64 mask = event->attr.sample_regs_user;
5446                         size += hweight64(mask) * sizeof(u64);
5447                 }
5448 
5449                 header->size += size;
5450         }
5451 
5452         if (sample_type & PERF_SAMPLE_STACK_USER) {
5453                 /*
5454                  * Either we need PERF_SAMPLE_STACK_USER bit to be allways
5455                  * processed as the last one or have additional check added
5456                  * in case new sample type is added, because we could eat
5457                  * up the rest of the sample size.
5458                  */
5459                 u16 stack_size = event->attr.sample_stack_user;
5460                 u16 size = sizeof(u64);
5461 
5462                 stack_size = perf_sample_ustack_size(stack_size, header->size,
5463                                                      data->regs_user.regs);
5464 
5465                 /*
5466                  * If there is something to dump, add space for the dump
5467                  * itself and for the field that tells the dynamic size,
5468                  * which is how many have been actually dumped.
5469                  */
5470                 if (stack_size)
5471                         size += sizeof(u64) + stack_size;
5472 
5473                 data->stack_user_size = stack_size;
5474                 header->size += size;
5475         }
5476 
5477         if (sample_type & PERF_SAMPLE_REGS_INTR) {
5478                 /* regs dump ABI info */
5479                 int size = sizeof(u64);
5480 
5481                 perf_sample_regs_intr(&data->regs_intr, regs);
5482 
5483                 if (data->regs_intr.regs) {
5484                         u64 mask = event->attr.sample_regs_intr;
5485 
5486                         size += hweight64(mask) * sizeof(u64);
5487                 }
5488 
5489                 header->size += size;
5490         }
5491 }
5492 
5493 void perf_event_output(struct perf_event *event,
5494                         struct perf_sample_data *data,
5495                         struct pt_regs *regs)
5496 {
5497         struct perf_output_handle handle;
5498         struct perf_event_header header;
5499 
5500         /* protect the callchain buffers */
5501         rcu_read_lock();
5502 
5503         perf_prepare_sample(&header, data, event, regs);
5504 
5505         if (perf_output_begin(&handle, event, header.size))
5506                 goto exit;
5507 
5508         perf_output_sample(&handle, &header, data, event);
5509 
5510         perf_output_end(&handle);
5511 
5512 exit:
5513         rcu_read_unlock();
5514 }
5515 
5516 /*
5517  * read event_id
5518  */
5519 
5520 struct perf_read_event {
5521         struct perf_event_header        header;
5522 
5523         u32                             pid;
5524         u32                             tid;
5525 };
5526 
5527 static void
5528 perf_event_read_event(struct perf_event *event,
5529                         struct task_struct *task)
5530 {
5531         struct perf_output_handle handle;
5532         struct perf_sample_data sample;
5533         struct perf_read_event read_event = {
5534                 .header = {
5535                         .type = PERF_RECORD_READ,
5536                         .misc = 0,
5537                         .size = sizeof(read_event) + event->read_size,
5538                 },
5539                 .pid = perf_event_pid(event, task),
5540                 .tid = perf_event_tid(event, task),
5541         };
5542         int ret;
5543 
5544         perf_event_header__init_id(&read_event.header, &sample, event);
5545         ret = perf_output_begin(&handle, event, read_event.header.size);
5546         if (ret)
5547                 return;
5548 
5549         perf_output_put(&handle, read_event);
5550         perf_output_read(&handle, event);
5551         perf_event__output_id_sample(event, &handle, &sample);
5552 
5553         perf_output_end(&handle);
5554 }
5555 
5556 typedef void (perf_event_aux_output_cb)(struct perf_event *event, void *data);
5557 
5558 static void
5559 perf_event_aux_ctx(struct perf_event_context *ctx,
5560                    perf_event_aux_output_cb output,
5561                    void *data)
5562 {
5563         struct perf_event *event;
5564 
5565         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
5566                 if (event->state < PERF_EVENT_STATE_INACTIVE)
5567                         continue;
5568                 if (!event_filter_match(event))
5569                         continue;
5570                 output(event, data);
5571         }
5572 }
5573 
5574 static void
5575 perf_event_aux(perf_event_aux_output_cb output, void *data,
5576                struct perf_event_context *task_ctx)
5577 {
5578         struct perf_cpu_context *cpuctx;
5579         struct perf_event_context *ctx;
5580         struct pmu *pmu;
5581         int ctxn;
5582 
5583         rcu_read_lock();
5584         list_for_each_entry_rcu(pmu, &pmus, entry) {
5585                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
5586                 if (cpuctx->unique_pmu != pmu)
5587                         goto next;
5588                 perf_event_aux_ctx(&cpuctx->ctx, output, data);
5589                 if (task_ctx)
5590                         goto next;
5591                 ctxn = pmu->task_ctx_nr;
5592                 if (ctxn < 0)
5593                         goto next;
5594                 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
5595                 if (ctx)
5596                         perf_event_aux_ctx(ctx, output, data);
5597 next:
5598                 put_cpu_ptr(pmu->pmu_cpu_context);
5599         }
5600 
5601         if (task_ctx) {
5602                 preempt_disable();
5603                 perf_event_aux_ctx(task_ctx, output, data);
5604                 preempt_enable();
5605         }
5606         rcu_read_unlock();
5607 }
5608 
5609 /*
5610  * task tracking -- fork/exit
5611  *
5612  * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task
5613  */
5614 
5615 struct perf_task_event {
5616         struct task_struct              *task;
5617         struct perf_event_context       *task_ctx;
5618 
5619         struct {
5620                 struct perf_event_header        header;
5621 
5622                 u32                             pid;
5623                 u32                             ppid;
5624                 u32                             tid;
5625                 u32                             ptid;
5626                 u64                             time;
5627         } event_id;
5628 };
5629 
5630 static int perf_event_task_match(struct perf_event *event)
5631 {
5632         return event->attr.comm  || event->attr.mmap ||
5633                event->attr.mmap2 || event->attr.mmap_data ||
5634                event->attr.task;
5635 }
5636 
5637 static void perf_event_task_output(struct perf_event *event,
5638                                    void *data)
5639 {
5640         struct perf_task_event *task_event = data;
5641         struct perf_output_handle handle;
5642         struct perf_sample_data sample;
5643         struct task_struct *task = task_event->task;
5644         int ret, size = task_event->event_id.header.size;
5645 
5646         if (!perf_event_task_match(event))
5647                 return;
5648 
5649         perf_event_header__init_id(&task_event->event_id.header, &sample, event);
5650 
5651         ret = perf_output_begin(&handle, event,
5652                                 task_event->event_id.header.size);
5653         if (ret)
5654                 goto out;
5655 
5656         task_event->event_id.pid = perf_event_pid(event, task);
5657         task_event->event_id.ppid = perf_event_pid(event, current);
5658 
5659         task_event->event_id.tid = perf_event_tid(event, task);
5660         task_event->event_id.ptid = perf_event_tid(event, current);
5661 
5662         task_event->event_id.time = perf_event_clock(event);
5663 
5664         perf_output_put(&handle, task_event->event_id);
5665 
5666         perf_event__output_id_sample(event, &handle, &sample);
5667 
5668         perf_output_end(&handle);
5669 out:
5670         task_event->event_id.header.size = size;
5671 }
5672 
5673 static void perf_event_task(struct task_struct *task,
5674                               struct perf_event_context *task_ctx,
5675                               int new)
5676 {
5677         struct perf_task_event task_event;
5678 
5679         if (!atomic_read(&nr_comm_events) &&
5680             !atomic_read(&nr_mmap_events) &&
5681             !atomic_read(&nr_task_events))
5682                 return;
5683 
5684         task_event = (struct perf_task_event){
5685                 .task     = task,
5686                 .task_ctx = task_ctx,
5687                 .event_id    = {
5688                         .header = {
5689                                 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
5690                                 .misc = 0,
5691                                 .size = sizeof(task_event.event_id),
5692                         },
5693                         /* .pid  */
5694                         /* .ppid */
5695                         /* .tid  */
5696                         /* .ptid */
5697                         /* .time */
5698                 },
5699         };
5700 
5701         perf_event_aux(perf_event_task_output,
5702                        &task_event,
5703                        task_ctx);
5704 }
5705 
5706 void perf_event_fork(struct task_struct *task)
5707 {
5708         perf_event_task(task, NULL, 1);
5709 }
5710 
5711 /*
5712  * comm tracking
5713  */
5714 
5715 struct perf_comm_event {
5716         struct task_struct      *task;
5717         char                    *comm;
5718         int                     comm_size;
5719 
5720         struct {
5721                 struct perf_event_header        header;
5722 
5723                 u32                             pid;
5724                 u32                             tid;
5725         } event_id;
5726 };
5727 
5728 static int perf_event_comm_match(struct perf_event *event)
5729 {
5730         return event->attr.comm;
5731 }
5732 
5733 static void perf_event_comm_output(struct perf_event *event,
5734                                    void *data)
5735 {
5736         struct perf_comm_event *comm_event = data;
5737         struct perf_output_handle handle;
5738         struct perf_sample_data sample;
5739         int size = comm_event->event_id.header.size;
5740         int ret;
5741 
5742         if (!perf_event_comm_match(event))
5743                 return;
5744 
5745         perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
5746         ret = perf_output_begin(&handle, event,
5747                                 comm_event->event_id.header.size);
5748 
5749         if (ret)
5750                 goto out;
5751 
5752         comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
5753         comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
5754 
5755         perf_output_put(&handle, comm_event->event_id);
5756         __output_copy(&handle, comm_event->comm,
5757                                    comm_event->comm_size);
5758 
5759         perf_event__output_id_sample(event, &handle, &sample);
5760 
5761         perf_output_end(&handle);
5762 out:
5763         comm_event->event_id.header.size = size;
5764 }
5765 
5766 static void perf_event_comm_event(struct perf_comm_event *comm_event)
5767 {
5768         char comm[TASK_COMM_LEN];
5769         unsigned int size;
5770 
5771         memset(comm, 0, sizeof(comm));
5772         strlcpy(comm, comm_event->task->comm, sizeof(comm));
5773         size = ALIGN(strlen(comm)+1, sizeof(u64));
5774 
5775         comm_event->comm = comm;
5776         comm_event->comm_size = size;
5777 
5778         comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
5779 
5780         perf_event_aux(perf_event_comm_output,
5781                        comm_event,
5782                        NULL);
5783 }
5784 
5785 void perf_event_comm(struct task_struct *task, bool exec)
5786 {
5787         struct perf_comm_event comm_event;
5788 
5789         if (!atomic_read(&nr_comm_events))
5790                 return;
5791 
5792         comm_event = (struct perf_comm_event){
5793                 .task   = task,
5794                 /* .comm      */
5795                 /* .comm_size */
5796                 .event_id  = {
5797                         .header = {
5798                                 .type = PERF_RECORD_COMM,
5799                                 .misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0,
5800                                 /* .size */
5801                         },
5802                         /* .pid */
5803                         /* .tid */
5804                 },
5805         };
5806 
5807         perf_event_comm_event(&comm_event);
5808 }
5809 
5810 /*
5811  * mmap tracking
5812  */
5813 
5814 struct perf_mmap_event {
5815         struct vm_area_struct   *vma;
5816 
5817         const char              *file_name;
5818         int                     file_size;
5819         int                     maj, min;
5820         u64                     ino;
5821         u64                     ino_generation;
5822         u32                     prot, flags;
5823 
5824         struct {
5825                 struct perf_event_header        header;
5826 
5827                 u32                             pid;
5828                 u32                             tid;
5829                 u64                             start;
5830                 u64                             len;
5831                 u64                             pgoff;
5832         } event_id;
5833 };
5834 
5835 static int perf_event_mmap_match(struct perf_event *event,
5836                                  void *data)
5837 {
5838         struct perf_mmap_event *mmap_event = data;
5839         struct vm_area_struct *vma = mmap_event->vma;
5840         int executable = vma->vm_flags & VM_EXEC;
5841 
5842         return (!executable && event->attr.mmap_data) ||
5843                (executable && (event->attr.mmap || event->attr.mmap2));
5844 }
5845 
5846 static void perf_event_mmap_output(struct perf_event *event,
5847                                    void *data)
5848 {
5849         struct perf_mmap_event *mmap_event = data;
5850         struct perf_output_handle handle;
5851         struct perf_sample_data sample;
5852         int size = mmap_event->event_id.header.size;
5853         int ret;
5854 
5855         if (!perf_event_mmap_match(event, data))
5856                 return;
5857 
5858         if (event->attr.mmap2) {
5859                 mmap_event->event_id.header.type = PERF_RECORD_MMAP2;
5860                 mmap_event->event_id.header.size += sizeof(mmap_event->maj);
5861                 mmap_event->event_id.header.size += sizeof(mmap_event->min);
5862                 mmap_event->event_id.header.size += sizeof(mmap_event->ino);
5863                 mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation);
5864                 mmap_event->event_id.header.size += sizeof(mmap_event->prot);
5865                 mmap_event->event_id.header.size += sizeof(mmap_event->flags);
5866         }
5867 
5868         perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
5869         ret = perf_output_begin(&handle, event,
5870                                 mmap_event->event_id.header.size);
5871         if (ret)
5872                 goto out;
5873 
5874         mmap_event->event_id.pid = perf_event_pid(event, current);
5875         mmap_event->event_id.tid = perf_event_tid(event, current);
5876 
5877         perf_output_put(&handle, mmap_event->event_id);
5878 
5879         if (event->attr.mmap2) {
5880                 perf_output_put(&handle, mmap_event->maj);
5881                 perf_output_put(&handle, mmap_event->min);
5882                 perf_output_put(&handle, mmap_event->ino);
5883                 perf_output_put(&handle, mmap_event->ino_generation);
5884                 perf_output_put(&handle, mmap_event->prot);
5885                 perf_output_put(&handle, mmap_event->flags);
5886         }
5887 
5888         __output_copy(&handle, mmap_event->file_name,
5889                                    mmap_event->file_size);
5890 
5891         perf_event__output_id_sample(event, &handle, &sample);
5892 
5893         perf_output_end(&handle);
5894 out:
5895         mmap_event->event_id.header.size = size;
5896 }
5897 
5898 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
5899 {
5900         struct vm_area_struct *vma = mmap_event->vma;
5901         struct file *file = vma->vm_file;
5902         int maj = 0, min = 0;
5903         u64 ino = 0, gen = 0;
5904         u32 prot = 0, flags = 0;
5905         unsigned int size;
5906         char tmp[16];
5907         char *buf = NULL;
5908         char *name;
5909 
5910         if (file) {
5911                 struct inode *inode;
5912                 dev_t dev;
5913 
5914                 buf = kmalloc(PATH_MAX, GFP_KERNEL);
5915                 if (!buf) {
5916                         name = "//enomem";
5917                         goto cpy_name;
5918                 }
5919                 /*
5920                  * d_path() works from the end of the rb backwards, so we
5921                  * need to add enough zero bytes after the string to handle
5922                  * the 64bit alignment we do later.
5923                  */
5924                 name = file_path(file, buf, PATH_MAX - sizeof(u64));
5925                 if (IS_ERR(name)) {
5926                         name = "//toolong";
5927                         goto cpy_name;
5928                 }
5929                 inode = file_inode(vma->vm_file);
5930                 dev = inode->i_sb->s_dev;
5931                 ino = inode->i_ino;
5932                 gen = inode->i_generation;
5933                 maj = MAJOR(dev);
5934                 min = MINOR(dev);
5935 
5936                 if (vma->vm_flags & VM_READ)
5937                         prot |= PROT_READ;
5938                 if (vma->vm_flags & VM_WRITE)
5939                         prot |= PROT_WRITE;
5940                 if (vma->vm_flags & VM_EXEC)
5941                         prot |= PROT_EXEC;
5942 
5943                 if (vma->vm_flags & VM_MAYSHARE)
5944                         flags = MAP_SHARED;
5945                 else
5946                         flags = MAP_PRIVATE;
5947 
5948                 if (vma->vm_flags & VM_DENYWRITE)
5949                         flags |= MAP_DENYWRITE;
5950                 if (vma->vm_flags & VM_MAYEXEC)
5951                         flags |= MAP_EXECUTABLE;
5952                 if (vma->vm_flags & VM_LOCKED)
5953                         flags |= MAP_LOCKED;
5954                 if (vma->vm_flags & VM_HUGETLB)
5955                         flags |= MAP_HUGETLB;
5956 
5957                 goto got_name;
5958         } else {
5959                 if (vma->vm_ops && vma->vm_ops->name) {
5960                         name = (char *) vma->vm_ops->name(vma);
5961                         if (name)
5962                                 goto cpy_name;
5963                 }
5964 
5965                 name = (char *)arch_vma_name(vma);
5966                 if (name)
5967                         goto cpy_name;
5968 
5969                 if (vma->vm_start <= vma->vm_mm->start_brk &&
5970                                 vma->vm_end >= vma->vm_mm->brk) {
5971                         name = "[heap]";
5972                         goto cpy_name;
5973                 }
5974                 if (vma->vm_start <= vma->vm_mm->start_stack &&
5975                                 vma->vm_end >= vma->vm_mm->start_stack) {
5976                         name = "[stack]";
5977                         goto cpy_name;
5978                 }
5979 
5980                 name = "//anon";
5981                 goto cpy_name;
5982         }
5983 
5984 cpy_name:
5985         strlcpy(tmp, name, sizeof(tmp));
5986         name = tmp;
5987 got_name:
5988         /*
5989          * Since our buffer works in 8 byte units we need to align our string
5990          * size to a multiple of 8. However, we must guarantee the tail end is
5991          * zero'd out to avoid leaking random bits to userspace.
5992          */
5993         size = strlen(name)+1;
5994         while (!IS_ALIGNED(size, sizeof(u64)))
5995                 name[size++] = '\0';
5996 
5997         mmap_event->file_name = name;
5998         mmap_event->file_size = size;
5999         mmap_event->maj = maj;
6000         mmap_event->min = min;
6001         mmap_event->ino = ino;
6002         mmap_event->ino_generation = gen;
6003         mmap_event->prot = prot;
6004         mmap_event->flags = flags;
6005 
6006         if (!(vma->vm_flags & VM_EXEC))
6007                 mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;
6008 
6009         mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
6010 
6011         perf_event_aux(perf_event_mmap_output,
6012                        mmap_event,
6013                        NULL);
6014 
6015         kfree(buf);
6016 }
6017 
6018 void perf_event_mmap(struct vm_area_struct *vma)
6019 {
6020         struct perf_mmap_event mmap_event;
6021 
6022         if (!atomic_read(&nr_mmap_events))
6023                 return;
6024 
6025         mmap_event = (struct perf_mmap_event){
6026                 .vma    = vma,
6027                 /* .file_name */
6028                 /* .file_size */
6029                 .event_id  = {
6030                         .header = {
6031                                 .type = PERF_RECORD_MMAP,
6032                                 .misc = PERF_RECORD_MISC_USER,
6033                                 /* .size */
6034                         },
6035                         /* .pid */
6036                         /* .tid */
6037                         .start  = vma->vm_start,
6038                         .len    = vma->vm_end - vma->vm_start,
6039                         .pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
6040                 },
6041                 /* .maj (attr_mmap2 only) */
6042                 /* .min (attr_mmap2 only) */
6043                 /* .ino (attr_mmap2 only) */
6044                 /* .ino_generation (attr_mmap2 only) */
6045                 /* .prot (attr_mmap2 only) */
6046                 /* .flags (attr_mmap2 only) */
6047         };
6048 
6049         perf_event_mmap_event(&mmap_event);
6050 }
6051 
6052 void perf_event_aux_event(struct perf_event *event, unsigned long head,
6053                           unsigned long size, u64 flags)
6054 {
6055         struct perf_output_handle handle;
6056         struct perf_sample_data sample;
6057         struct perf_aux_event {
6058                 struct perf_event_header        header;
6059                 u64                             offset;
6060                 u64                             size;
6061                 u64                             flags;
6062         } rec = {
6063                 .header = {
6064                         .type = PERF_RECORD_AUX,
6065                         .misc = 0,
6066                         .size = sizeof(rec),
6067                 },
6068                 .offset         = head,
6069                 .size           = size,
6070                 .flags          = flags,
6071         };
6072         int ret;
6073 
6074         perf_event_header__init_id(&rec.header, &sample, event);
6075         ret = perf_output_begin(&handle, event, rec.header.size);
6076 
6077         if (ret)
6078                 return;
6079 
6080         perf_output_put(&handle, rec);
6081         perf_event__output_id_sample(event, &handle, &sample);
6082 
6083         perf_output_end(&handle);
6084 }
6085 
6086 /*
6087  * Lost/dropped samples logging
6088  */
6089 void perf_log_lost_samples(struct perf_event *event, u64 lost)
6090 {
6091         struct perf_output_handle handle;
6092         struct perf_sample_data sample;
6093         int ret;
6094 
6095         struct {
6096                 struct perf_event_header        header;
6097                 u64                             lost;
6098         } lost_samples_event = {
6099                 .header = {
6100                         .type = PERF_RECORD_LOST_SAMPLES,
6101                         .misc = 0,
6102                         .size = sizeof(lost_samples_event),
6103                 },
6104                 .lost           = lost,
6105         };
6106 
6107         perf_event_header__init_id(&lost_samples_event.header, &sample, event);
6108 
6109         ret = perf_output_begin(&handle, event,
6110                                 lost_samples_event.header.size);
6111         if (ret)
6112                 return;
6113 
6114         perf_output_put(&handle, lost_samples_event);
6115         perf_event__output_id_sample(event, &handle, &sample);
6116         perf_output_end(&handle);
6117 }
6118 
6119 /*
6120  * context_switch tracking
6121  */
6122 
6123 struct perf_switch_event {
6124         struct task_struct      *task;
6125         struct task_struct      *next_prev;
6126 
6127         struct {
6128                 struct perf_event_header        header;
6129                 u32                             next_prev_pid;
6130                 u32                             next_prev_tid;
6131         } event_id;
6132 };
6133 
6134 static int perf_event_switch_match(struct perf_event *event)
6135 {
6136         return event->attr.context_switch;
6137 }
6138 
6139 static void perf_event_switch_output(struct perf_event *event, void *data)
6140 {
6141         struct perf_switch_event *se = data;
6142         struct perf_output_handle handle;
6143         struct perf_sample_data sample;
6144         int ret;
6145 
6146         if (!perf_event_switch_match(event))
6147                 return;
6148 
6149         /* Only CPU-wide events are allowed to see next/prev pid/tid */
6150         if (event->ctx->task) {
6151                 se->event_id.header.type = PERF_RECORD_SWITCH;
6152                 se->event_id.header.size = sizeof(se->event_id.header);
6153         } else {
6154                 se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE;
6155                 se->event_id.header.size = sizeof(se->event_id);
6156                 se->event_id.next_prev_pid =
6157                                         perf_event_pid(event, se->next_prev);
6158                 se->event_id.next_prev_tid =
6159                                         perf_event_tid(event, se->next_prev);
6160         }
6161 
6162         perf_event_header__init_id(&se->event_id.header, &sample, event);
6163 
6164         ret = perf_output_begin(&handle, event, se->event_id.header.size);
6165         if (ret)
6166                 return;
6167 
6168         if (event->ctx->task)
6169                 perf_output_put(&handle, se->event_id.header);
6170         else
6171                 perf_output_put(&handle, se->event_id);
6172 
6173         perf_event__output_id_sample(event, &handle, &sample);
6174 
6175         perf_output_end(&handle);
6176 }
6177 
6178 static void perf_event_switch(struct task_struct *task,
6179                               struct task_struct *next_prev, bool sched_in)
6180 {
6181         struct perf_switch_event switch_event;
6182 
6183         /* N.B. caller checks nr_switch_events != 0 */
6184 
6185         switch_event = (struct perf_switch_event){
6186                 .task           = task,
6187                 .next_prev      = next_prev,
6188                 .event_id       = {
6189                         .header = {
6190                                 /* .type */
6191                                 .misc = sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT,
6192                                 /* .size */
6193                         },
6194                         /* .next_prev_pid */
6195                         /* .next_prev_tid */
6196                 },
6197         };
6198 
6199         perf_event_aux(perf_event_switch_output,
6200                        &switch_event,
6201                        NULL);
6202 }
6203 
6204 /*
6205  * IRQ throttle logging
6206  */
6207 
6208 static void perf_log_throttle(struct perf_event *event, int enable)
6209 {
6210         struct perf_output_handle handle;
6211         struct perf_sample_data sample;
6212         int ret;
6213 
6214         struct {
6215                 struct perf_event_header        header;
6216                 u64                             time;
6217                 u64                             id;
6218                 u64                             stream_id;
6219         } throttle_event = {
6220                 .header = {
6221                         .type = PERF_RECORD_THROTTLE,
6222                         .misc = 0,
6223                         .size = sizeof(throttle_event),
6224                 },
6225                 .time           = perf_event_clock(event),
6226                 .id             = primary_event_id(event),
6227                 .stream_id      = event->id,
6228         };
6229 
6230         if (enable)
6231                 throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
6232 
6233         perf_event_header__init_id(&throttle_event.header, &sample, event);
6234 
6235         ret = perf_output_begin(&handle, event,
6236                                 throttle_event.header.size);
6237         if (ret)
6238                 return;
6239 
6240         perf_output_put(&handle, throttle_event);
6241         perf_event__output_id_sample(event, &handle, &sample);
6242         perf_output_end(&handle);
6243 }
6244 
6245 static void perf_log_itrace_start(struct perf_event *event)
6246 {
6247         struct perf_output_handle handle;
6248         struct perf_sample_data sample;
6249         struct perf_aux_event {
6250                 struct perf_event_header        header;
6251                 u32                             pid;
6252                 u32                             tid;
6253         } rec;
6254         int ret;
6255 
6256         if (event->parent)
6257                 event = event->parent;
6258 
6259         if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) ||
6260             event->hw.itrace_started)
6261                 return;
6262 
6263         rec.header.type = PERF_RECORD_ITRACE_START;
6264         rec.header.misc = 0;
6265         rec.header.size = sizeof(rec);
6266         rec.pid = perf_event_pid(event, current);
6267         rec.tid = perf_event_tid(event, current);
6268 
6269         perf_event_header__init_id(&rec.header, &sample, event);
6270         ret = perf_output_begin(&handle, event, rec.header.size);
6271 
6272         if (ret)
6273                 return;
6274 
6275         perf_output_put(&handle, rec);
6276         perf_event__output_id_sample(event, &handle, &sample);
6277 
6278         perf_output_end(&handle);
6279 }
6280 
6281 /*
6282  * Generic event overflow handling, sampling.
6283  */
6284 
6285 static int __perf_event_overflow(struct perf_event *event,
6286                                    int throttle, struct perf_sample_data *data,
6287                                    struct pt_regs *regs)
6288 {
6289         int events = atomic_read(&event->event_limit);
6290         struct hw_perf_event *hwc = &event->hw;
6291         u64 seq;
6292         int ret = 0;
6293 
6294         /*
6295          * Non-sampling counters might still use the PMI to fold short
6296          * hardware counters, ignore those.
6297          */
6298         if (unlikely(!is_sampling_event(event)))
6299                 return 0;
6300 
6301         seq = __this_cpu_read(perf_throttled_seq);
6302         if (seq != hwc->interrupts_seq) {
6303                 hwc->interrupts_seq = seq;
6304                 hwc->interrupts = 1;
6305         } else {
6306                 hwc->interrupts++;
6307                 if (unlikely(throttle
6308                              && hwc->interrupts >= max_samples_per_tick)) {
6309                         __this_cpu_inc(perf_throttled_count);
6310                         hwc->interrupts = MAX_INTERRUPTS;
6311                         perf_log_throttle(event, 0);
6312                         tick_nohz_full_kick();
6313                         ret = 1;
6314                 }
6315         }
6316 
6317         if (event->attr.freq) {
6318                 u64 now = perf_clock();
6319                 s64 delta = now - hwc->freq_time_stamp;
6320 
6321                 hwc->freq_time_stamp = now;
6322 
6323                 if (delta > 0 && delta < 2*TICK_NSEC)
6324                         perf_adjust_period(event, delta, hwc->last_period, true);
6325         }
6326 
6327         /*
6328          * XXX event_limit might not quite work as expected on inherited
6329          * events
6330          */
6331 
6332         event->pending_kill = POLL_IN;
6333         if (events && atomic_dec_and_test(&event->event_limit)) {
6334                 ret = 1;
6335                 event->pending_kill = POLL_HUP;
6336                 event->pending_disable = 1;
6337                 irq_work_queue(&event->pending);
6338         }
6339 
6340         if (event->overflow_handler)
6341                 event->overflow_handler(event, data, regs);
6342         else
6343                 perf_event_output(event, data, regs);
6344 
6345         if (*perf_event_fasync(event) && event->pending_kill) {
6346                 event->pending_wakeup = 1;
6347                 irq_work_queue(&event->pending);
6348         }
6349 
6350         return ret;
6351 }
6352 
6353 int perf_event_overflow(struct perf_event *event,
6354                           struct perf_sample_data *data,
6355                           struct pt_regs *regs)
6356 {
6357         return __perf_event_overflow(event, 1, data, regs);
6358 }
6359 
6360 /*
6361  * Generic software event infrastructure
6362  */
6363 
6364 struct swevent_htable {
6365         struct swevent_hlist            *swevent_hlist;
6366         struct mutex                    hlist_mutex;
6367         int                             hlist_refcount;
6368 
6369         /* Recursion avoidance in each contexts */
6370         int                             recursion[PERF_NR_CONTEXTS];
6371 
6372         /* Keeps track of cpu being initialized/exited */
6373         bool                            online;
6374 };
6375 
6376 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
6377 
6378 /*
6379  * We directly increment event->count and keep a second value in
6380  * event->hw.period_left to count intervals. This period event
6381  * is kept in the range [-sample_period, 0] so that we can use the
6382  * sign as trigger.
6383  */
6384 
6385 u64 perf_swevent_set_period(struct perf_event *event)
6386 {
6387         struct hw_perf_event *hwc = &event->hw;
6388         u64 period = hwc->last_period;
6389         u64 nr, offset;
6390         s64 old, val;
6391 
6392         hwc->last_period = hwc->sample_period;
6393 
6394 again:
6395         old = val = local64_read(&hwc->period_left);
6396         if (val < 0)
6397                 return 0;
6398 
6399         nr = div64_u64(period + val, period);
6400         offset = nr * period;
6401         val -= offset;
6402         if (local64_cmpxchg(&hwc->period_left, old, val) != old)
6403                 goto again;
6404 
6405         return nr;
6406 }
6407 
6408 static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
6409                                     struct perf_sample_data *data,
6410                                     struct pt_regs *regs)
6411 {
6412         struct hw_perf_event *hwc = &event->hw;
6413         int throttle = 0;
6414 
6415         if (!overflow)
6416                 overflow = perf_swevent_set_period(event);
6417 
6418         if (hwc->interrupts == MAX_INTERRUPTS)
6419                 return;
6420 
6421         for (; overflow; overflow--) {
6422                 if (__perf_event_overflow(event, throttle,
6423                                             data, regs)) {
6424                         /*
6425                          * We inhibit the overflow from happening when
6426                          * hwc->interrupts == MAX_INTERRUPTS.
6427                          */
6428                         break;
6429                 }
6430                 throttle = 1;
6431         }
6432 }
6433 
6434 static void perf_swevent_event(struct perf_event *event, u64 nr,
6435                                struct perf_sample_data *data,
6436                                struct pt_regs *regs)
6437 {
6438         struct hw_perf_event *hwc = &event->hw;
6439 
6440         local64_add(nr, &event->count);
6441 
6442         if (!regs)
6443                 return;
6444 
6445         if (!is_sampling_event(event))
6446                 return;
6447 
6448         if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
6449                 data->period = nr;
6450                 return perf_swevent_overflow(event, 1, data, regs);
6451         } else
6452                 data->period = event->hw.last_period;
6453 
6454         if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
6455                 return perf_swevent_overflow(event, 1, data, regs);
6456 
6457         if (local64_add_negative(nr, &hwc->period_left))
6458                 return;
6459 
6460         perf_swevent_overflow(event, 0, data, regs);
6461 }
6462 
6463 static int perf_exclude_event(struct perf_event *event,
6464                               struct pt_regs *regs)
6465 {
6466         if (event->hw.state & PERF_HES_STOPPED)
6467                 return 1;
6468 
6469         if (regs) {
6470                 if (event->attr.exclude_user && user_mode(regs))
6471                         return 1;
6472 
6473                 if (event->attr.exclude_kernel && !user_mode(regs))
6474                         return 1;
6475         }
6476 
6477         return 0;
6478 }
6479 
6480 static int perf_swevent_match(struct perf_event *event,
6481                                 enum perf_type_id type,
6482                                 u32 event_id,
6483                                 struct perf_sample_data *data,
6484                                 struct pt_regs *regs)
6485 {
6486         if (event->attr.type != type)
6487                 return 0;
6488 
6489         if (event->attr.config != event_id)
6490                 return 0;
6491 
6492         if (perf_exclude_event(event, regs))
6493                 return 0;
6494 
6495         return 1;
6496 }
6497 
6498 static inline u64 swevent_hash(u64 type, u32 event_id)
6499 {
6500         u64 val = event_id | (type << 32);
6501 
6502         return hash_64(val, SWEVENT_HLIST_BITS);
6503 }
6504 
6505 static inline struct hlist_head *
6506 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
6507 {
6508         u64 hash = swevent_hash(type, event_id);
6509 
6510         return &hlist->heads[hash];
6511 }
6512 
6513 /* For the read side: events when they trigger */
6514 static inline struct hlist_head *
6515 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
6516 {
6517         struct swevent_hlist *hlist;
6518 
6519         hlist = rcu_dereference(swhash->swevent_hlist);
6520         if (!hlist)
6521                 return NULL;
6522 
6523         return __find_swevent_head(hlist, type, event_id);
6524 }
6525 
6526 /* For the event head insertion and removal in the hlist */
6527 static inline struct hlist_head *
6528 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
6529 {
6530         struct swevent_hlist *hlist;
6531         u32 event_id = event->attr.config;
6532         u64 type = event->attr.type;
6533 
6534         /*
6535          * Event scheduling is always serialized against hlist allocation
6536          * and release. Which makes the protected version suitable here.
6537          * The context lock guarantees that.
6538          */
6539         hlist = rcu_dereference_protected(swhash->swevent_hlist,
6540                                           lockdep_is_held(&event->ctx->lock));
6541         if (!hlist)
6542                 return NULL;
6543 
6544         return __find_swevent_head(hlist, type, event_id);
6545 }
6546 
6547 static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
6548                                     u64 nr,
6549                                     struct perf_sample_data *data,
6550                                     struct pt_regs *regs)
6551 {
6552         struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
6553         struct perf_event *event;
6554         struct hlist_head *head;
6555 
6556         rcu_read_lock();
6557         head = find_swevent_head_rcu(swhash, type, event_id);
6558         if (!head)
6559                 goto end;
6560 
6561         hlist_for_each_entry_rcu(event, head, hlist_entry) {
6562                 if (perf_swevent_match(event, type, event_id, data, regs))
6563                         perf_swevent_event(event, nr, data, regs);
6564         }
6565 end:
6566         rcu_read_unlock();
6567 }
6568 
6569 DEFINE_PER_CPU(struct pt_regs, __perf_regs[4]);
6570 
6571 int perf_swevent_get_recursion_context(void)
6572 {
6573         struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
6574 
6575         return get_recursion_context(swhash->recursion);
6576 }
6577 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
6578 
6579 inline void perf_swevent_put_recursion_context(int rctx)
6580 {
6581         struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
6582 
6583         put_recursion_context(swhash->recursion, rctx);
6584 }
6585 
6586 void ___perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
6587 {
6588         struct perf_sample_data data;
6589 
6590         if (WARN_ON_ONCE(!regs))
6591                 return;
6592 
6593         perf_sample_data_init(&data, addr, 0);
6594         do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
6595 }
6596