~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/events/core.c

Version: ~ [ linux-5.15-rc1 ] ~ [ linux-5.14.5 ] ~ [ linux-5.13.18 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.66 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.147 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.206 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.246 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.282 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.283 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Performance events core code:
  3  *
  4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  5  *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
  6  *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
  7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  8  *
  9  * For licensing details see kernel-base/COPYING
 10  */
 11 
 12 #include <linux/fs.h>
 13 #include <linux/mm.h>
 14 #include <linux/cpu.h>
 15 #include <linux/smp.h>
 16 #include <linux/idr.h>
 17 #include <linux/file.h>
 18 #include <linux/poll.h>
 19 #include <linux/slab.h>
 20 #include <linux/hash.h>
 21 #include <linux/tick.h>
 22 #include <linux/sysfs.h>
 23 #include <linux/dcache.h>
 24 #include <linux/percpu.h>
 25 #include <linux/ptrace.h>
 26 #include <linux/reboot.h>
 27 #include <linux/vmstat.h>
 28 #include <linux/device.h>
 29 #include <linux/export.h>
 30 #include <linux/vmalloc.h>
 31 #include <linux/hardirq.h>
 32 #include <linux/rculist.h>
 33 #include <linux/uaccess.h>
 34 #include <linux/syscalls.h>
 35 #include <linux/anon_inodes.h>
 36 #include <linux/kernel_stat.h>
 37 #include <linux/cgroup.h>
 38 #include <linux/perf_event.h>
 39 #include <linux/trace_events.h>
 40 #include <linux/hw_breakpoint.h>
 41 #include <linux/mm_types.h>
 42 #include <linux/module.h>
 43 #include <linux/mman.h>
 44 #include <linux/compat.h>
 45 #include <linux/bpf.h>
 46 #include <linux/filter.h>
 47 #include <linux/namei.h>
 48 #include <linux/parser.h>
 49 
 50 #include "internal.h"
 51 
 52 #include <asm/irq_regs.h>
 53 
 54 typedef int (*remote_function_f)(void *);
 55 
 56 struct remote_function_call {
 57         struct task_struct      *p;
 58         remote_function_f       func;
 59         void                    *info;
 60         int                     ret;
 61 };
 62 
 63 static void remote_function(void *data)
 64 {
 65         struct remote_function_call *tfc = data;
 66         struct task_struct *p = tfc->p;
 67 
 68         if (p) {
 69                 /* -EAGAIN */
 70                 if (task_cpu(p) != smp_processor_id())
 71                         return;
 72 
 73                 /*
 74                  * Now that we're on right CPU with IRQs disabled, we can test
 75                  * if we hit the right task without races.
 76                  */
 77 
 78                 tfc->ret = -ESRCH; /* No such (running) process */
 79                 if (p != current)
 80                         return;
 81         }
 82 
 83         tfc->ret = tfc->func(tfc->info);
 84 }
 85 
 86 /**
 87  * task_function_call - call a function on the cpu on which a task runs
 88  * @p:          the task to evaluate
 89  * @func:       the function to be called
 90  * @info:       the function call argument
 91  *
 92  * Calls the function @func when the task is currently running. This might
 93  * be on the current CPU, which just calls the function directly
 94  *
 95  * returns: @func return value, or
 96  *          -ESRCH  - when the process isn't running
 97  *          -EAGAIN - when the process moved away
 98  */
 99 static int
100 task_function_call(struct task_struct *p, remote_function_f func, void *info)
101 {
102         struct remote_function_call data = {
103                 .p      = p,
104                 .func   = func,
105                 .info   = info,
106                 .ret    = -EAGAIN,
107         };
108         int ret;
109 
110         do {
111                 ret = smp_call_function_single(task_cpu(p), remote_function, &data, 1);
112                 if (!ret)
113                         ret = data.ret;
114         } while (ret == -EAGAIN);
115 
116         return ret;
117 }
118 
119 /**
120  * cpu_function_call - call a function on the cpu
121  * @func:       the function to be called
122  * @info:       the function call argument
123  *
124  * Calls the function @func on the remote cpu.
125  *
126  * returns: @func return value or -ENXIO when the cpu is offline
127  */
128 static int cpu_function_call(int cpu, remote_function_f func, void *info)
129 {
130         struct remote_function_call data = {
131                 .p      = NULL,
132                 .func   = func,
133                 .info   = info,
134                 .ret    = -ENXIO, /* No such CPU */
135         };
136 
137         smp_call_function_single(cpu, remote_function, &data, 1);
138 
139         return data.ret;
140 }
141 
142 static inline struct perf_cpu_context *
143 __get_cpu_context(struct perf_event_context *ctx)
144 {
145         return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
146 }
147 
148 static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
149                           struct perf_event_context *ctx)
150 {
151         raw_spin_lock(&cpuctx->ctx.lock);
152         if (ctx)
153                 raw_spin_lock(&ctx->lock);
154 }
155 
156 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
157                             struct perf_event_context *ctx)
158 {
159         if (ctx)
160                 raw_spin_unlock(&ctx->lock);
161         raw_spin_unlock(&cpuctx->ctx.lock);
162 }
163 
164 #define TASK_TOMBSTONE ((void *)-1L)
165 
166 static bool is_kernel_event(struct perf_event *event)
167 {
168         return READ_ONCE(event->owner) == TASK_TOMBSTONE;
169 }
170 
171 /*
172  * On task ctx scheduling...
173  *
174  * When !ctx->nr_events a task context will not be scheduled. This means
175  * we can disable the scheduler hooks (for performance) without leaving
176  * pending task ctx state.
177  *
178  * This however results in two special cases:
179  *
180  *  - removing the last event from a task ctx; this is relatively straight
181  *    forward and is done in __perf_remove_from_context.
182  *
183  *  - adding the first event to a task ctx; this is tricky because we cannot
184  *    rely on ctx->is_active and therefore cannot use event_function_call().
185  *    See perf_install_in_context().
186  *
187  * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set.
188  */
189 
190 typedef void (*event_f)(struct perf_event *, struct perf_cpu_context *,
191                         struct perf_event_context *, void *);
192 
193 struct event_function_struct {
194         struct perf_event *event;
195         event_f func;
196         void *data;
197 };
198 
199 static int event_function(void *info)
200 {
201         struct event_function_struct *efs = info;
202         struct perf_event *event = efs->event;
203         struct perf_event_context *ctx = event->ctx;
204         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
205         struct perf_event_context *task_ctx = cpuctx->task_ctx;
206         int ret = 0;
207 
208         WARN_ON_ONCE(!irqs_disabled());
209 
210         perf_ctx_lock(cpuctx, task_ctx);
211         /*
212          * Since we do the IPI call without holding ctx->lock things can have
213          * changed, double check we hit the task we set out to hit.
214          */
215         if (ctx->task) {
216                 if (ctx->task != current) {
217                         ret = -ESRCH;
218                         goto unlock;
219                 }
220 
221                 /*
222                  * We only use event_function_call() on established contexts,
223                  * and event_function() is only ever called when active (or
224                  * rather, we'll have bailed in task_function_call() or the
225                  * above ctx->task != current test), therefore we must have
226                  * ctx->is_active here.
227                  */
228                 WARN_ON_ONCE(!ctx->is_active);
229                 /*
230                  * And since we have ctx->is_active, cpuctx->task_ctx must
231                  * match.
232                  */
233                 WARN_ON_ONCE(task_ctx != ctx);
234         } else {
235                 WARN_ON_ONCE(&cpuctx->ctx != ctx);
236         }
237 
238         efs->func(event, cpuctx, ctx, efs->data);
239 unlock:
240         perf_ctx_unlock(cpuctx, task_ctx);
241 
242         return ret;
243 }
244 
245 static void event_function_call(struct perf_event *event, event_f func, void *data)
246 {
247         struct perf_event_context *ctx = event->ctx;
248         struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */
249         struct event_function_struct efs = {
250                 .event = event,
251                 .func = func,
252                 .data = data,
253         };
254 
255         if (!event->parent) {
256                 /*
257                  * If this is a !child event, we must hold ctx::mutex to
258                  * stabilize the the event->ctx relation. See
259                  * perf_event_ctx_lock().
260                  */
261                 lockdep_assert_held(&ctx->mutex);
262         }
263 
264         if (!task) {
265                 cpu_function_call(event->cpu, event_function, &efs);
266                 return;
267         }
268 
269         if (task == TASK_TOMBSTONE)
270                 return;
271 
272 again:
273         if (!task_function_call(task, event_function, &efs))
274                 return;
275 
276         raw_spin_lock_irq(&ctx->lock);
277         /*
278          * Reload the task pointer, it might have been changed by
279          * a concurrent perf_event_context_sched_out().
280          */
281         task = ctx->task;
282         if (task == TASK_TOMBSTONE) {
283                 raw_spin_unlock_irq(&ctx->lock);
284                 return;
285         }
286         if (ctx->is_active) {
287                 raw_spin_unlock_irq(&ctx->lock);
288                 goto again;
289         }
290         func(event, NULL, ctx, data);
291         raw_spin_unlock_irq(&ctx->lock);
292 }
293 
294 /*
295  * Similar to event_function_call() + event_function(), but hard assumes IRQs
296  * are already disabled and we're on the right CPU.
297  */
298 static void event_function_local(struct perf_event *event, event_f func, void *data)
299 {
300         struct perf_event_context *ctx = event->ctx;
301         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
302         struct task_struct *task = READ_ONCE(ctx->task);
303         struct perf_event_context *task_ctx = NULL;
304 
305         WARN_ON_ONCE(!irqs_disabled());
306 
307         if (task) {
308                 if (task == TASK_TOMBSTONE)
309                         return;
310 
311                 task_ctx = ctx;
312         }
313 
314         perf_ctx_lock(cpuctx, task_ctx);
315 
316         task = ctx->task;
317         if (task == TASK_TOMBSTONE)
318                 goto unlock;
319 
320         if (task) {
321                 /*
322                  * We must be either inactive or active and the right task,
323                  * otherwise we're screwed, since we cannot IPI to somewhere
324                  * else.
325                  */
326                 if (ctx->is_active) {
327                         if (WARN_ON_ONCE(task != current))
328                                 goto unlock;
329 
330                         if (WARN_ON_ONCE(cpuctx->task_ctx != ctx))
331                                 goto unlock;
332                 }
333         } else {
334                 WARN_ON_ONCE(&cpuctx->ctx != ctx);
335         }
336 
337         func(event, cpuctx, ctx, data);
338 unlock:
339         perf_ctx_unlock(cpuctx, task_ctx);
340 }
341 
342 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
343                        PERF_FLAG_FD_OUTPUT  |\
344                        PERF_FLAG_PID_CGROUP |\
345                        PERF_FLAG_FD_CLOEXEC)
346 
347 /*
348  * branch priv levels that need permission checks
349  */
350 #define PERF_SAMPLE_BRANCH_PERM_PLM \
351         (PERF_SAMPLE_BRANCH_KERNEL |\
352          PERF_SAMPLE_BRANCH_HV)
353 
354 enum event_type_t {
355         EVENT_FLEXIBLE = 0x1,
356         EVENT_PINNED = 0x2,
357         EVENT_TIME = 0x4,
358         EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
359 };
360 
361 /*
362  * perf_sched_events : >0 events exist
363  * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
364  */
365 
366 static void perf_sched_delayed(struct work_struct *work);
367 DEFINE_STATIC_KEY_FALSE(perf_sched_events);
368 static DECLARE_DELAYED_WORK(perf_sched_work, perf_sched_delayed);
369 static DEFINE_MUTEX(perf_sched_mutex);
370 static atomic_t perf_sched_count;
371 
372 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
373 static DEFINE_PER_CPU(int, perf_sched_cb_usages);
374 static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events);
375 
376 static atomic_t nr_mmap_events __read_mostly;
377 static atomic_t nr_comm_events __read_mostly;
378 static atomic_t nr_task_events __read_mostly;
379 static atomic_t nr_freq_events __read_mostly;
380 static atomic_t nr_switch_events __read_mostly;
381 
382 static LIST_HEAD(pmus);
383 static DEFINE_MUTEX(pmus_lock);
384 static struct srcu_struct pmus_srcu;
385 
386 /*
387  * perf event paranoia level:
388  *  -1 - not paranoid at all
389  *   0 - disallow raw tracepoint access for unpriv
390  *   1 - disallow cpu events for unpriv
391  *   2 - disallow kernel profiling for unpriv
392  */
393 int sysctl_perf_event_paranoid __read_mostly = 2;
394 
395 /* Minimum for 512 kiB + 1 user control page */
396 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
397 
398 /*
399  * max perf event sample rate
400  */
401 #define DEFAULT_MAX_SAMPLE_RATE         100000
402 #define DEFAULT_SAMPLE_PERIOD_NS        (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)
403 #define DEFAULT_CPU_TIME_MAX_PERCENT    25
404 
405 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
406 
407 static int max_samples_per_tick __read_mostly   = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
408 static int perf_sample_period_ns __read_mostly  = DEFAULT_SAMPLE_PERIOD_NS;
409 
410 static int perf_sample_allowed_ns __read_mostly =
411         DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100;
412 
413 static void update_perf_cpu_limits(void)
414 {
415         u64 tmp = perf_sample_period_ns;
416 
417         tmp *= sysctl_perf_cpu_time_max_percent;
418         tmp = div_u64(tmp, 100);
419         if (!tmp)
420                 tmp = 1;
421 
422         WRITE_ONCE(perf_sample_allowed_ns, tmp);
423 }
424 
425 static int perf_rotate_context(struct perf_cpu_context *cpuctx);
426 
427 int perf_proc_update_handler(struct ctl_table *table, int write,
428                 void __user *buffer, size_t *lenp,
429                 loff_t *ppos)
430 {
431         int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
432 
433         if (ret || !write)
434                 return ret;
435 
436         /*
437          * If throttling is disabled don't allow the write:
438          */
439         if (sysctl_perf_cpu_time_max_percent == 100 ||
440             sysctl_perf_cpu_time_max_percent == 0)
441                 return -EINVAL;
442 
443         max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
444         perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
445         update_perf_cpu_limits();
446 
447         return 0;
448 }
449 
450 int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT;
451 
452 int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
453                                 void __user *buffer, size_t *lenp,
454                                 loff_t *ppos)
455 {
456         int ret = proc_dointvec(table, write, buffer, lenp, ppos);
457 
458         if (ret || !write)
459                 return ret;
460 
461         if (sysctl_perf_cpu_time_max_percent == 100 ||
462             sysctl_perf_cpu_time_max_percent == 0) {
463                 printk(KERN_WARNING
464                        "perf: Dynamic interrupt throttling disabled, can hang your system!\n");
465                 WRITE_ONCE(perf_sample_allowed_ns, 0);
466         } else {
467                 update_perf_cpu_limits();
468         }
469 
470         return 0;
471 }
472 
473 /*
474  * perf samples are done in some very critical code paths (NMIs).
475  * If they take too much CPU time, the system can lock up and not
476  * get any real work done.  This will drop the sample rate when
477  * we detect that events are taking too long.
478  */
479 #define NR_ACCUMULATED_SAMPLES 128
480 static DEFINE_PER_CPU(u64, running_sample_length);
481 
482 static u64 __report_avg;
483 static u64 __report_allowed;
484 
485 static void perf_duration_warn(struct irq_work *w)
486 {
487         printk_ratelimited(KERN_INFO
488                 "perf: interrupt took too long (%lld > %lld), lowering "
489                 "kernel.perf_event_max_sample_rate to %d\n",
490                 __report_avg, __report_allowed,
491                 sysctl_perf_event_sample_rate);
492 }
493 
494 static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn);
495 
496 void perf_sample_event_took(u64 sample_len_ns)
497 {
498         u64 max_len = READ_ONCE(perf_sample_allowed_ns);
499         u64 running_len;
500         u64 avg_len;
501         u32 max;
502 
503         if (max_len == 0)
504                 return;
505 
506         /* Decay the counter by 1 average sample. */
507         running_len = __this_cpu_read(running_sample_length);
508         running_len -= running_len/NR_ACCUMULATED_SAMPLES;
509         running_len += sample_len_ns;
510         __this_cpu_write(running_sample_length, running_len);
511 
512         /*
513          * Note: this will be biased artifically low until we have
514          * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us
515          * from having to maintain a count.
516          */
517         avg_len = running_len/NR_ACCUMULATED_SAMPLES;
518         if (avg_len <= max_len)
519                 return;
520 
521         __report_avg = avg_len;
522         __report_allowed = max_len;
523 
524         /*
525          * Compute a throttle threshold 25% below the current duration.
526          */
527         avg_len += avg_len / 4;
528         max = (TICK_NSEC / 100) * sysctl_perf_cpu_time_max_percent;
529         if (avg_len < max)
530                 max /= (u32)avg_len;
531         else
532                 max = 1;
533 
534         WRITE_ONCE(perf_sample_allowed_ns, avg_len);
535         WRITE_ONCE(max_samples_per_tick, max);
536 
537         sysctl_perf_event_sample_rate = max * HZ;
538         perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
539 
540         if (!irq_work_queue(&perf_duration_work)) {
541                 early_printk("perf: interrupt took too long (%lld > %lld), lowering "
542                              "kernel.perf_event_max_sample_rate to %d\n",
543                              __report_avg, __report_allowed,
544                              sysctl_perf_event_sample_rate);
545         }
546 }
547 
548 static atomic64_t perf_event_id;
549 
550 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
551                               enum event_type_t event_type);
552 
553 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
554                              enum event_type_t event_type,
555                              struct task_struct *task);
556 
557 static void update_context_time(struct perf_event_context *ctx);
558 static u64 perf_event_time(struct perf_event *event);
559 
560 void __weak perf_event_print_debug(void)        { }
561 
562 extern __weak const char *perf_pmu_name(void)
563 {
564         return "pmu";
565 }
566 
567 static inline u64 perf_clock(void)
568 {
569         return local_clock();
570 }
571 
572 static inline u64 perf_event_clock(struct perf_event *event)
573 {
574         return event->clock();
575 }
576 
577 #ifdef CONFIG_CGROUP_PERF
578 
579 static inline bool
580 perf_cgroup_match(struct perf_event *event)
581 {
582         struct perf_event_context *ctx = event->ctx;
583         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
584 
585         /* @event doesn't care about cgroup */
586         if (!event->cgrp)
587                 return true;
588 
589         /* wants specific cgroup scope but @cpuctx isn't associated with any */
590         if (!cpuctx->cgrp)
591                 return false;
592 
593         /*
594          * Cgroup scoping is recursive.  An event enabled for a cgroup is
595          * also enabled for all its descendant cgroups.  If @cpuctx's
596          * cgroup is a descendant of @event's (the test covers identity
597          * case), it's a match.
598          */
599         return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,
600                                     event->cgrp->css.cgroup);
601 }
602 
603 static inline void perf_detach_cgroup(struct perf_event *event)
604 {
605         css_put(&event->cgrp->css);
606         event->cgrp = NULL;
607 }
608 
609 static inline int is_cgroup_event(struct perf_event *event)
610 {
611         return event->cgrp != NULL;
612 }
613 
614 static inline u64 perf_cgroup_event_time(struct perf_event *event)
615 {
616         struct perf_cgroup_info *t;
617 
618         t = per_cpu_ptr(event->cgrp->info, event->cpu);
619         return t->time;
620 }
621 
622 static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
623 {
624         struct perf_cgroup_info *info;
625         u64 now;
626 
627         now = perf_clock();
628 
629         info = this_cpu_ptr(cgrp->info);
630 
631         info->time += now - info->timestamp;
632         info->timestamp = now;
633 }
634 
635 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
636 {
637         struct perf_cgroup *cgrp_out = cpuctx->cgrp;
638         if (cgrp_out)
639                 __update_cgrp_time(cgrp_out);
640 }
641 
642 static inline void update_cgrp_time_from_event(struct perf_event *event)
643 {
644         struct perf_cgroup *cgrp;
645 
646         /*
647          * ensure we access cgroup data only when needed and
648          * when we know the cgroup is pinned (css_get)
649          */
650         if (!is_cgroup_event(event))
651                 return;
652 
653         cgrp = perf_cgroup_from_task(current, event->ctx);
654         /*
655          * Do not update time when cgroup is not active
656          */
657         if (cgrp == event->cgrp)
658                 __update_cgrp_time(event->cgrp);
659 }
660 
661 static inline void
662 perf_cgroup_set_timestamp(struct task_struct *task,
663                           struct perf_event_context *ctx)
664 {
665         struct perf_cgroup *cgrp;
666         struct perf_cgroup_info *info;
667 
668         /*
669          * ctx->lock held by caller
670          * ensure we do not access cgroup data
671          * unless we have the cgroup pinned (css_get)
672          */
673         if (!task || !ctx->nr_cgroups)
674                 return;
675 
676         cgrp = perf_cgroup_from_task(task, ctx);
677         info = this_cpu_ptr(cgrp->info);
678         info->timestamp = ctx->timestamp;
679 }
680 
681 #define PERF_CGROUP_SWOUT       0x1 /* cgroup switch out every event */
682 #define PERF_CGROUP_SWIN        0x2 /* cgroup switch in events based on task */
683 
684 /*
685  * reschedule events based on the cgroup constraint of task.
686  *
687  * mode SWOUT : schedule out everything
688  * mode SWIN : schedule in based on cgroup for next
689  */
690 static void perf_cgroup_switch(struct task_struct *task, int mode)
691 {
692         struct perf_cpu_context *cpuctx;
693         struct pmu *pmu;
694         unsigned long flags;
695 
696         /*
697          * disable interrupts to avoid geting nr_cgroup
698          * changes via __perf_event_disable(). Also
699          * avoids preemption.
700          */
701         local_irq_save(flags);
702 
703         /*
704          * we reschedule only in the presence of cgroup
705          * constrained events.
706          */
707 
708         list_for_each_entry_rcu(pmu, &pmus, entry) {
709                 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
710                 if (cpuctx->unique_pmu != pmu)
711                         continue; /* ensure we process each cpuctx once */
712 
713                 /*
714                  * perf_cgroup_events says at least one
715                  * context on this CPU has cgroup events.
716                  *
717                  * ctx->nr_cgroups reports the number of cgroup
718                  * events for a context.
719                  */
720                 if (cpuctx->ctx.nr_cgroups > 0) {
721                         perf_ctx_lock(cpuctx, cpuctx->task_ctx);
722                         perf_pmu_disable(cpuctx->ctx.pmu);
723 
724                         if (mode & PERF_CGROUP_SWOUT) {
725                                 cpu_ctx_sched_out(cpuctx, EVENT_ALL);
726                                 /*
727                                  * must not be done before ctxswout due
728                                  * to event_filter_match() in event_sched_out()
729                                  */
730                                 cpuctx->cgrp = NULL;
731                         }
732 
733                         if (mode & PERF_CGROUP_SWIN) {
734                                 WARN_ON_ONCE(cpuctx->cgrp);
735                                 /*
736                                  * set cgrp before ctxsw in to allow
737                                  * event_filter_match() to not have to pass
738                                  * task around
739                                  * we pass the cpuctx->ctx to perf_cgroup_from_task()
740                                  * because cgorup events are only per-cpu
741                                  */
742                                 cpuctx->cgrp = perf_cgroup_from_task(task, &cpuctx->ctx);
743                                 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
744                         }
745                         perf_pmu_enable(cpuctx->ctx.pmu);
746                         perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
747                 }
748         }
749 
750         local_irq_restore(flags);
751 }
752 
753 static inline void perf_cgroup_sched_out(struct task_struct *task,
754                                          struct task_struct *next)
755 {
756         struct perf_cgroup *cgrp1;
757         struct perf_cgroup *cgrp2 = NULL;
758 
759         rcu_read_lock();
760         /*
761          * we come here when we know perf_cgroup_events > 0
762          * we do not need to pass the ctx here because we know
763          * we are holding the rcu lock
764          */
765         cgrp1 = perf_cgroup_from_task(task, NULL);
766         cgrp2 = perf_cgroup_from_task(next, NULL);
767 
768         /*
769          * only schedule out current cgroup events if we know
770          * that we are switching to a different cgroup. Otherwise,
771          * do no touch the cgroup events.
772          */
773         if (cgrp1 != cgrp2)
774                 perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
775 
776         rcu_read_unlock();
777 }
778 
779 static inline void perf_cgroup_sched_in(struct task_struct *prev,
780                                         struct task_struct *task)
781 {
782         struct perf_cgroup *cgrp1;
783         struct perf_cgroup *cgrp2 = NULL;
784 
785         rcu_read_lock();
786         /*
787          * we come here when we know perf_cgroup_events > 0
788          * we do not need to pass the ctx here because we know
789          * we are holding the rcu lock
790          */
791         cgrp1 = perf_cgroup_from_task(task, NULL);
792         cgrp2 = perf_cgroup_from_task(prev, NULL);
793 
794         /*
795          * only need to schedule in cgroup events if we are changing
796          * cgroup during ctxsw. Cgroup events were not scheduled
797          * out of ctxsw out if that was not the case.
798          */
799         if (cgrp1 != cgrp2)
800                 perf_cgroup_switch(task, PERF_CGROUP_SWIN);
801 
802         rcu_read_unlock();
803 }
804 
805 static inline int perf_cgroup_connect(int fd, struct perf_event *event,
806                                       struct perf_event_attr *attr,
807                                       struct perf_event *group_leader)
808 {
809         struct perf_cgroup *cgrp;
810         struct cgroup_subsys_state *css;
811         struct fd f = fdget(fd);
812         int ret = 0;
813 
814         if (!f.file)
815                 return -EBADF;
816 
817         css = css_tryget_online_from_dir(f.file->f_path.dentry,
818                                          &perf_event_cgrp_subsys);
819         if (IS_ERR(css)) {
820                 ret = PTR_ERR(css);
821                 goto out;
822         }
823 
824         cgrp = container_of(css, struct perf_cgroup, css);
825         event->cgrp = cgrp;
826 
827         /*
828          * all events in a group must monitor
829          * the same cgroup because a task belongs
830          * to only one perf cgroup at a time
831          */
832         if (group_leader && group_leader->cgrp != cgrp) {
833                 perf_detach_cgroup(event);
834                 ret = -EINVAL;
835         }
836 out:
837         fdput(f);
838         return ret;
839 }
840 
841 static inline void
842 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
843 {
844         struct perf_cgroup_info *t;
845         t = per_cpu_ptr(event->cgrp->info, event->cpu);
846         event->shadow_ctx_time = now - t->timestamp;
847 }
848 
849 static inline void
850 perf_cgroup_defer_enabled(struct perf_event *event)
851 {
852         /*
853          * when the current task's perf cgroup does not match
854          * the event's, we need to remember to call the
855          * perf_mark_enable() function the first time a task with
856          * a matching perf cgroup is scheduled in.
857          */
858         if (is_cgroup_event(event) && !perf_cgroup_match(event))
859                 event->cgrp_defer_enabled = 1;
860 }
861 
862 static inline void
863 perf_cgroup_mark_enabled(struct perf_event *event,
864                          struct perf_event_context *ctx)
865 {
866         struct perf_event *sub;
867         u64 tstamp = perf_event_time(event);
868 
869         if (!event->cgrp_defer_enabled)
870                 return;
871 
872         event->cgrp_defer_enabled = 0;
873 
874         event->tstamp_enabled = tstamp - event->total_time_enabled;
875         list_for_each_entry(sub, &event->sibling_list, group_entry) {
876                 if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
877                         sub->tstamp_enabled = tstamp - sub->total_time_enabled;
878                         sub->cgrp_defer_enabled = 0;
879                 }
880         }
881 }
882 
883 /*
884  * Update cpuctx->cgrp so that it is set when first cgroup event is added and
885  * cleared when last cgroup event is removed.
886  */
887 static inline void
888 list_update_cgroup_event(struct perf_event *event,
889                          struct perf_event_context *ctx, bool add)
890 {
891         struct perf_cpu_context *cpuctx;
892 
893         if (!is_cgroup_event(event))
894                 return;
895 
896         if (add && ctx->nr_cgroups++)
897                 return;
898         else if (!add && --ctx->nr_cgroups)
899                 return;
900         /*
901          * Because cgroup events are always per-cpu events,
902          * this will always be called from the right CPU.
903          */
904         cpuctx = __get_cpu_context(ctx);
905         cpuctx->cgrp = add ? event->cgrp : NULL;
906 }
907 
908 #else /* !CONFIG_CGROUP_PERF */
909 
910 static inline bool
911 perf_cgroup_match(struct perf_event *event)
912 {
913         return true;
914 }
915 
916 static inline void perf_detach_cgroup(struct perf_event *event)
917 {}
918 
919 static inline int is_cgroup_event(struct perf_event *event)
920 {
921         return 0;
922 }
923 
924 static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
925 {
926         return 0;
927 }
928 
929 static inline void update_cgrp_time_from_event(struct perf_event *event)
930 {
931 }
932 
933 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
934 {
935 }
936 
937 static inline void perf_cgroup_sched_out(struct task_struct *task,
938                                          struct task_struct *next)
939 {
940 }
941 
942 static inline void perf_cgroup_sched_in(struct task_struct *prev,
943                                         struct task_struct *task)
944 {
945 }
946 
947 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
948                                       struct perf_event_attr *attr,
949                                       struct perf_event *group_leader)
950 {
951         return -EINVAL;
952 }
953 
954 static inline void
955 perf_cgroup_set_timestamp(struct task_struct *task,
956                           struct perf_event_context *ctx)
957 {
958 }
959 
960 void
961 perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
962 {
963 }
964 
965 static inline void
966 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
967 {
968 }
969 
970 static inline u64 perf_cgroup_event_time(struct perf_event *event)
971 {
972         return 0;
973 }
974 
975 static inline void
976 perf_cgroup_defer_enabled(struct perf_event *event)
977 {
978 }
979 
980 static inline void
981 perf_cgroup_mark_enabled(struct perf_event *event,
982                          struct perf_event_context *ctx)
983 {
984 }
985 
986 static inline void
987 list_update_cgroup_event(struct perf_event *event,
988                          struct perf_event_context *ctx, bool add)
989 {
990 }
991 
992 #endif
993 
994 /*
995  * set default to be dependent on timer tick just
996  * like original code
997  */
998 #define PERF_CPU_HRTIMER (1000 / HZ)
999 /*
1000  * function must be called with interrupts disbled
1001  */
1002 static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr)
1003 {
1004         struct perf_cpu_context *cpuctx;
1005         int rotations = 0;
1006 
1007         WARN_ON(!irqs_disabled());
1008 
1009         cpuctx = container_of(hr, struct perf_cpu_context, hrtimer);
1010         rotations = perf_rotate_context(cpuctx);
1011 
1012         raw_spin_lock(&cpuctx->hrtimer_lock);
1013         if (rotations)
1014                 hrtimer_forward_now(hr, cpuctx->hrtimer_interval);
1015         else
1016                 cpuctx->hrtimer_active = 0;
1017         raw_spin_unlock(&cpuctx->hrtimer_lock);
1018 
1019         return rotations ? HRTIMER_RESTART : HRTIMER_NORESTART;
1020 }
1021 
1022 static void __perf_mux_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu)
1023 {
1024         struct hrtimer *timer = &cpuctx->hrtimer;
1025         struct pmu *pmu = cpuctx->ctx.pmu;
1026         u64 interval;
1027 
1028         /* no multiplexing needed for SW PMU */
1029         if (pmu->task_ctx_nr == perf_sw_context)
1030                 return;
1031 
1032         /*
1033          * check default is sane, if not set then force to
1034          * default interval (1/tick)
1035          */
1036         interval = pmu->hrtimer_interval_ms;
1037         if (interval < 1)
1038                 interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;
1039 
1040         cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);
1041 
1042         raw_spin_lock_init(&cpuctx->hrtimer_lock);
1043         hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
1044         timer->function = perf_mux_hrtimer_handler;
1045 }
1046 
1047 static int perf_mux_hrtimer_restart(struct perf_cpu_context *cpuctx)
1048 {
1049         struct hrtimer *timer = &cpuctx->hrtimer;
1050         struct pmu *pmu = cpuctx->ctx.pmu;
1051         unsigned long flags;
1052 
1053         /* not for SW PMU */
1054         if (pmu->task_ctx_nr == perf_sw_context)
1055                 return 0;
1056 
1057         raw_spin_lock_irqsave(&cpuctx->hrtimer_lock, flags);
1058         if (!cpuctx->hrtimer_active) {
1059                 cpuctx->hrtimer_active = 1;
1060                 hrtimer_forward_now(timer, cpuctx->hrtimer_interval);
1061                 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED);
1062         }
1063         raw_spin_unlock_irqrestore(&cpuctx->hrtimer_lock, flags);
1064 
1065         return 0;
1066 }
1067 
1068 void perf_pmu_disable(struct pmu *pmu)
1069 {
1070         int *count = this_cpu_ptr(pmu->pmu_disable_count);
1071         if (!(*count)++)
1072                 pmu->pmu_disable(pmu);
1073 }
1074 
1075 void perf_pmu_enable(struct pmu *pmu)
1076 {
1077         int *count = this_cpu_ptr(pmu->pmu_disable_count);
1078         if (!--(*count))
1079                 pmu->pmu_enable(pmu);
1080 }
1081 
1082 static DEFINE_PER_CPU(struct list_head, active_ctx_list);
1083 
1084 /*
1085  * perf_event_ctx_activate(), perf_event_ctx_deactivate(), and
1086  * perf_event_task_tick() are fully serialized because they're strictly cpu
1087  * affine and perf_event_ctx{activate,deactivate} are called with IRQs
1088  * disabled, while perf_event_task_tick is called from IRQ context.
1089  */
1090 static void perf_event_ctx_activate(struct perf_event_context *ctx)
1091 {
1092         struct list_head *head = this_cpu_ptr(&active_ctx_list);
1093 
1094         WARN_ON(!irqs_disabled());
1095 
1096         WARN_ON(!list_empty(&ctx->active_ctx_list));
1097 
1098         list_add(&ctx->active_ctx_list, head);
1099 }
1100 
1101 static void perf_event_ctx_deactivate(struct perf_event_context *ctx)
1102 {
1103         WARN_ON(!irqs_disabled());
1104 
1105         WARN_ON(list_empty(&ctx->active_ctx_list));
1106 
1107         list_del_init(&ctx->active_ctx_list);
1108 }
1109 
1110 static void get_ctx(struct perf_event_context *ctx)
1111 {
1112         WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
1113 }
1114 
1115 static void free_ctx(struct rcu_head *head)
1116 {
1117         struct perf_event_context *ctx;
1118 
1119         ctx = container_of(head, struct perf_event_context, rcu_head);
1120         kfree(ctx->task_ctx_data);
1121         kfree(ctx);
1122 }
1123 
1124 static void put_ctx(struct perf_event_context *ctx)
1125 {
1126         if (atomic_dec_and_test(&ctx->refcount)) {
1127                 if (ctx->parent_ctx)
1128                         put_ctx(ctx->parent_ctx);
1129                 if (ctx->task && ctx->task != TASK_TOMBSTONE)
1130                         put_task_struct(ctx->task);
1131                 call_rcu(&ctx->rcu_head, free_ctx);
1132         }
1133 }
1134 
1135 /*
1136  * Because of perf_event::ctx migration in sys_perf_event_open::move_group and
1137  * perf_pmu_migrate_context() we need some magic.
1138  *
1139  * Those places that change perf_event::ctx will hold both
1140  * perf_event_ctx::mutex of the 'old' and 'new' ctx value.
1141  *
1142  * Lock ordering is by mutex address. There are two other sites where
1143  * perf_event_context::mutex nests and those are:
1144  *
1145  *  - perf_event_exit_task_context()    [ child , 0 ]
1146  *      perf_event_exit_event()
1147  *        put_event()                   [ parent, 1 ]
1148  *
1149  *  - perf_event_init_context()         [ parent, 0 ]
1150  *      inherit_task_group()
1151  *        inherit_group()
1152  *          inherit_event()
1153  *            perf_event_alloc()
1154  *              perf_init_event()
1155  *                perf_try_init_event() [ child , 1 ]
1156  *
1157  * While it appears there is an obvious deadlock here -- the parent and child
1158  * nesting levels are inverted between the two. This is in fact safe because
1159  * life-time rules separate them. That is an exiting task cannot fork, and a
1160  * spawning task cannot (yet) exit.
1161  *
1162  * But remember that that these are parent<->child context relations, and
1163  * migration does not affect children, therefore these two orderings should not
1164  * interact.
1165  *
1166  * The change in perf_event::ctx does not affect children (as claimed above)
1167  * because the sys_perf_event_open() case will install a new event and break
1168  * the ctx parent<->child relation, and perf_pmu_migrate_context() is only
1169  * concerned with cpuctx and that doesn't have children.
1170  *
1171  * The places that change perf_event::ctx will issue:
1172  *
1173  *   perf_remove_from_context();
1174  *   synchronize_rcu();
1175  *   perf_install_in_context();
1176  *
1177  * to affect the change. The remove_from_context() + synchronize_rcu() should
1178  * quiesce the event, after which we can install it in the new location. This
1179  * means that only external vectors (perf_fops, prctl) can perturb the event
1180  * while in transit. Therefore all such accessors should also acquire
1181  * perf_event_context::mutex to serialize against this.
1182  *
1183  * However; because event->ctx can change while we're waiting to acquire
1184  * ctx->mutex we must be careful and use the below perf_event_ctx_lock()
1185  * function.
1186  *
1187  * Lock order:
1188  *    cred_guard_mutex
1189  *      task_struct::perf_event_mutex
1190  *        perf_event_context::mutex
1191  *          perf_event::child_mutex;
1192  *            perf_event_context::lock
1193  *          perf_event::mmap_mutex
1194  *          mmap_sem
1195  */
1196 static struct perf_event_context *
1197 perf_event_ctx_lock_nested(struct perf_event *event, int nesting)
1198 {
1199         struct perf_event_context *ctx;
1200 
1201 again:
1202         rcu_read_lock();
1203         ctx = ACCESS_ONCE(event->ctx);
1204         if (!atomic_inc_not_zero(&ctx->refcount)) {
1205                 rcu_read_unlock();
1206                 goto again;
1207         }
1208         rcu_read_unlock();
1209 
1210         mutex_lock_nested(&ctx->mutex, nesting);
1211         if (event->ctx != ctx) {
1212                 mutex_unlock(&ctx->mutex);
1213                 put_ctx(ctx);
1214                 goto again;
1215         }
1216 
1217         return ctx;
1218 }
1219 
1220 static inline struct perf_event_context *
1221 perf_event_ctx_lock(struct perf_event *event)
1222 {
1223         return perf_event_ctx_lock_nested(event, 0);
1224 }
1225 
1226 static void perf_event_ctx_unlock(struct perf_event *event,
1227                                   struct perf_event_context *ctx)
1228 {
1229         mutex_unlock(&ctx->mutex);
1230         put_ctx(ctx);
1231 }
1232 
1233 /*
1234  * This must be done under the ctx->lock, such as to serialize against
1235  * context_equiv(), therefore we cannot call put_ctx() since that might end up
1236  * calling scheduler related locks and ctx->lock nests inside those.
1237  */
1238 static __must_check struct perf_event_context *
1239 unclone_ctx(struct perf_event_context *ctx)
1240 {
1241         struct perf_event_context *parent_ctx = ctx->parent_ctx;
1242 
1243         lockdep_assert_held(&ctx->lock);
1244 
1245         if (parent_ctx)
1246                 ctx->parent_ctx = NULL;
1247         ctx->generation++;
1248 
1249         return parent_ctx;
1250 }
1251 
1252 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
1253 {
1254         /*
1255          * only top level events have the pid namespace they were created in
1256          */
1257         if (event->parent)
1258                 event = event->parent;
1259 
1260         return task_tgid_nr_ns(p, event->ns);
1261 }
1262 
1263 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
1264 {
1265         /*
1266          * only top level events have the pid namespace they were created in
1267          */
1268         if (event->parent)
1269                 event = event->parent;
1270 
1271         return task_pid_nr_ns(p, event->ns);
1272 }
1273 
1274 /*
1275  * If we inherit events we want to return the parent event id
1276  * to userspace.
1277  */
1278 static u64 primary_event_id(struct perf_event *event)
1279 {
1280         u64 id = event->id;
1281 
1282         if (event->parent)
1283                 id = event->parent->id;
1284 
1285         return id;
1286 }
1287 
1288 /*
1289  * Get the perf_event_context for a task and lock it.
1290  *
1291  * This has to cope with with the fact that until it is locked,
1292  * the context could get moved to another task.
1293  */
1294 static struct perf_event_context *
1295 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
1296 {
1297         struct perf_event_context *ctx;
1298 
1299 retry:
1300         /*
1301          * One of the few rules of preemptible RCU is that one cannot do
1302          * rcu_read_unlock() while holding a scheduler (or nested) lock when
1303          * part of the read side critical section was irqs-enabled -- see
1304          * rcu_read_unlock_special().
1305          *
1306          * Since ctx->lock nests under rq->lock we must ensure the entire read
1307          * side critical section has interrupts disabled.
1308          */
1309         local_irq_save(*flags);
1310         rcu_read_lock();
1311         ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
1312         if (ctx) {
1313                 /*
1314                  * If this context is a clone of another, it might
1315                  * get swapped for another underneath us by
1316                  * perf_event_task_sched_out, though the
1317                  * rcu_read_lock() protects us from any context
1318                  * getting freed.  Lock the context and check if it
1319                  * got swapped before we could get the lock, and retry
1320                  * if so.  If we locked the right context, then it
1321                  * can't get swapped on us any more.
1322                  */
1323                 raw_spin_lock(&ctx->lock);
1324                 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
1325                         raw_spin_unlock(&ctx->lock);
1326                         rcu_read_unlock();
1327                         local_irq_restore(*flags);
1328                         goto retry;
1329                 }
1330 
1331                 if (ctx->task == TASK_TOMBSTONE ||
1332                     !atomic_inc_not_zero(&ctx->refcount)) {
1333                         raw_spin_unlock(&ctx->lock);
1334                         ctx = NULL;
1335                 } else {
1336                         WARN_ON_ONCE(ctx->task != task);
1337                 }
1338         }
1339         rcu_read_unlock();
1340         if (!ctx)
1341                 local_irq_restore(*flags);
1342         return ctx;
1343 }
1344 
1345 /*
1346  * Get the context for a task and increment its pin_count so it
1347  * can't get swapped to another task.  This also increments its
1348  * reference count so that the context can't get freed.
1349  */
1350 static struct perf_event_context *
1351 perf_pin_task_context(struct task_struct *task, int ctxn)
1352 {
1353         struct perf_event_context *ctx;
1354         unsigned long flags;
1355 
1356         ctx = perf_lock_task_context(task, ctxn, &flags);
1357         if (ctx) {
1358                 ++ctx->pin_count;
1359                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
1360         }
1361         return ctx;
1362 }
1363 
1364 static void perf_unpin_context(struct perf_event_context *ctx)
1365 {
1366         unsigned long flags;
1367 
1368         raw_spin_lock_irqsave(&ctx->lock, flags);
1369         --ctx->pin_count;
1370         raw_spin_unlock_irqrestore(&ctx->lock, flags);
1371 }
1372 
1373 /*
1374  * Update the record of the current time in a context.
1375  */
1376 static void update_context_time(struct perf_event_context *ctx)
1377 {
1378         u64 now = perf_clock();
1379 
1380         ctx->time += now - ctx->timestamp;
1381         ctx->timestamp = now;
1382 }
1383 
1384 static u64 perf_event_time(struct perf_event *event)
1385 {
1386         struct perf_event_context *ctx = event->ctx;
1387 
1388         if (is_cgroup_event(event))
1389                 return perf_cgroup_event_time(event);
1390 
1391         return ctx ? ctx->time : 0;
1392 }
1393 
1394 /*
1395  * Update the total_time_enabled and total_time_running fields for a event.
1396  */
1397 static void update_event_times(struct perf_event *event)
1398 {
1399         struct perf_event_context *ctx = event->ctx;
1400         u64 run_end;
1401 
1402         lockdep_assert_held(&ctx->lock);
1403 
1404         if (event->state < PERF_EVENT_STATE_INACTIVE ||
1405             event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
1406                 return;
1407 
1408         /*
1409          * in cgroup mode, time_enabled represents
1410          * the time the event was enabled AND active
1411          * tasks were in the monitored cgroup. This is
1412          * independent of the activity of the context as
1413          * there may be a mix of cgroup and non-cgroup events.
1414          *
1415          * That is why we treat cgroup events differently
1416          * here.
1417          */
1418         if (is_cgroup_event(event))
1419                 run_end = perf_cgroup_event_time(event);
1420         else if (ctx->is_active)
1421                 run_end = ctx->time;
1422         else
1423                 run_end = event->tstamp_stopped;
1424 
1425         event->total_time_enabled = run_end - event->tstamp_enabled;
1426 
1427         if (event->state == PERF_EVENT_STATE_INACTIVE)
1428                 run_end = event->tstamp_stopped;
1429         else
1430                 run_end = perf_event_time(event);
1431 
1432         event->total_time_running = run_end - event->tstamp_running;
1433 
1434 }
1435 
1436 /*
1437  * Update total_time_enabled and total_time_running for all events in a group.
1438  */
1439 static void update_group_times(struct perf_event *leader)
1440 {
1441         struct perf_event *event;
1442 
1443         update_event_times(leader);
1444         list_for_each_entry(event, &leader->sibling_list, group_entry)
1445                 update_event_times(event);
1446 }
1447 
1448 static struct list_head *
1449 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
1450 {
1451         if (event->attr.pinned)
1452                 return &ctx->pinned_groups;
1453         else
1454                 return &ctx->flexible_groups;
1455 }
1456 
1457 /*
1458  * Add a event from the lists for its context.
1459  * Must be called with ctx->mutex and ctx->lock held.
1460  */
1461 static void
1462 list_add_event(struct perf_event *event, struct perf_event_context *ctx)
1463 {
1464 
1465         lockdep_assert_held(&ctx->lock);
1466 
1467         WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
1468         event->attach_state |= PERF_ATTACH_CONTEXT;
1469 
1470         /*
1471          * If we're a stand alone event or group leader, we go to the context
1472          * list, group events are kept attached to the group so that
1473          * perf_group_detach can, at all times, locate all siblings.
1474          */
1475         if (event->group_leader == event) {
1476                 struct list_head *list;
1477 
1478                 if (is_software_event(event))
1479                         event->group_flags |= PERF_GROUP_SOFTWARE;
1480 
1481                 list = ctx_group_list(event, ctx);
1482                 list_add_tail(&event->group_entry, list);
1483         }
1484 
1485         list_update_cgroup_event(event, ctx, true);
1486 
1487         list_add_rcu(&event->event_entry, &ctx->event_list);
1488         ctx->nr_events++;
1489         if (event->attr.inherit_stat)
1490                 ctx->nr_stat++;
1491 
1492         ctx->generation++;
1493 }
1494 
1495 /*
1496  * Initialize event state based on the perf_event_attr::disabled.
1497  */
1498 static inline void perf_event__state_init(struct perf_event *event)
1499 {
1500         event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
1501                                               PERF_EVENT_STATE_INACTIVE;
1502 }
1503 
1504 static void __perf_event_read_size(struct perf_event *event, int nr_siblings)
1505 {
1506         int entry = sizeof(u64); /* value */
1507         int size = 0;
1508         int nr = 1;
1509 
1510         if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1511                 size += sizeof(u64);
1512 
1513         if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1514                 size += sizeof(u64);
1515 
1516         if (event->attr.read_format & PERF_FORMAT_ID)
1517                 entry += sizeof(u64);
1518 
1519         if (event->attr.read_format & PERF_FORMAT_GROUP) {
1520                 nr += nr_siblings;
1521                 size += sizeof(u64);
1522         }
1523 
1524         size += entry * nr;
1525         event->read_size = size;
1526 }
1527 
1528 static void __perf_event_header_size(struct perf_event *event, u64 sample_type)
1529 {
1530         struct perf_sample_data *data;
1531         u16 size = 0;
1532 
1533         if (sample_type & PERF_SAMPLE_IP)
1534                 size += sizeof(data->ip);
1535 
1536         if (sample_type & PERF_SAMPLE_ADDR)
1537                 size += sizeof(data->addr);
1538 
1539         if (sample_type & PERF_SAMPLE_PERIOD)
1540                 size += sizeof(data->period);
1541 
1542         if (sample_type & PERF_SAMPLE_WEIGHT)
1543                 size += sizeof(data->weight);
1544 
1545         if (sample_type & PERF_SAMPLE_READ)
1546                 size += event->read_size;
1547 
1548         if (sample_type & PERF_SAMPLE_DATA_SRC)
1549                 size += sizeof(data->data_src.val);
1550 
1551         if (sample_type & PERF_SAMPLE_TRANSACTION)
1552                 size += sizeof(data->txn);
1553 
1554         event->header_size = size;
1555 }
1556 
1557 /*
1558  * Called at perf_event creation and when events are attached/detached from a
1559  * group.
1560  */
1561 static void perf_event__header_size(struct perf_event *event)
1562 {
1563         __perf_event_read_size(event,
1564                                event->group_leader->nr_siblings);
1565         __perf_event_header_size(event, event->attr.sample_type);
1566 }
1567 
1568 static void perf_event__id_header_size(struct perf_event *event)
1569 {
1570         struct perf_sample_data *data;
1571         u64 sample_type = event->attr.sample_type;
1572         u16 size = 0;
1573 
1574         if (sample_type & PERF_SAMPLE_TID)
1575                 size += sizeof(data->tid_entry);
1576 
1577         if (sample_type & PERF_SAMPLE_TIME)
1578                 size += sizeof(data->time);
1579 
1580         if (sample_type & PERF_SAMPLE_IDENTIFIER)
1581                 size += sizeof(data->id);
1582 
1583         if (sample_type & PERF_SAMPLE_ID)
1584                 size += sizeof(data->id);
1585 
1586         if (sample_type & PERF_SAMPLE_STREAM_ID)
1587                 size += sizeof(data->stream_id);
1588 
1589         if (sample_type & PERF_SAMPLE_CPU)
1590                 size += sizeof(data->cpu_entry);
1591 
1592         event->id_header_size = size;
1593 }
1594 
1595 static bool perf_event_validate_size(struct perf_event *event)
1596 {
1597         /*
1598          * The values computed here will be over-written when we actually
1599          * attach the event.
1600          */
1601         __perf_event_read_size(event, event->group_leader->nr_siblings + 1);
1602         __perf_event_header_size(event, event->attr.sample_type & ~PERF_SAMPLE_READ);
1603         perf_event__id_header_size(event);
1604 
1605         /*
1606          * Sum the lot; should not exceed the 64k limit we have on records.
1607          * Conservative limit to allow for callchains and other variable fields.
1608          */
1609         if (event->read_size + event->header_size +
1610             event->id_header_size + sizeof(struct perf_event_header) >= 16*1024)
1611                 return false;
1612 
1613         return true;
1614 }
1615 
1616 static void perf_group_attach(struct perf_event *event)
1617 {
1618         struct perf_event *group_leader = event->group_leader, *pos;
1619 
1620         /*
1621          * We can have double attach due to group movement in perf_event_open.
1622          */
1623         if (event->attach_state & PERF_ATTACH_GROUP)
1624                 return;
1625 
1626         event->attach_state |= PERF_ATTACH_GROUP;
1627 
1628         if (group_leader == event)
1629                 return;
1630 
1631         WARN_ON_ONCE(group_leader->ctx != event->ctx);
1632 
1633         if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
1634                         !is_software_event(event))
1635                 group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
1636 
1637         list_add_tail(&event->group_entry, &group_leader->sibling_list);
1638         group_leader->nr_siblings++;
1639 
1640         perf_event__header_size(group_leader);
1641 
1642         list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
1643                 perf_event__header_size(pos);
1644 }
1645 
1646 /*
1647  * Remove a event from the lists for its context.
1648  * Must be called with ctx->mutex and ctx->lock held.
1649  */
1650 static void
1651 list_del_event(struct perf_event *event, struct perf_event_context *ctx)
1652 {
1653         WARN_ON_ONCE(event->ctx != ctx);
1654         lockdep_assert_held(&ctx->lock);
1655 
1656         /*
1657          * We can have double detach due to exit/hot-unplug + close.
1658          */
1659         if (!(event->attach_state & PERF_ATTACH_CONTEXT))
1660                 return;
1661 
1662         event->attach_state &= ~PERF_ATTACH_CONTEXT;
1663 
1664         list_update_cgroup_event(event, ctx, false);
1665 
1666         ctx->nr_events--;
1667         if (event->attr.inherit_stat)
1668                 ctx->nr_stat--;
1669 
1670         list_del_rcu(&event->event_entry);
1671 
1672         if (event->group_leader == event)
1673                 list_del_init(&event->group_entry);
1674 
1675         update_group_times(event);
1676 
1677         /*
1678          * If event was in error state, then keep it
1679          * that way, otherwise bogus counts will be
1680          * returned on read(). The only way to get out
1681          * of error state is by explicit re-enabling
1682          * of the event
1683          */
1684         if (event->state > PERF_EVENT_STATE_OFF)
1685                 event->state = PERF_EVENT_STATE_OFF;
1686 
1687         ctx->generation++;
1688 }
1689 
1690 static void perf_group_detach(struct perf_event *event)
1691 {
1692         struct perf_event *sibling, *tmp;
1693         struct list_head *list = NULL;
1694 
1695         /*
1696          * We can have double detach due to exit/hot-unplug + close.
1697          */
1698         if (!(event->attach_state & PERF_ATTACH_GROUP))
1699                 return;
1700 
1701         event->attach_state &= ~PERF_ATTACH_GROUP;
1702 
1703         /*
1704          * If this is a sibling, remove it from its group.
1705          */
1706         if (event->group_leader != event) {
1707                 list_del_init(&event->group_entry);
1708                 event->group_leader->nr_siblings--;
1709                 goto out;
1710         }
1711 
1712         if (!list_empty(&event->group_entry))
1713                 list = &event->group_entry;
1714 
1715         /*
1716          * If this was a group event with sibling events then
1717          * upgrade the siblings to singleton events by adding them
1718          * to whatever list we are on.
1719          */
1720         list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
1721                 if (list)
1722                         list_move_tail(&sibling->group_entry, list);
1723                 sibling->group_leader = sibling;
1724 
1725                 /* Inherit group flags from the previous leader */
1726                 sibling->group_flags = event->group_flags;
1727 
1728                 WARN_ON_ONCE(sibling->ctx != event->ctx);
1729         }
1730 
1731 out:
1732         perf_event__header_size(event->group_leader);
1733 
1734         list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
1735                 perf_event__header_size(tmp);
1736 }
1737 
1738 static bool is_orphaned_event(struct perf_event *event)
1739 {
1740         return event->state == PERF_EVENT_STATE_DEAD;
1741 }
1742 
1743 static inline int __pmu_filter_match(struct perf_event *event)
1744 {
1745         struct pmu *pmu = event->pmu;
1746         return pmu->filter_match ? pmu->filter_match(event) : 1;
1747 }
1748 
1749 /*
1750  * Check whether we should attempt to schedule an event group based on
1751  * PMU-specific filtering. An event group can consist of HW and SW events,
1752  * potentially with a SW leader, so we must check all the filters, to
1753  * determine whether a group is schedulable:
1754  */
1755 static inline int pmu_filter_match(struct perf_event *event)
1756 {
1757         struct perf_event *child;
1758 
1759         if (!__pmu_filter_match(event))
1760                 return 0;
1761 
1762         list_for_each_entry(child, &event->sibling_list, group_entry) {
1763                 if (!__pmu_filter_match(child))
1764                         return 0;
1765         }
1766 
1767         return 1;
1768 }
1769 
1770 static inline int
1771 event_filter_match(struct perf_event *event)
1772 {
1773         return (event->cpu == -1 || event->cpu == smp_processor_id()) &&
1774                perf_cgroup_match(event) && pmu_filter_match(event);
1775 }
1776 
1777 static void
1778 event_sched_out(struct perf_event *event,
1779                   struct perf_cpu_context *cpuctx,
1780                   struct perf_event_context *ctx)
1781 {
1782         u64 tstamp = perf_event_time(event);
1783         u64 delta;
1784 
1785         WARN_ON_ONCE(event->ctx != ctx);
1786         lockdep_assert_held(&ctx->lock);
1787 
1788         /*
1789          * An event which could not be activated because of
1790          * filter mismatch still needs to have its timings
1791          * maintained, otherwise bogus information is return
1792          * via read() for time_enabled, time_running:
1793          */
1794         if (event->state == PERF_EVENT_STATE_INACTIVE &&
1795             !event_filter_match(event)) {
1796                 delta = tstamp - event->tstamp_stopped;
1797                 event->tstamp_running += delta;
1798                 event->tstamp_stopped = tstamp;
1799         }
1800 
1801         if (event->state != PERF_EVENT_STATE_ACTIVE)
1802                 return;
1803 
1804         perf_pmu_disable(event->pmu);
1805 
1806         event->tstamp_stopped = tstamp;
1807         event->pmu->del(event, 0);
1808         event->oncpu = -1;
1809         event->state = PERF_EVENT_STATE_INACTIVE;
1810         if (event->pending_disable) {
1811                 event->pending_disable = 0;
1812                 event->state = PERF_EVENT_STATE_OFF;
1813         }
1814 
1815         if (!is_software_event(event))
1816                 cpuctx->active_oncpu--;
1817         if (!--ctx->nr_active)
1818                 perf_event_ctx_deactivate(ctx);
1819         if (event->attr.freq && event->attr.sample_freq)
1820                 ctx->nr_freq--;
1821         if (event->attr.exclusive || !cpuctx->active_oncpu)
1822                 cpuctx->exclusive = 0;
1823 
1824         perf_pmu_enable(event->pmu);
1825 }
1826 
1827 static void
1828 group_sched_out(struct perf_event *group_event,
1829                 struct perf_cpu_context *cpuctx,
1830                 struct perf_event_context *ctx)
1831 {
1832         struct perf_event *event;
1833         int state = group_event->state;
1834 
1835         event_sched_out(group_event, cpuctx, ctx);
1836 
1837         /*
1838          * Schedule out siblings (if any):
1839          */
1840         list_for_each_entry(event, &group_event->sibling_list, group_entry)
1841                 event_sched_out(event, cpuctx, ctx);
1842 
1843         if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
1844                 cpuctx->exclusive = 0;
1845 }
1846 
1847 #define DETACH_GROUP    0x01UL
1848 
1849 /*
1850  * Cross CPU call to remove a performance event
1851  *
1852  * We disable the event on the hardware level first. After that we
1853  * remove it from the context list.
1854  */
1855 static void
1856 __perf_remove_from_context(struct perf_event *event,
1857                            struct perf_cpu_context *cpuctx,
1858                            struct perf_event_context *ctx,
1859                            void *info)
1860 {
1861         unsigned long flags = (unsigned long)info;
1862 
1863         event_sched_out(event, cpuctx, ctx);
1864         if (flags & DETACH_GROUP)
1865                 perf_group_detach(event);
1866         list_del_event(event, ctx);
1867 
1868         if (!ctx->nr_events && ctx->is_active) {
1869                 ctx->is_active = 0;
1870                 if (ctx->task) {
1871                         WARN_ON_ONCE(cpuctx->task_ctx != ctx);
1872                         cpuctx->task_ctx = NULL;
1873                 }
1874         }
1875 }
1876 
1877 /*
1878  * Remove the event from a task's (or a CPU's) list of events.
1879  *
1880  * If event->ctx is a cloned context, callers must make sure that
1881  * every task struct that event->ctx->task could possibly point to
1882  * remains valid.  This is OK when called from perf_release since
1883  * that only calls us on the top-level context, which can't be a clone.
1884  * When called from perf_event_exit_task, it's OK because the
1885  * context has been detached from its task.
1886  */
1887 static void perf_remove_from_context(struct perf_event *event, unsigned long flags)
1888 {
1889         lockdep_assert_held(&event->ctx->mutex);
1890 
1891         event_function_call(event, __perf_remove_from_context, (void *)flags);
1892 }
1893 
1894 /*
1895  * Cross CPU call to disable a performance event
1896  */
1897 static void __perf_event_disable(struct perf_event *event,
1898                                  struct perf_cpu_context *cpuctx,
1899                                  struct perf_event_context *ctx,
1900                                  void *info)
1901 {
1902         if (event->state < PERF_EVENT_STATE_INACTIVE)
1903                 return;
1904 
1905         update_context_time(ctx);
1906         update_cgrp_time_from_event(event);
1907         update_group_times(event);
1908         if (event == event->group_leader)
1909                 group_sched_out(event, cpuctx, ctx);
1910         else
1911                 event_sched_out(event, cpuctx, ctx);
1912         event->state = PERF_EVENT_STATE_OFF;
1913 }
1914 
1915 /*
1916  * Disable a event.
1917  *
1918  * If event->ctx is a cloned context, callers must make sure that
1919  * every task struct that event->ctx->task could possibly point to
1920  * remains valid.  This condition is satisifed when called through
1921  * perf_event_for_each_child or perf_event_for_each because they
1922  * hold the top-level event's child_mutex, so any descendant that
1923  * goes to exit will block in perf_event_exit_event().
1924  *
1925  * When called from perf_pending_event it's OK because event->ctx
1926  * is the current context on this CPU and preemption is disabled,
1927  * hence we can't get into perf_event_task_sched_out for this context.
1928  */
1929 static void _perf_event_disable(struct perf_event *event)
1930 {
1931         struct perf_event_context *ctx = event->ctx;
1932 
1933         raw_spin_lock_irq(&ctx->lock);
1934         if (event->state <= PERF_EVENT_STATE_OFF) {
1935                 raw_spin_unlock_irq(&ctx->lock);
1936                 return;
1937         }
1938         raw_spin_unlock_irq(&ctx->lock);
1939 
1940         event_function_call(event, __perf_event_disable, NULL);
1941 }
1942 
1943 void perf_event_disable_local(struct perf_event *event)
1944 {
1945         event_function_local(event, __perf_event_disable, NULL);
1946 }
1947 
1948 /*
1949  * Strictly speaking kernel users cannot create groups and therefore this
1950  * interface does not need the perf_event_ctx_lock() magic.
1951  */
1952 void perf_event_disable(struct perf_event *event)
1953 {
1954         struct perf_event_context *ctx;
1955 
1956         ctx = perf_event_ctx_lock(event);
1957         _perf_event_disable(event);
1958         perf_event_ctx_unlock(event, ctx);
1959 }
1960 EXPORT_SYMBOL_GPL(perf_event_disable);
1961 
1962 static void perf_set_shadow_time(struct perf_event *event,
1963                                  struct perf_event_context *ctx,
1964                                  u64 tstamp)
1965 {
1966         /*
1967          * use the correct time source for the time snapshot
1968          *
1969          * We could get by without this by leveraging the
1970          * fact that to get to this function, the caller
1971          * has most likely already called update_context_time()
1972          * and update_cgrp_time_xx() and thus both timestamp
1973          * are identical (or very close). Given that tstamp is,
1974          * already adjusted for cgroup, we could say that:
1975          *    tstamp - ctx->timestamp
1976          * is equivalent to
1977          *    tstamp - cgrp->timestamp.
1978          *
1979          * Then, in perf_output_read(), the calculation would
1980          * work with no changes because:
1981          * - event is guaranteed scheduled in
1982          * - no scheduled out in between
1983          * - thus the timestamp would be the same
1984          *
1985          * But this is a bit hairy.
1986          *
1987          * So instead, we have an explicit cgroup call to remain
1988          * within the time time source all along. We believe it
1989          * is cleaner and simpler to understand.
1990          */
1991         if (is_cgroup_event(event))
1992                 perf_cgroup_set_shadow_time(event, tstamp);
1993         else
1994                 event->shadow_ctx_time = tstamp - ctx->timestamp;
1995 }
1996 
1997 #define MAX_INTERRUPTS (~0ULL)
1998 
1999 static void perf_log_throttle(struct perf_event *event, int enable);
2000 static void perf_log_itrace_start(struct perf_event *event);
2001 
2002 static int
2003 event_sched_in(struct perf_event *event,
2004                  struct perf_cpu_context *cpuctx,
2005                  struct perf_event_context *ctx)
2006 {
2007         u64 tstamp = perf_event_time(event);
2008         int ret = 0;
2009 
2010         lockdep_assert_held(&ctx->lock);
2011 
2012         if (event->state <= PERF_EVENT_STATE_OFF)
2013                 return 0;
2014 
2015         WRITE_ONCE(event->oncpu, smp_processor_id());
2016         /*
2017          * Order event::oncpu write to happen before the ACTIVE state
2018          * is visible.
2019          */
2020         smp_wmb();
2021         WRITE_ONCE(event->state, PERF_EVENT_STATE_ACTIVE);
2022 
2023         /*
2024          * Unthrottle events, since we scheduled we might have missed several
2025          * ticks already, also for a heavily scheduling task there is little
2026          * guarantee it'll get a tick in a timely manner.
2027          */
2028         if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
2029                 perf_log_throttle(event, 1);
2030                 event->hw.interrupts = 0;
2031         }
2032 
2033         /*
2034          * The new state must be visible before we turn it on in the hardware:
2035          */
2036         smp_wmb();
2037 
2038         perf_pmu_disable(event->pmu);
2039 
2040         perf_set_shadow_time(event, ctx, tstamp);
2041 
2042         perf_log_itrace_start(event);
2043 
2044         if (event->pmu->add(event, PERF_EF_START)) {
2045                 event->state = PERF_EVENT_STATE_INACTIVE;
2046                 event->oncpu = -1;
2047                 ret = -EAGAIN;
2048                 goto out;
2049         }
2050 
2051         event->tstamp_running += tstamp - event->tstamp_stopped;
2052 
2053         if (!is_software_event(event))
2054                 cpuctx->active_oncpu++;
2055         if (!ctx->nr_active++)
2056                 perf_event_ctx_activate(ctx);
2057         if (event->attr.freq && event->attr.sample_freq)
2058                 ctx->nr_freq++;
2059 
2060         if (event->attr.exclusive)
2061                 cpuctx->exclusive = 1;
2062 
2063 out:
2064         perf_pmu_enable(event->pmu);
2065 
2066         return ret;
2067 }
2068 
2069 static int
2070 group_sched_in(struct perf_event *group_event,
2071                struct perf_cpu_context *cpuctx,
2072                struct perf_event_context *ctx)
2073 {
2074         struct perf_event *event, *partial_group = NULL;
2075         struct pmu *pmu = ctx->pmu;
2076         u64 now = ctx->time;
2077         bool simulate = false;
2078 
2079         if (group_event->state == PERF_EVENT_STATE_OFF)
2080                 return 0;
2081 
2082         pmu->start_txn(pmu, PERF_PMU_TXN_ADD);
2083 
2084         if (event_sched_in(group_event, cpuctx, ctx)) {
2085                 pmu->cancel_txn(pmu);
2086                 perf_mux_hrtimer_restart(cpuctx);
2087                 return -EAGAIN;
2088         }
2089 
2090         /*
2091          * Schedule in siblings as one group (if any):
2092          */
2093         list_for_each_entry(event, &group_event->sibling_list, group_entry) {
2094                 if (event_sched_in(event, cpuctx, ctx)) {
2095                         partial_group = event;
2096                         goto group_error;
2097                 }
2098         }
2099 
2100         if (!pmu->commit_txn(pmu))
2101                 return 0;
2102 
2103 group_error:
2104         /*
2105          * Groups can be scheduled in as one unit only, so undo any
2106          * partial group before returning:
2107          * The events up to the failed event are scheduled out normally,
2108          * tstamp_stopped will be updated.
2109          *
2110          * The failed events and the remaining siblings need to have
2111          * their timings updated as if they had gone thru event_sched_in()
2112          * and event_sched_out(). This is required to get consistent timings
2113          * across the group. This also takes care of the case where the group
2114          * could never be scheduled by ensuring tstamp_stopped is set to mark
2115          * the time the event was actually stopped, such that time delta
2116          * calculation in update_event_times() is correct.
2117          */
2118         list_for_each_entry(event, &group_event->sibling_list, group_entry) {
2119                 if (event == partial_group)
2120                         simulate = true;
2121 
2122                 if (simulate) {
2123                         event->tstamp_running += now - event->tstamp_stopped;
2124                         event->tstamp_stopped = now;
2125                 } else {
2126                         event_sched_out(event, cpuctx, ctx);
2127                 }
2128         }
2129         event_sched_out(group_event, cpuctx, ctx);
2130 
2131         pmu->cancel_txn(pmu);
2132 
2133         perf_mux_hrtimer_restart(cpuctx);
2134 
2135         return -EAGAIN;
2136 }
2137 
2138 /*
2139  * Work out whether we can put this event group on the CPU now.
2140  */
2141 static int group_can_go_on(struct perf_event *event,
2142                            struct perf_cpu_context *cpuctx,
2143                            int can_add_hw)
2144 {
2145         /*
2146          * Groups consisting entirely of software events can always go on.
2147          */
2148         if (event->group_flags & PERF_GROUP_SOFTWARE)
2149                 return 1;
2150         /*
2151          * If an exclusive group is already on, no other hardware
2152          * events can go on.
2153          */
2154         if (cpuctx->exclusive)
2155                 return 0;
2156         /*
2157          * If this group is exclusive and there are already
2158          * events on the CPU, it can't go on.
2159          */
2160         if (event->attr.exclusive && cpuctx->active_oncpu)
2161                 return 0;
2162         /*
2163          * Otherwise, try to add it if all previous groups were able
2164          * to go on.
2165          */
2166         return can_add_hw;
2167 }
2168 
2169 static void add_event_to_ctx(struct perf_event *event,
2170                                struct perf_event_context *ctx)
2171 {
2172         u64 tstamp = perf_event_time(event);
2173 
2174         list_add_event(event, ctx);
2175         perf_group_attach(event);
2176         event->tstamp_enabled = tstamp;
2177         event->tstamp_running = tstamp;
2178         event->tstamp_stopped = tstamp;
2179 }
2180 
2181 static void ctx_sched_out(struct perf_event_context *ctx,
2182                           struct perf_cpu_context *cpuctx,
2183                           enum event_type_t event_type);
2184 static void
2185 ctx_sched_in(struct perf_event_context *ctx,
2186              struct perf_cpu_context *cpuctx,
2187              enum event_type_t event_type,
2188              struct task_struct *task);
2189 
2190 static void task_ctx_sched_out(struct perf_cpu_context *cpuctx,
2191                                struct perf_event_context *ctx)
2192 {
2193         if (!cpuctx->task_ctx)
2194                 return;
2195 
2196         if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2197                 return;
2198 
2199         ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2200 }
2201 
2202 static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
2203                                 struct perf_event_context *ctx,
2204                                 struct task_struct *task)
2205 {
2206         cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
2207         if (ctx)
2208                 ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
2209         cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
2210         if (ctx)
2211                 ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
2212 }
2213 
2214 static void ctx_resched(struct perf_cpu_context *cpuctx,
2215                         struct perf_event_context *task_ctx)
2216 {
2217         perf_pmu_disable(cpuctx->ctx.pmu);
2218         if (task_ctx)
2219                 task_ctx_sched_out(cpuctx, task_ctx);
2220         cpu_ctx_sched_out(cpuctx, EVENT_ALL);
2221         perf_event_sched_in(cpuctx, task_ctx, current);
2222         perf_pmu_enable(cpuctx->ctx.pmu);
2223 }
2224 
2225 /*
2226  * Cross CPU call to install and enable a performance event
2227  *
2228  * Very similar to remote_function() + event_function() but cannot assume that
2229  * things like ctx->is_active and cpuctx->task_ctx are set.
2230  */
2231 static int  __perf_install_in_context(void *info)
2232 {
2233         struct perf_event *event = info;
2234         struct perf_event_context *ctx = event->ctx;
2235         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2236         struct perf_event_context *task_ctx = cpuctx->task_ctx;
2237         bool activate = true;
2238         int ret = 0;
2239 
2240         raw_spin_lock(&cpuctx->ctx.lock);
2241         if (ctx->task) {
2242                 raw_spin_lock(&ctx->lock);
2243                 task_ctx = ctx;
2244 
2245                 /* If we're on the wrong CPU, try again */
2246                 if (task_cpu(ctx->task) != smp_processor_id()) {
2247                         ret = -ESRCH;
2248                         goto unlock;
2249                 }
2250 
2251                 /*
2252                  * If we're on the right CPU, see if the task we target is
2253                  * current, if not we don't have to activate the ctx, a future
2254                  * context switch will do that for us.
2255                  */
2256                 if (ctx->task != current)
2257                         activate = false;
2258                 else
2259                         WARN_ON_ONCE(cpuctx->task_ctx && cpuctx->task_ctx != ctx);
2260 
2261         } else if (task_ctx) {
2262                 raw_spin_lock(&task_ctx->lock);
2263         }
2264 
2265         if (activate) {
2266                 ctx_sched_out(ctx, cpuctx, EVENT_TIME);
2267                 add_event_to_ctx(event, ctx);
2268                 ctx_resched(cpuctx, task_ctx);
2269         } else {
2270                 add_event_to_ctx(event, ctx);
2271         }
2272 
2273 unlock:
2274         perf_ctx_unlock(cpuctx, task_ctx);
2275 
2276         return ret;
2277 }
2278 
2279 /*
2280  * Attach a performance event to a context.
2281  *
2282  * Very similar to event_function_call, see comment there.
2283  */
2284 static void
2285 perf_install_in_context(struct perf_event_context *ctx,
2286                         struct perf_event *event,
2287                         int cpu)
2288 {
2289         struct task_struct *task = READ_ONCE(ctx->task);
2290 
2291         lockdep_assert_held(&ctx->mutex);
2292 
2293         if (event->cpu != -1)
2294                 event->cpu = cpu;
2295 
2296         /*
2297          * Ensures that if we can observe event->ctx, both the event and ctx
2298          * will be 'complete'. See perf_iterate_sb_cpu().
2299          */
2300         smp_store_release(&event->ctx, ctx);
2301 
2302         if (!task) {
2303                 cpu_function_call(cpu, __perf_install_in_context, event);
2304                 return;
2305         }
2306 
2307         /*
2308          * Should not happen, we validate the ctx is still alive before calling.
2309          */
2310         if (WARN_ON_ONCE(task == TASK_TOMBSTONE))
2311                 return;
2312 
2313         /*
2314          * Installing events is tricky because we cannot rely on ctx->is_active
2315          * to be set in case this is the nr_events 0 -> 1 transition.
2316          */
2317 again:
2318         /*
2319          * Cannot use task_function_call() because we need to run on the task's
2320          * CPU regardless of whether its current or not.
2321          */
2322         if (!cpu_function_call(task_cpu(task), __perf_install_in_context, event))
2323                 return;
2324 
2325         raw_spin_lock_irq(&ctx->lock);
2326         task = ctx->task;
2327         if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) {
2328                 /*
2329                  * Cannot happen because we already checked above (which also
2330                  * cannot happen), and we hold ctx->mutex, which serializes us
2331                  * against perf_event_exit_task_context().
2332                  */
2333                 raw_spin_unlock_irq(&ctx->lock);
2334                 return;
2335         }
2336         raw_spin_unlock_irq(&ctx->lock);
2337         /*
2338          * Since !ctx->is_active doesn't mean anything, we must IPI
2339          * unconditionally.
2340          */
2341         goto again;
2342 }
2343 
2344 /*
2345  * Put a event into inactive state and update time fields.
2346  * Enabling the leader of a group effectively enables all
2347  * the group members that aren't explicitly disabled, so we
2348  * have to update their ->tstamp_enabled also.
2349  * Note: this works for group members as well as group leaders
2350  * since the non-leader members' sibling_lists will be empty.
2351  */
2352 static void __perf_event_mark_enabled(struct perf_event *event)
2353 {
2354         struct perf_event *sub;
2355         u64 tstamp = perf_event_time(event);
2356 
2357         event->state = PERF_EVENT_STATE_INACTIVE;
2358         event->tstamp_enabled = tstamp - event->total_time_enabled;
2359         list_for_each_entry(sub, &event->sibling_list, group_entry) {
2360                 if (sub->state >= PERF_EVENT_STATE_INACTIVE)
2361                         sub->tstamp_enabled = tstamp - sub->total_time_enabled;
2362         }
2363 }
2364 
2365 /*
2366  * Cross CPU call to enable a performance event
2367  */
2368 static void __perf_event_enable(struct perf_event *event,
2369                                 struct perf_cpu_context *cpuctx,
2370                                 struct perf_event_context *ctx,
2371                                 void *info)
2372 {
2373         struct perf_event *leader = event->group_leader;
2374         struct perf_event_context *task_ctx;
2375 
2376         if (event->state >= PERF_EVENT_STATE_INACTIVE ||
2377             event->state <= PERF_EVENT_STATE_ERROR)
2378                 return;
2379 
2380         if (ctx->is_active)
2381                 ctx_sched_out(ctx, cpuctx, EVENT_TIME);
2382 
2383         __perf_event_mark_enabled(event);
2384 
2385         if (!ctx->is_active)
2386                 return;
2387 
2388         if (!event_filter_match(event)) {
2389                 if (is_cgroup_event(event))
2390                         perf_cgroup_defer_enabled(event);
2391                 ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);
2392                 return;
2393         }
2394 
2395         /*
2396          * If the event is in a group and isn't the group leader,
2397          * then don't put it on unless the group is on.
2398          */
2399         if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) {
2400                 ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);
2401                 return;
2402         }
2403 
2404         task_ctx = cpuctx->task_ctx;
2405         if (ctx->task)
2406                 WARN_ON_ONCE(task_ctx != ctx);
2407 
2408         ctx_resched(cpuctx, task_ctx);
2409 }
2410 
2411 /*
2412  * Enable a event.
2413  *
2414  * If event->ctx is a cloned context, callers must make sure that
2415  * every task struct that event->ctx->task could possibly point to
2416  * remains valid.  This condition is satisfied when called through
2417  * perf_event_for_each_child or perf_event_for_each as described
2418  * for perf_event_disable.
2419  */
2420 static void _perf_event_enable(struct perf_event *event)
2421 {
2422         struct perf_event_context *ctx = event->ctx;
2423 
2424         raw_spin_lock_irq(&ctx->lock);
2425         if (event->state >= PERF_EVENT_STATE_INACTIVE ||
2426             event->state <  PERF_EVENT_STATE_ERROR) {
2427                 raw_spin_unlock_irq(&ctx->lock);
2428                 return;
2429         }
2430 
2431         /*
2432          * If the event is in error state, clear that first.
2433          *
2434          * That way, if we see the event in error state below, we know that it
2435          * has gone back into error state, as distinct from the task having
2436          * been scheduled away before the cross-call arrived.
2437          */
2438         if (event->state == PERF_EVENT_STATE_ERROR)
2439                 event->state = PERF_EVENT_STATE_OFF;
2440         raw_spin_unlock_irq(&ctx->lock);
2441 
2442         event_function_call(event, __perf_event_enable, NULL);
2443 }
2444 
2445 /*
2446  * See perf_event_disable();
2447  */
2448 void perf_event_enable(struct perf_event *event)
2449 {
2450         struct perf_event_context *ctx;
2451 
2452         ctx = perf_event_ctx_lock(event);
2453         _perf_event_enable(event);
2454         perf_event_ctx_unlock(event, ctx);
2455 }
2456 EXPORT_SYMBOL_GPL(perf_event_enable);
2457 
2458 struct stop_event_data {
2459         struct perf_event       *event;
2460         unsigned int            restart;
2461 };
2462 
2463 static int __perf_event_stop(void *info)
2464 {
2465         struct stop_event_data *sd = info;
2466         struct perf_event *event = sd->event;
2467 
2468         /* if it's already INACTIVE, do nothing */
2469         if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)
2470                 return 0;
2471 
2472         /* matches smp_wmb() in event_sched_in() */
2473         smp_rmb();
2474 
2475         /*
2476          * There is a window with interrupts enabled before we get here,
2477          * so we need to check again lest we try to stop another CPU's event.
2478          */
2479         if (READ_ONCE(event->oncpu) != smp_processor_id())
2480                 return -EAGAIN;
2481 
2482         event->pmu->stop(event, PERF_EF_UPDATE);
2483 
2484         /*
2485          * May race with the actual stop (through perf_pmu_output_stop()),
2486          * but it is only used for events with AUX ring buffer, and such
2487          * events will refuse to restart because of rb::aux_mmap_count==0,
2488          * see comments in perf_aux_output_begin().
2489          *
2490          * Since this is happening on a event-local CPU, no trace is lost
2491          * while restarting.
2492          */
2493         if (sd->restart)
2494                 event->pmu->start(event, PERF_EF_START);
2495 
2496         return 0;
2497 }
2498 
2499 static int perf_event_stop(struct perf_event *event, int restart)
2500 {
2501         struct stop_event_data sd = {
2502                 .event          = event,
2503                 .restart        = restart,
2504         };
2505         int ret = 0;
2506 
2507         do {
2508                 if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)
2509                         return 0;
2510 
2511                 /* matches smp_wmb() in event_sched_in() */
2512                 smp_rmb();
2513 
2514                 /*
2515                  * We only want to restart ACTIVE events, so if the event goes
2516                  * inactive here (event->oncpu==-1), there's nothing more to do;
2517                  * fall through with ret==-ENXIO.
2518                  */
2519                 ret = cpu_function_call(READ_ONCE(event->oncpu),
2520                                         __perf_event_stop, &sd);
2521         } while (ret == -EAGAIN);
2522 
2523         return ret;
2524 }
2525 
2526 /*
2527  * In order to contain the amount of racy and tricky in the address filter
2528  * configuration management, it is a two part process:
2529  *
2530  * (p1) when userspace mappings change as a result of (1) or (2) or (3) below,
2531  *      we update the addresses of corresponding vmas in
2532  *      event::addr_filters_offs array and bump the event::addr_filters_gen;
2533  * (p2) when an event is scheduled in (pmu::add), it calls
2534  *      perf_event_addr_filters_sync() which calls pmu::addr_filters_sync()
2535  *      if the generation has changed since the previous call.
2536  *
2537  * If (p1) happens while the event is active, we restart it to force (p2).
2538  *
2539  * (1) perf_addr_filters_apply(): adjusting filters' offsets based on
2540  *     pre-existing mappings, called once when new filters arrive via SET_FILTER
2541  *     ioctl;
2542  * (2) perf_addr_filters_adjust(): adjusting filters' offsets based on newly
2543  *     registered mapping, called for every new mmap(), with mm::mmap_sem down
2544  *     for reading;
2545  * (3) perf_event_addr_filters_exec(): clearing filters' offsets in the process
2546  *     of exec.
2547  */
2548 void perf_event_addr_filters_sync(struct perf_event *event)
2549 {
2550         struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);
2551 
2552         if (!has_addr_filter(event))
2553                 return;
2554 
2555         raw_spin_lock(&ifh->lock);
2556         if (event->addr_filters_gen != event->hw.addr_filters_gen) {
2557                 event->pmu->addr_filters_sync(event);
2558                 event->hw.addr_filters_gen = event->addr_filters_gen;
2559         }
2560         raw_spin_unlock(&ifh->lock);
2561 }
2562 EXPORT_SYMBOL_GPL(perf_event_addr_filters_sync);
2563 
2564 static int _perf_event_refresh(struct perf_event *event, int refresh)
2565 {
2566         /*
2567          * not supported on inherited events
2568          */
2569         if (event->attr.inherit || !is_sampling_event(event))
2570                 return -EINVAL;
2571 
2572         atomic_add(refresh, &event->event_limit);
2573         _perf_event_enable(event);
2574 
2575         return 0;
2576 }
2577 
2578 /*
2579  * See perf_event_disable()
2580  */
2581 int perf_event_refresh(struct perf_event *event, int refresh)
2582 {
2583         struct perf_event_context *ctx;
2584         int ret;
2585 
2586         ctx = perf_event_ctx_lock(event);
2587         ret = _perf_event_refresh(event, refresh);
2588         perf_event_ctx_unlock(event, ctx);
2589 
2590         return ret;
2591 }
2592 EXPORT_SYMBOL_GPL(perf_event_refresh);
2593 
2594 static void ctx_sched_out(struct perf_event_context *ctx,
2595                           struct perf_cpu_context *cpuctx,
2596                           enum event_type_t event_type)
2597 {
2598         int is_active = ctx->is_active;
2599         struct perf_event *event;
2600 
2601         lockdep_assert_held(&ctx->lock);
2602 
2603         if (likely(!ctx->nr_events)) {
2604                 /*
2605                  * See __perf_remove_from_context().
2606                  */
2607                 WARN_ON_ONCE(ctx->is_active);
2608                 if (ctx->task)
2609                         WARN_ON_ONCE(cpuctx->task_ctx);
2610                 return;
2611         }
2612 
2613         ctx->is_active &= ~event_type;
2614         if (!(ctx->is_active & EVENT_ALL))
2615                 ctx->is_active = 0;
2616 
2617         if (ctx->task) {
2618                 WARN_ON_ONCE(cpuctx->task_ctx != ctx);
2619                 if (!ctx->is_active)
2620                         cpuctx->task_ctx = NULL;
2621         }
2622 
2623         /*
2624          * Always update time if it was set; not only when it changes.
2625          * Otherwise we can 'forget' to update time for any but the last
2626          * context we sched out. For example:
2627          *
2628          *   ctx_sched_out(.event_type = EVENT_FLEXIBLE)
2629          *   ctx_sched_out(.event_type = EVENT_PINNED)
2630          *
2631          * would only update time for the pinned events.
2632          */
2633         if (is_active & EVENT_TIME) {
2634                 /* update (and stop) ctx time */
2635                 update_context_time(ctx);
2636                 update_cgrp_time_from_cpuctx(cpuctx);
2637         }
2638 
2639         is_active ^= ctx->is_active; /* changed bits */
2640 
2641         if (!ctx->nr_active || !(is_active & EVENT_ALL))
2642                 return;
2643 
2644         perf_pmu_disable(ctx->pmu);
2645         if (is_active & EVENT_PINNED) {
2646                 list_for_each_entry(event, &ctx->pinned_groups, group_entry)
2647                         group_sched_out(event, cpuctx, ctx);
2648         }
2649 
2650         if (is_active & EVENT_FLEXIBLE) {
2651                 list_for_each_entry(event, &ctx->flexible_groups, group_entry)
2652                         group_sched_out(event, cpuctx, ctx);
2653         }
2654         perf_pmu_enable(ctx->pmu);
2655 }
2656 
2657 /*
2658  * Test whether two contexts are equivalent, i.e. whether they have both been
2659  * cloned from the same version of the same context.
2660  *
2661  * Equivalence is measured using a generation number in the context that is
2662  * incremented on each modification to it; see unclone_ctx(), list_add_event()
2663  * and list_del_event().
2664  */
2665 static int context_equiv(struct perf_event_context *ctx1,
2666                          struct perf_event_context *ctx2)
2667 {
2668         lockdep_assert_held(&ctx1->lock);
2669         lockdep_assert_held(&ctx2->lock);
2670 
2671         /* Pinning disables the swap optimization */
2672         if (ctx1->pin_count || ctx2->pin_count)
2673                 return 0;
2674 
2675         /* If ctx1 is the parent of ctx2 */
2676         if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)
2677                 return 1;
2678 
2679         /* If ctx2 is the parent of ctx1 */
2680         if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)
2681                 return 1;
2682 
2683         /*
2684          * If ctx1 and ctx2 have the same parent; we flatten the parent
2685          * hierarchy, see perf_event_init_context().
2686          */
2687         if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&
2688                         ctx1->parent_gen == ctx2->parent_gen)
2689                 return 1;
2690 
2691         /* Unmatched */
2692         return 0;
2693 }
2694 
2695 static void __perf_event_sync_stat(struct perf_event *event,
2696                                      struct perf_event *next_event)
2697 {
2698         u64 value;
2699 
2700         if (!event->attr.inherit_stat)
2701                 return;
2702 
2703         /*
2704          * Update the event value, we cannot use perf_event_read()
2705          * because we're in the middle of a context switch and have IRQs
2706          * disabled, which upsets smp_call_function_single(), however
2707          * we know the event must be on the current CPU, therefore we
2708          * don't need to use it.
2709          */
2710         switch (event->state) {
2711         case PERF_EVENT_STATE_ACTIVE:
2712                 event->pmu->read(event);
2713                 /* fall-through */
2714 
2715         case PERF_EVENT_STATE_INACTIVE:
2716                 update_event_times(event);
2717                 break;
2718 
2719         default:
2720                 break;
2721         }
2722 
2723         /*
2724          * In order to keep per-task stats reliable we need to flip the event
2725          * values when we flip the contexts.
2726          */
2727         value = local64_read(&next_event->count);
2728         value = local64_xchg(&event->count, value);
2729         local64_set(&next_event->count, value);
2730 
2731         swap(event->total_time_enabled, next_event->total_time_enabled);
2732         swap(event->total_time_running, next_event->total_time_running);
2733 
2734         /*
2735          * Since we swizzled the values, update the user visible data too.
2736          */
2737         perf_event_update_userpage(event);
2738         perf_event_update_userpage(next_event);
2739 }
2740 
2741 static void perf_event_sync_stat(struct perf_event_context *ctx,
2742                                    struct perf_event_context *next_ctx)
2743 {
2744         struct perf_event *event, *next_event;
2745 
2746         if (!ctx->nr_stat)
2747                 return;
2748 
2749         update_context_time(ctx);
2750 
2751         event = list_first_entry(&ctx->event_list,
2752                                    struct perf_event, event_entry);
2753 
2754         next_event = list_first_entry(&next_ctx->event_list,
2755                                         struct perf_event, event_entry);
2756 
2757         while (&event->event_entry != &ctx->event_list &&
2758                &next_event->event_entry != &next_ctx->event_list) {
2759 
2760                 __perf_event_sync_stat(event, next_event);
2761 
2762                 event = list_next_entry(event, event_entry);
2763                 next_event = list_next_entry(next_event, event_entry);
2764         }
2765 }
2766 
2767 static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
2768                                          struct task_struct *next)
2769 {
2770         struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
2771         struct perf_event_context *next_ctx;
2772         struct perf_event_context *parent, *next_parent;
2773         struct perf_cpu_context *cpuctx;
2774         int do_switch = 1;
2775 
2776         if (likely(!ctx))
2777                 return;
2778 
2779         cpuctx = __get_cpu_context(ctx);
2780         if (!cpuctx->task_ctx)
2781                 return;
2782 
2783         rcu_read_lock();
2784         next_ctx = next->perf_event_ctxp[ctxn];
2785         if (!next_ctx)
2786                 goto unlock;
2787 
2788         parent = rcu_dereference(ctx->parent_ctx);
2789         next_parent = rcu_dereference(next_ctx->parent_ctx);
2790 
2791         /* If neither context have a parent context; they cannot be clones. */
2792         if (!parent && !next_parent)
2793                 goto unlock;
2794 
2795         if (next_parent == ctx || next_ctx == parent || next_parent == parent) {
2796                 /*
2797                  * Looks like the two contexts are clones, so we might be
2798                  * able to optimize the context switch.  We lock both
2799                  * contexts and check that they are clones under the
2800                  * lock (including re-checking that neither has been
2801                  * uncloned in the meantime).  It doesn't matter which
2802                  * order we take the locks because no other cpu could
2803                  * be trying to lock both of these tasks.
2804                  */
2805                 raw_spin_lock(&ctx->lock);
2806                 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
2807                 if (context_equiv(ctx, next_ctx)) {
2808                         WRITE_ONCE(ctx->task, next);
2809                         WRITE_ONCE(next_ctx->task, task);
2810 
2811                         swap(ctx->task_ctx_data, next_ctx->task_ctx_data);
2812 
2813                         /*
2814                          * RCU_INIT_POINTER here is safe because we've not
2815                          * modified the ctx and the above modification of
2816                          * ctx->task and ctx->task_ctx_data are immaterial
2817                          * since those values are always verified under
2818                          * ctx->lock which we're now holding.
2819                          */
2820                         RCU_INIT_POINTER(task->perf_event_ctxp[ctxn], next_ctx);
2821                         RCU_INIT_POINTER(next->perf_event_ctxp[ctxn], ctx);
2822 
2823                         do_switch = 0;
2824 
2825                         perf_event_sync_stat(ctx, next_ctx);
2826                 }
2827                 raw_spin_unlock(&next_ctx->lock);
2828                 raw_spin_unlock(&ctx->lock);
2829         }
2830 unlock:
2831         rcu_read_unlock();
2832 
2833         if (do_switch) {
2834                 raw_spin_lock(&ctx->lock);
2835                 task_ctx_sched_out(cpuctx, ctx);
2836                 raw_spin_unlock(&ctx->lock);
2837         }
2838 }
2839 
2840 void perf_sched_cb_dec(struct pmu *pmu)
2841 {
2842         this_cpu_dec(perf_sched_cb_usages);
2843 }
2844 
2845 void perf_sched_cb_inc(struct pmu *pmu)
2846 {
2847         this_cpu_inc(perf_sched_cb_usages);
2848 }
2849 
2850 /*
2851  * This function provides the context switch callback to the lower code
2852  * layer. It is invoked ONLY when the context switch callback is enabled.
2853  */
2854 static void perf_pmu_sched_task(struct task_struct *prev,
2855                                 struct task_struct *next,
2856                                 bool sched_in)
2857 {
2858         struct perf_cpu_context *cpuctx;
2859         struct pmu *pmu;
2860         unsigned long flags;
2861 
2862         if (prev == next)
2863                 return;
2864 
2865         local_irq_save(flags);
2866 
2867         rcu_read_lock();
2868 
2869         list_for_each_entry_rcu(pmu, &pmus, entry) {
2870                 if (pmu->sched_task) {
2871                         cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
2872 
2873                         perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2874 
2875                         perf_pmu_disable(pmu);
2876 
2877                         pmu->sched_task(cpuctx->task_ctx, sched_in);
2878 
2879                         perf_pmu_enable(pmu);
2880 
2881                         perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2882                 }
2883         }
2884 
2885         rcu_read_unlock();
2886 
2887         local_irq_restore(flags);
2888 }
2889 
2890 static void perf_event_switch(struct task_struct *task,
2891                               struct task_struct *next_prev, bool sched_in);
2892 
2893 #define for_each_task_context_nr(ctxn)                                  \
2894         for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
2895 
2896 /*
2897  * Called from scheduler to remove the events of the current task,
2898  * with interrupts disabled.
2899  *
2900  * We stop each event and update the event value in event->count.
2901  *
2902  * This does not protect us against NMI, but disable()
2903  * sets the disabled bit in the control field of event _before_
2904  * accessing the event control register. If a NMI hits, then it will
2905  * not restart the event.
2906  */
2907 void __perf_event_task_sched_out(struct task_struct *task,
2908                                  struct task_struct *next)
2909 {
2910         int ctxn;
2911 
2912         if (__this_cpu_read(perf_sched_cb_usages))
2913                 perf_pmu_sched_task(task, next, false);
2914 
2915         if (atomic_read(&nr_switch_events))
2916                 perf_event_switch(task, next, false);
2917 
2918         for_each_task_context_nr(ctxn)
2919                 perf_event_context_sched_out(task, ctxn, next);
2920 
2921         /*
2922          * if cgroup events exist on this CPU, then we need
2923          * to check if we have to switch out PMU state.
2924          * cgroup event are system-wide mode only
2925          */
2926         if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
2927                 perf_cgroup_sched_out(task, next);
2928 }
2929 
2930 /*
2931  * Called with IRQs disabled
2932  */
2933 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
2934                               enum event_type_t event_type)
2935 {
2936         ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
2937 }
2938 
2939 static void
2940 ctx_pinned_sched_in(struct perf_event_context *ctx,
2941                     struct perf_cpu_context *cpuctx)
2942 {
2943         struct perf_event *event;
2944 
2945         list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
2946                 if (event->state <= PERF_EVENT_STATE_OFF)
2947                         continue;
2948                 if (!event_filter_match(event))
2949                         continue;
2950 
2951                 /* may need to reset tstamp_enabled */
2952                 if (is_cgroup_event(event))
2953                         perf_cgroup_mark_enabled(event, ctx);
2954 
2955                 if (group_can_go_on(event, cpuctx, 1))
2956                         group_sched_in(event, cpuctx, ctx);
2957 
2958                 /*
2959                  * If this pinned group hasn't been scheduled,
2960                  * put it in error state.
2961                  */
2962                 if (event->state == PERF_EVENT_STATE_INACTIVE) {
2963                         update_group_times(event);
2964                         event->state = PERF_EVENT_STATE_ERROR;
2965                 }
2966         }
2967 }
2968 
2969 static void
2970 ctx_flexible_sched_in(struct perf_event_context *ctx,
2971                       struct perf_cpu_context *cpuctx)
2972 {
2973         struct perf_event *event;
2974         int can_add_hw = 1;
2975 
2976         list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
2977                 /* Ignore events in OFF or ERROR state */
2978                 if (event->state <= PERF_EVENT_STATE_OFF)
2979                         continue;
2980                 /*
2981                  * Listen to the 'cpu' scheduling filter constraint
2982                  * of events:
2983                  */
2984                 if (!event_filter_match(event))
2985                         continue;
2986 
2987                 /* may need to reset tstamp_enabled */
2988                 if (is_cgroup_event(event))
2989                         perf_cgroup_mark_enabled(event, ctx);
2990 
2991                 if (group_can_go_on(event, cpuctx, can_add_hw)) {
2992                         if (group_sched_in(event, cpuctx, ctx))
2993                                 can_add_hw = 0;
2994                 }
2995         }
2996 }
2997 
2998 static void
2999 ctx_sched_in(struct perf_event_context *ctx,
3000              struct perf_cpu_context *cpuctx,
3001              enum event_type_t event_type,
3002              struct task_struct *task)
3003 {
3004         int is_active = ctx->is_active;
3005         u64 now;
3006 
3007         lockdep_assert_held(&ctx->lock);
3008 
3009         if (likely(!ctx->nr_events))
3010                 return;
3011 
3012         ctx->is_active |= (event_type | EVENT_TIME);
3013         if (ctx->task) {
3014                 if (!is_active)
3015                         cpuctx->task_ctx = ctx;
3016                 else
3017                         WARN_ON_ONCE(cpuctx->task_ctx != ctx);
3018         }
3019 
3020         is_active ^= ctx->is_active; /* changed bits */
3021 
3022         if (is_active & EVENT_TIME) {
3023                 /* start ctx time */
3024                 now = perf_clock();
3025                 ctx->timestamp = now;
3026                 perf_cgroup_set_timestamp(task, ctx);
3027         }
3028 
3029         /*
3030          * First go through the list and put on any pinned groups
3031          * in order to give them the best chance of going on.
3032          */
3033         if (is_active & EVENT_PINNED)
3034                 ctx_pinned_sched_in(ctx, cpuctx);
3035 
3036         /* Then walk through the lower prio flexible groups */
3037         if (is_active & EVENT_FLEXIBLE)
3038                 ctx_flexible_sched_in(ctx, cpuctx);
3039 }
3040 
3041 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
3042                              enum event_type_t event_type,
3043                              struct task_struct *task)
3044 {
3045         struct perf_event_context *ctx = &cpuctx->ctx;
3046 
3047         ctx_sched_in(ctx, cpuctx, event_type, task);
3048 }
3049 
3050 static void perf_event_context_sched_in(struct perf_event_context *ctx,
3051                                         struct task_struct *task)
3052 {
3053         struct perf_cpu_context *cpuctx;
3054 
3055         cpuctx = __get_cpu_context(ctx);
3056         if (cpuctx->task_ctx == ctx)
3057                 return;
3058 
3059         perf_ctx_lock(cpuctx, ctx);
3060         perf_pmu_disable(ctx->pmu);
3061         /*
3062          * We want to keep the following priority order:
3063          * cpu pinned (that don't need to move), task pinned,
3064          * cpu flexible, task flexible.
3065          */
3066         cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
3067         perf_event_sched_in(cpuctx, ctx, task);
3068         perf_pmu_enable(ctx->pmu);
3069         perf_ctx_unlock(cpuctx, ctx);
3070 }
3071 
3072 /*
3073  * Called from scheduler to add the events of the current task
3074  * with interrupts disabled.
3075  *
3076  * We restore the event value and then enable it.
3077  *
3078  * This does not protect us against NMI, but enable()
3079  * sets the enabled bit in the control field of event _before_
3080  * accessing the event control register. If a NMI hits, then it will
3081  * keep the event running.
3082  */
3083 void __perf_event_task_sched_in(struct task_struct *prev,
3084                                 struct task_struct *task)
3085 {
3086         struct perf_event_context *ctx;
3087         int ctxn;
3088 
3089         /*
3090          * If cgroup events exist on this CPU, then we need to check if we have
3091          * to switch in PMU state; cgroup event are system-wide mode only.
3092          *
3093          * Since cgroup events are CPU events, we must schedule these in before
3094          * we schedule in the task events.
3095          */
3096         if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
3097                 perf_cgroup_sched_in(prev, task);
3098 
3099         for_each_task_context_nr(ctxn) {
3100                 ctx = task->perf_event_ctxp[ctxn];
3101                 if (likely(!ctx))
3102                         continue;
3103 
3104                 perf_event_context_sched_in(ctx, task);
3105         }
3106 
3107         if (atomic_read(&nr_switch_events))
3108                 perf_event_switch(task, prev, true);
3109 
3110         if (__this_cpu_read(perf_sched_cb_usages))
3111                 perf_pmu_sched_task(prev, task, true);
3112 }
3113 
3114 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
3115 {
3116         u64 frequency = event->attr.sample_freq;
3117         u64 sec = NSEC_PER_SEC;
3118         u64 divisor, dividend;
3119 
3120         int count_fls, nsec_fls, frequency_fls, sec_fls;
3121 
3122         count_fls = fls64(count);
3123         nsec_fls = fls64(nsec);
3124         frequency_fls = fls64(frequency);
3125         sec_fls = 30;
3126 
3127         /*
3128          * We got @count in @nsec, with a target of sample_freq HZ
3129          * the target period becomes:
3130          *
3131          *             @count * 10^9
3132          * period = -------------------
3133          *          @nsec * sample_freq
3134          *
3135          */
3136 
3137         /*
3138          * Reduce accuracy by one bit such that @a and @b converge
3139          * to a similar magnitude.
3140          */
3141 #define REDUCE_FLS(a, b)                \
3142 do {                                    \
3143         if (a##_fls > b##_fls) {        \
3144                 a >>= 1;                \
3145                 a##_fls--;              \
3146         } else {                        \
3147                 b >>= 1;                \
3148                 b##_fls--;              \
3149         }                               \
3150 } while (0)
3151 
3152         /*
3153          * Reduce accuracy until either term fits in a u64, then proceed with
3154          * the other, so that finally we can do a u64/u64 division.
3155          */
3156         while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
3157                 REDUCE_FLS(nsec, frequency);
3158                 REDUCE_FLS(sec, count);
3159         }
3160 
3161         if (count_fls + sec_fls > 64) {
3162                 divisor = nsec * frequency;
3163 
3164                 while (count_fls + sec_fls > 64) {
3165                         REDUCE_FLS(count, sec);
3166                         divisor >>= 1;
3167                 }
3168 
3169                 dividend = count * sec;
3170         } else {
3171                 dividend = count * sec;
3172 
3173                 while (nsec_fls + frequency_fls > 64) {
3174                         REDUCE_FLS(nsec, frequency);
3175                         dividend >>= 1;
3176                 }
3177 
3178                 divisor = nsec * frequency;
3179         }
3180 
3181         if (!divisor)
3182                 return dividend;
3183 
3184         return div64_u64(dividend, divisor);
3185 }
3186 
3187 static DEFINE_PER_CPU(int, perf_throttled_count);
3188 static DEFINE_PER_CPU(u64, perf_throttled_seq);
3189 
3190 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
3191 {
3192         struct hw_perf_event *hwc = &event->hw;
3193         s64 period, sample_period;
3194         s64 delta;
3195 
3196         period = perf_calculate_period(event, nsec, count);
3197 
3198         delta = (s64)(period - hwc->sample_period);
3199         delta = (delta + 7) / 8; /* low pass filter */
3200 
3201         sample_period = hwc->sample_period + delta;
3202 
3203         if (!sample_period)
3204                 sample_period = 1;
3205 
3206         hwc->sample_period = sample_period;
3207 
3208         if (local64_read(&hwc->period_left) > 8*sample_period) {
3209                 if (disable)
3210                         event->pmu->stop(event, PERF_EF_UPDATE);
3211 
3212                 local64_set(&hwc->period_left, 0);
3213 
3214                 if (disable)
3215                         event->pmu->start(event, PERF_EF_RELOAD);
3216         }
3217 }
3218 
3219 /*
3220  * combine freq adjustment with unthrottling to avoid two passes over the
3221  * events. At the same time, make sure, having freq events does not change
3222  * the rate of unthrottling as that would introduce bias.
3223  */
3224 static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
3225                                            int needs_unthr)
3226 {
3227         struct perf_event *event;
3228         struct hw_perf_event *hwc;
3229         u64 now, period = TICK_NSEC;
3230         s64 delta;
3231 
3232         /*
3233          * only need to iterate over all events iff:
3234          * - context have events in frequency mode (needs freq adjust)
3235          * - there are events to unthrottle on this cpu
3236          */
3237         if (!(ctx->nr_freq || needs_unthr))
3238                 return;
3239 
3240         raw_spin_lock(&ctx->lock);
3241         perf_pmu_disable(ctx->pmu);
3242 
3243         list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
3244                 if (event->state != PERF_EVENT_STATE_ACTIVE)
3245                         continue;
3246 
3247                 if (!event_filter_match(event))
3248                         continue;
3249 
3250                 perf_pmu_disable(event->pmu);
3251 
3252                 hwc = &event->hw;
3253 
3254                 if (hwc->interrupts == MAX_INTERRUPTS) {
3255                         hwc->interrupts = 0;
3256                         perf_log_throttle(event, 1);
3257                         event->pmu->start(event, 0);
3258                 }
3259 
3260                 if (!event->attr.freq || !event->attr.sample_freq)
3261                         goto next;
3262 
3263                 /*
3264                  * stop the event and update event->count
3265                  */
3266                 event->pmu->stop(event, PERF_EF_UPDATE);
3267 
3268                 now = local64_read(&event->count);
3269                 delta = now - hwc->freq_count_stamp;
3270                 hwc->freq_count_stamp = now;
3271 
3272                 /*
3273                  * restart the event
3274                  * reload only if value has changed
3275                  * we have stopped the event so tell that
3276                  * to perf_adjust_period() to avoid stopping it
3277                  * twice.
3278                  */
3279                 if (delta > 0)
3280                         perf_adjust_period(event, period, delta, false);
3281 
3282                 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
3283         next:
3284                 perf_pmu_enable(event->pmu);
3285         }
3286 
3287         perf_pmu_enable(ctx->pmu);
3288         raw_spin_unlock(&ctx->lock);
3289 }
3290 
3291 /*
3292  * Round-robin a context's events:
3293  */
3294 static void rotate_ctx(struct perf_event_context *ctx)
3295 {
3296         /*
3297          * Rotate the first entry last of non-pinned groups. Rotation might be
3298          * disabled by the inheritance code.
3299          */
3300         if (!ctx->rotate_disable)
3301                 list_rotate_left(&ctx->flexible_groups);
3302 }
3303 
3304 static int perf_rotate_context(struct perf_cpu_context *cpuctx)
3305 {
3306         struct perf_event_context *ctx = NULL;
3307         int rotate = 0;
3308 
3309         if (cpuctx->ctx.nr_events) {
3310                 if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
3311                         rotate = 1;
3312         }
3313 
3314         ctx = cpuctx->task_ctx;
3315         if (ctx && ctx->nr_events) {
3316                 if (ctx->nr_events != ctx->nr_active)
3317                         rotate = 1;
3318         }
3319 
3320         if (!rotate)
3321                 goto done;
3322 
3323         perf_ctx_lock(cpuctx, cpuctx->task_ctx);
3324         perf_pmu_disable(cpuctx->ctx.pmu);
3325 
3326         cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
3327         if (ctx)
3328                 ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
3329 
3330         rotate_ctx(&cpuctx->ctx);
3331         if (ctx)
3332                 rotate_ctx(ctx);
3333 
3334         perf_event_sched_in(cpuctx, ctx, current);
3335 
3336         perf_pmu_enable(cpuctx->ctx.pmu);
3337         perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
3338 done:
3339 
3340         return rotate;
3341 }
3342 
3343 void perf_event_task_tick(void)
3344 {
3345         struct list_head *head = this_cpu_ptr(&active_ctx_list);
3346         struct perf_event_context *ctx, *tmp;
3347         int throttled;
3348 
3349         WARN_ON(!irqs_disabled());
3350 
3351         __this_cpu_inc(perf_throttled_seq);
3352         throttled = __this_cpu_xchg(perf_throttled_count, 0);
3353         tick_dep_clear_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS);
3354 
3355         list_for_each_entry_safe(ctx, tmp, head, active_ctx_list)
3356                 perf_adjust_freq_unthr_context(ctx, throttled);
3357 }
3358 
3359 static int event_enable_on_exec(struct perf_event *event,
3360                                 struct perf_event_context *ctx)
3361 {
3362         if (!event->attr.enable_on_exec)
3363                 return 0;
3364 
3365         event->attr.enable_on_exec = 0;
3366         if (event->state >= PERF_EVENT_STATE_INACTIVE)
3367                 return 0;
3368 
3369         __perf_event_mark_enabled(event);
3370 
3371         return 1;
3372 }
3373 
3374 /*
3375  * Enable all of a task's events that have been marked enable-on-exec.
3376  * This expects task == current.
3377  */
3378 static void perf_event_enable_on_exec(int ctxn)
3379 {
3380         struct perf_event_context *ctx, *clone_ctx = NULL;
3381         struct perf_cpu_context *cpuctx;
3382         struct perf_event *event;
3383         unsigned long flags;
3384         int enabled = 0;
3385 
3386         local_irq_save(flags);
3387         ctx = current->perf_event_ctxp[ctxn];
3388         if (!ctx || !ctx->nr_events)
3389                 goto out;
3390 
3391         cpuctx = __get_cpu_context(ctx);
3392         perf_ctx_lock(cpuctx, ctx);
3393         ctx_sched_out(ctx, cpuctx, EVENT_TIME);
3394         list_for_each_entry(event, &ctx->event_list, event_entry)
3395                 enabled |= event_enable_on_exec(event, ctx);
3396 
3397         /*
3398          * Unclone and reschedule this context if we enabled any event.
3399          */
3400         if (enabled) {
3401                 clone_ctx = unclone_ctx(ctx);
3402                 ctx_resched(cpuctx, ctx);
3403         }
3404         perf_ctx_unlock(cpuctx, ctx);
3405 
3406 out:
3407         local_irq_restore(flags);
3408 
3409         if (clone_ctx)
3410                 put_ctx(clone_ctx);
3411 }
3412 
3413 struct perf_read_data {
3414         struct perf_event *event;
3415         bool group;
3416         int ret;
3417 };
3418 
3419 /*
3420  * Cross CPU call to read the hardware event
3421  */
3422 static void __perf_event_read(void *info)
3423 {
3424         struct perf_read_data *data = info;
3425         struct perf_event *sub, *event = data->event;
3426         struct perf_event_context *ctx = event->ctx;
3427         struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
3428         struct pmu *pmu = event->pmu;
3429 
3430         /*
3431          * If this is a task context, we need to check whether it is
3432          * the current task context of this cpu.  If not it has been
3433          * scheduled out before the smp call arrived.  In that case
3434          * event->count would have been updated to a recent sample
3435          * when the event was scheduled out.
3436          */
3437         if (ctx->task && cpuctx->task_ctx != ctx)
3438                 return;
3439 
3440         raw_spin_lock(&ctx->lock);
3441         if (ctx->is_active) {
3442                 update_context_time(ctx);
3443                 update_cgrp_time_from_event(event);
3444         }
3445 
3446         update_event_times(event);
3447         if (event->state != PERF_EVENT_STATE_ACTIVE)
3448                 goto unlock;
3449 
3450         if (!data->group) {
3451                 pmu->read(event);
3452                 data->ret = 0;
3453                 goto unlock;
3454         }
3455 
3456         pmu->start_txn(pmu, PERF_PMU_TXN_READ);
3457 
3458         pmu->read(event);
3459 
3460         list_for_each_entry(sub, &event->sibling_list, group_entry) {
3461                 update_event_times(sub);
3462                 if (sub->state == PERF_EVENT_STATE_ACTIVE) {
3463                         /*
3464                          * Use sibling's PMU rather than @event's since
3465                          * sibling could be on different (eg: software) PMU.
3466                          */
3467                         sub->pmu->read(sub);
3468                 }
3469         }
3470 
3471         data->ret = pmu->commit_txn(pmu);
3472 
3473 unlock:
3474         raw_spin_unlock(&ctx->lock);
3475 }
3476 
3477 static inline u64 perf_event_count(struct perf_event *event)
3478 {
3479         if (event->pmu->count)
3480                 return event->pmu->count(event);
3481 
3482         return __perf_event_count(event);
3483 }
3484 
3485 /*
3486  * NMI-safe method to read a local event, that is an event that
3487  * is:
3488  *   - either for the current task, or for this CPU
3489  *   - does not have inherit set, for inherited task events
3490  *     will not be local and we cannot read them atomically
3491  *   - must not have a pmu::count method
3492  */
3493 u64 perf_event_read_local(struct perf_event *event)
3494 {
3495         unsigned long flags;
3496         u64 val;
3497 
3498         /*
3499          * Disabling interrupts avoids all counter scheduling (context
3500          * switches, timer based rotation and IPIs).
3501          */
3502         local_irq_save(flags);
3503 
3504         /* If this is a per-task event, it must be for current */
3505         WARN_ON_ONCE((event->attach_state & PERF_ATTACH_TASK) &&
3506                      event->hw.target != current);
3507 
3508         /* If this is a per-CPU event, it must be for this CPU */
3509         WARN_ON_ONCE(!(event->attach_state & PERF_ATTACH_TASK) &&
3510                      event->cpu != smp_processor_id());
3511 
3512         /*
3513          * It must not be an event with inherit set, we cannot read
3514          * all child counters from atomic context.
3515          */
3516         WARN_ON_ONCE(event->attr.inherit);
3517 
3518         /*
3519          * It must not have a pmu::count method, those are not
3520          * NMI safe.
3521          */
3522         WARN_ON_ONCE(event->pmu->count);
3523 
3524         /*
3525          * If the event is currently on this CPU, its either a per-task event,
3526          * or local to this CPU. Furthermore it means its ACTIVE (otherwise
3527          * oncpu == -1).
3528          */
3529         if (event->oncpu == smp_processor_id())
3530                 event->pmu->read(event);
3531 
3532         val = local64_read(&event->count);
3533         local_irq_restore(flags);
3534 
3535         return val;
3536 }
3537 
3538 static int perf_event_read(struct perf_event *event, bool group)
3539 {
3540         int ret = 0;
3541 
3542         /*
3543          * If event is enabled and currently active on a CPU, update the
3544          * value in the event structure:
3545          */
3546         if (event->state == PERF_EVENT_STATE_ACTIVE) {
3547                 struct perf_read_data data = {
3548                         .event = event,
3549                         .group = group,
3550                         .ret = 0,
3551                 };
3552                 /*
3553                  * Purposely ignore the smp_call_function_single() return
3554                  * value.
3555                  *
3556                  * If event->oncpu isn't a valid CPU it means the event got
3557                  * scheduled out and that will have updated the event count.
3558                  *
3559                  * Therefore, either way, we'll have an up-to-date event count
3560                  * after this.
3561                  */
3562                 (void)smp_call_function_single(event->oncpu, __perf_event_read, &data, 1);
3563                 ret = data.ret;
3564         } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
3565                 struct perf_event_context *ctx = event->ctx;
3566                 unsigned long flags;
3567 
3568                 raw_spin_lock_irqsave(&ctx->lock, flags);
3569                 /*
3570                  * may read while context is not active
3571                  * (e.g., thread is blocked), in that case
3572                  * we cannot update context time
3573                  */
3574                 if (ctx->is_active) {
3575                         update_context_time(ctx);
3576                         update_cgrp_time_from_event(event);
3577                 }
3578                 if (group)
3579                         update_group_times(event);
3580                 else
3581                         update_event_times(event);
3582                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
3583         }
3584 
3585         return ret;
3586 }
3587 
3588 /*
3589  * Initialize the perf_event context in a task_struct:
3590  */
3591 static void __perf_event_init_context(struct perf_event_context *ctx)
3592 {
3593         raw_spin_lock_init(&ctx->lock);
3594         mutex_init(&ctx->mutex);
3595         INIT_LIST_HEAD(&ctx->active_ctx_list);
3596         INIT_LIST_HEAD(&ctx->pinned_groups);
3597         INIT_LIST_HEAD(&ctx->flexible_groups);
3598         INIT_LIST_HEAD(&ctx->event_list);
3599         atomic_set(&ctx->refcount, 1);
3600 }
3601 
3602 static struct perf_event_context *
3603 alloc_perf_context(struct pmu *pmu, struct task_struct *task)
3604 {
3605         struct perf_event_context *ctx;
3606 
3607         ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
3608         if (!ctx)
3609                 return NULL;
3610 
3611         __perf_event_init_context(ctx);
3612         if (task) {
3613                 ctx->task = task;
3614                 get_task_struct(task);
3615         }
3616         ctx->pmu = pmu;
3617 
3618         return ctx;
3619 }
3620 
3621 static struct task_struct *
3622 find_lively_task_by_vpid(pid_t vpid)
3623 {
3624         struct task_struct *task;
3625 
3626         rcu_read_lock();
3627         if (!vpid)
3628                 task = current;
3629         else
3630                 task = find_task_by_vpid(vpid);
3631         if (task)
3632                 get_task_struct(task);
3633         rcu_read_unlock();
3634 
3635         if (!task)
3636                 return ERR_PTR(-ESRCH);
3637 
3638         return task;
3639 }
3640 
3641 /*
3642  * Returns a matching context with refcount and pincount.
3643  */
3644 static struct perf_event_context *
3645 find_get_context(struct pmu *pmu, struct task_struct *task,
3646                 struct perf_event *event)
3647 {
3648         struct perf_event_context *ctx, *clone_ctx = NULL;
3649         struct perf_cpu_context *cpuctx;
3650         void *task_ctx_data = NULL;
3651         unsigned long flags;
3652         int ctxn, err;
3653         int cpu = event->cpu;
3654 
3655         if (!task) {
3656                 /* Must be root to operate on a CPU event: */
3657                 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
3658                         return ERR_PTR(-EACCES);
3659 
3660                 /*
3661                  * We could be clever and allow to attach a event to an
3662                  * offline CPU and activate it when the CPU comes up, but
3663                  * that's for later.
3664                  */
3665                 if (!cpu_online(cpu))
3666                         return ERR_PTR(-ENODEV);
3667 
3668                 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
3669                 ctx = &cpuctx->ctx;
3670                 get_ctx(ctx);
3671                 ++ctx->pin_count;
3672 
3673                 return ctx;
3674         }
3675 
3676         err = -EINVAL;
3677         ctxn = pmu->task_ctx_nr;
3678         if (ctxn < 0)
3679                 goto errout;
3680 
3681         if (event->attach_state & PERF_ATTACH_TASK_DATA) {
3682                 task_ctx_data = kzalloc(pmu->task_ctx_size, GFP_KERNEL);
3683                 if (!task_ctx_data) {
3684                         err = -ENOMEM;
3685                         goto errout;
3686                 }
3687         }
3688 
3689 retry:
3690         ctx = perf_lock_task_context(task, ctxn, &flags);
3691         if (ctx) {
3692                 clone_ctx = unclone_ctx(ctx);
3693                 ++ctx->pin_count;
3694 
3695                 if (task_ctx_data && !ctx->task_ctx_data) {
3696                         ctx->task_ctx_data = task_ctx_data;
3697                         task_ctx_data = NULL;
3698                 }
3699                 raw_spin_unlock_irqrestore(&ctx->lock, flags);
3700 
3701                 if (clone_ctx)
3702                         put_ctx(clone_ctx);
3703         } else {
3704                 ctx = alloc_perf_context(pmu, task);
3705                 err = -ENOMEM;
3706                 if (!ctx)
3707                         goto errout;
3708 
3709                 if (task_ctx_data) {
3710                         ctx->task_ctx_data = task_ctx_data;
3711                         task_ctx_data = NULL;
3712                 }
3713 
3714                 err = 0;
3715                 mutex_lock(&task->perf_event_mutex);
3716                 /*
3717                  * If it has already passed perf_event_exit_task().
3718                  * we must see PF_EXITING, it takes this mutex too.
3719                  */
3720                 if (task->flags & PF_EXITING)
3721                         err = -ESRCH;
3722                 else if (task->perf_event_ctxp[ctxn])
3723                         err = -EAGAIN;
3724                 else {
3725                         get_ctx(ctx);
3726                         ++ctx->pin_count;
3727                         rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
3728                 }
3729                 mutex_unlock(&task->perf_event_mutex);
3730 
3731                 if (unlikely(err)) {
3732                         put_ctx(ctx);
3733 
3734                         if (err == -EAGAIN)
3735                                 goto retry;
3736                         goto errout;
3737                 }
3738         }
3739 
3740         kfree(task_ctx_data);
3741         return ctx;
3742 
3743 errout:
3744         kfree(task_ctx_data);
3745         return ERR_PTR(err);
3746 }
3747 
3748 static void perf_event_free_filter(struct perf_event *event);
3749 static void perf_event_free_bpf_prog(struct perf_event *event);
3750 
3751 static void free_event_rcu(struct rcu_head *head)
3752 {
3753         struct perf_event *event;
3754 
3755         event = container_of(head, struct perf_event, rcu_head);
3756         if (event->ns)
3757                 put_pid_ns(event->ns);
3758         perf_event_free_filter(event);
3759         kfree(event);
3760 }
3761 
3762 static void ring_buffer_attach(struct perf_event *event,
3763                                struct ring_buffer *rb);
3764 
3765 static void detach_sb_event(struct perf_event *event)
3766 {
3767         struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);
3768 
3769         raw_spin_lock(&pel->lock);
3770         list_del_rcu(&event->sb_list);
3771         raw_spin_unlock(&pel->lock);
3772 }
3773 
3774 static bool is_sb_event(struct perf_event *event)
3775 {
3776         struct perf_event_attr *attr = &event->attr;
3777 
3778         if (event->parent)
3779                 return false;
3780 
3781         if (event->attach_state & PERF_ATTACH_TASK)
3782                 return false;
3783 
3784         if (attr->mmap || attr->mmap_data || attr->mmap2 ||
3785             attr->comm || attr->comm_exec ||
3786             attr->task ||
3787             attr->context_switch)
3788                 return true;
3789         return false;
3790 }
3791 
3792 static void unaccount_pmu_sb_event(struct perf_event *event)
3793 {
3794         if (is_sb_event(event))
3795                 detach_sb_event(event);
3796 }
3797 
3798 static void unaccount_event_cpu(struct perf_event *event, int cpu)
3799 {
3800         if (event->parent)
3801                 return;
3802 
3803         if (is_cgroup_event(event))
3804                 atomic_dec(&per_cpu(perf_cgroup_events, cpu));
3805 }
3806 
3807 #ifdef CONFIG_NO_HZ_FULL
3808 static DEFINE_SPINLOCK(nr_freq_lock);
3809 #endif
3810 
3811 static void unaccount_freq_event_nohz(void)
3812 {
3813 #ifdef CONFIG_NO_HZ_FULL
3814         spin_lock(&nr_freq_lock);
3815         if (atomic_dec_and_test(&nr_freq_events))
3816                 tick_nohz_dep_clear(TICK_DEP_BIT_PERF_EVENTS);
3817         spin_unlock(&nr_freq_lock);
3818 #endif
3819 }
3820 
3821 static void unaccount_freq_event(void)
3822 {
3823         if (tick_nohz_full_enabled())
3824                 unaccount_freq_event_nohz();
3825         else
3826                 atomic_dec(&nr_freq_events);
3827 }
3828 
3829 static void unaccount_event(struct perf_event *event)
3830 {
3831         bool dec = false;
3832 
3833         if (event->parent)
3834                 return;
3835 
3836         if (event->attach_state & PERF_ATTACH_TASK)
3837                 dec = true;
3838         if (event->attr.mmap || event->attr.mmap_data)
3839                 atomic_dec(&nr_mmap_events);
3840         if (event->attr.comm)
3841                 atomic_dec(&nr_comm_events);
3842         if (event->attr.task)
3843                 atomic_dec(&nr_task_events);
3844         if (event->attr.freq)
3845                 unaccount_freq_event();
3846         if (event->attr.context_switch) {
3847                 dec = true;
3848                 atomic_dec(&nr_switch_events);
3849         }
3850         if (is_cgroup_event(event))
3851                 dec = true;
3852         if (has_branch_stack(event))
3853                 dec = true;
3854 
3855         if (dec) {
3856                 if (!atomic_add_unless(&perf_sched_count, -1, 1))
3857                         schedule_delayed_work(&perf_sched_work, HZ);
3858         }
3859 
3860         unaccount_event_cpu(event, event->cpu);
3861 
3862         unaccount_pmu_sb_event(event);
3863 }
3864 
3865 static void perf_sched_delayed(struct work_struct *work)
3866 {
3867         mutex_lock(&perf_sched_mutex);
3868         if (atomic_dec_and_test(&perf_sched_count))
3869                 static_branch_disable(&perf_sched_events);
3870         mutex_unlock(&perf_sched_mutex);
3871 }
3872 
3873 /*
3874  * The following implement mutual exclusion of events on "exclusive" pmus
3875  * (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled
3876  * at a time, so we disallow creating events that might conflict, namely:
3877  *
3878  *  1) cpu-wide events in the presence of per-task events,
3879  *  2) per-task events in the presence of cpu-wide events,
3880  *  3) two matching events on the same context.
3881  *
3882  * The former two cases are handled in the allocation path (perf_event_alloc(),
3883  * _free_event()), the latter -- before the first perf_install_in_context().
3884  */
3885 static int exclusive_event_init(struct perf_event *event)
3886 {
3887         struct pmu *pmu = event->pmu;
3888 
3889         if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE))
3890                 return 0;
3891 
3892         /*
3893          * Prevent co-existence of per-task and cpu-wide events on the
3894          * same exclusive pmu.
3895          *
3896          * Negative pmu::exclusive_cnt means there are cpu-wide
3897          * events on this "exclusive" pmu, positive means there are
3898          * per-task events.
3899          *
3900          * Since this is called in perf_event_alloc() path, event::ctx
3901          * doesn't exist yet; it is, however, safe to use PERF_ATTACH_TASK
3902          * to mean "per-task event", because unlike other attach states it
3903          * never gets cleared.
3904          */
3905         if (event->attach_state & PERF_ATTACH_TASK) {
3906                 if (!atomic_inc_unless_negative(&pmu->exclusive_cnt))
3907                         return -EBUSY;
3908         } else {
3909                 if (!atomic_dec_unless_positive(&pmu->exclusive_cnt))
3910                         return -EBUSY;
3911         }
3912 
3913         return 0;
3914 }
3915 
3916 static void exclusive_event_destroy(struct perf_event *event)
3917 {
3918         struct pmu *pmu = event->pmu;
3919 
3920         if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE))
3921                 return;
3922 
3923         /* see comment in exclusive_event_init() */
3924         if (event->attach_state & PERF_ATTACH_TASK)
3925                 atomic_dec(&pmu->exclusive_cnt);
3926         else
3927                 atomic_inc(&pmu->exclusive_cnt);
3928 }
3929 
3930 static bool exclusive_event_match(struct perf_event *e1, struct perf_event *e2)
3931 {
3932         if ((e1->pmu == e2->pmu) &&
3933             (e1->cpu == e2->cpu ||
3934              e1->cpu == -1 ||
3935              e2->cpu == -1))
3936                 return true;
3937         return false;
3938 }
3939 
3940 /* Called under the same ctx::mutex as perf_install_in_context() */
3941 static bool exclusive_event_installable(struct perf_event *event,
3942                                         struct perf_event_context *ctx)
3943 {
3944         struct perf_event *iter_event;
3945         struct pmu *pmu = event->pmu;
3946 
3947         if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE))
3948                 return true;
3949 
3950         list_for_each_entry(iter_event, &ctx->event_list, event_entry) {
3951                 if (exclusive_event_match(iter_event, event))
3952                         return false;
3953         }
3954 
3955         return true;
3956 }
3957 
3958 static void perf_addr_filters_splice(struct perf_event *event,
3959                                        struct list_head *head);
3960 
3961 static void _free_event(struct perf_event *event)
3962 {
3963         irq_work_sync(&event->pending);
3964 
3965         unaccount_event(event);
3966 
3967         if (event->rb) {
3968                 /*
3969                  * Can happen when we close an event with re-directed output.
3970                  *
3971                  * Since we have a 0 refcount, perf_mmap_close() will skip
3972                  * over us; possibly making our ring_buffer_put() the last.
3973                  */
3974                 mutex_lock(&event->mmap_mutex);
3975                 ring_buffer_attach(event, NULL);
3976                 mutex_unlock(&event->mmap_mutex);
3977         }
3978 
3979         if (is_cgroup_event(event))
3980                 perf_detach_cgroup(event);
3981 
3982         if (!event->parent) {
3983                 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
3984                         put_callchain_buffers();
3985         }
3986 
3987         perf_event_free_bpf_prog(event);
3988         perf_addr_filters_splice(event, NULL);
3989         kfree(event->addr_filters_offs);
3990 
3991         if (event->destroy)
3992                 event->destroy(event);
3993 
3994         if (event->ctx)
3995                 put_ctx(event->ctx);
3996 
3997         exclusive_event_destroy(event);
3998         module_put(event->pmu->module);
3999 
4000         call_rcu(&event->rcu_head, free_event_rcu);
4001 }
4002 
4003 /*
4004  * Used to free events which have a known refcount of 1, such as in error paths
4005  * where the event isn't exposed yet and inherited events.
4006  */
4007 static void free_event(struct perf_event *event)
4008 {
4009         if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,
4010                                 "unexpected event refcount: %ld; ptr=%p\n",
4011                                 atomic_long_read(&event->refcount), event)) {
4012                 /* leak to avoid use-after-free */
4013                 return;
4014         }
4015 
4016         _free_event(event);
4017 }
4018 
4019 /*
4020  * Remove user event from the owner task.
4021  */
4022 static void perf_remove_from_owner(struct perf_event *event)
4023 {
4024         struct task_struct *owner;
4025 
4026         rcu_read_lock();
4027         /*
4028          * Matches the smp_store_release() in perf_event_exit_task(). If we
4029          * observe !owner it means the list deletion is complete and we can
4030          * indeed free this event, otherwise we need to serialize on
4031          * owner->perf_event_mutex.
4032          */
4033         owner = lockless_dereference(event->owner);
4034         if (owner) {
4035                 /*
4036                  * Since delayed_put_task_struct() also drops the last
4037                  * task reference we can safely take a new reference
4038                  * while holding the rcu_read_lock().
4039                  */
4040                 get_task_struct(owner);
4041         }
4042         rcu_read_unlock();
4043 
4044         if (owner) {
4045                 /*
4046                  * If we're here through perf_event_exit_task() we're already
4047                  * holding ctx->mutex which would be an inversion wrt. the
4048                  * normal lock order.
4049                  *
4050                  * However we can safely take this lock because its the child
4051                  * ctx->mutex.
4052                  */
4053                 mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING);
4054 
4055                 /*
4056                  * We have to re-check the event->owner field, if it is cleared
4057                  * we raced with perf_event_exit_task(), acquiring the mutex
4058                  * ensured they're done, and we can proceed with freeing the
4059                  * event.
4060                  */
4061                 if (event->owner) {
4062                         list_del_init(&event->owner_entry);
4063                         smp_store_release(&event->owner, NULL);
4064                 }
4065                 mutex_unlock(&owner->perf_event_mutex);
4066                 put_task_struct(owner);
4067         }
4068 }
4069 
4070 static void put_event(struct perf_event *event)
4071 {
4072         if (!atomic_long_dec_and_test(&event->refcount))
4073                 return;
4074 
4075         _free_event(event);
4076 }
4077 
4078 /*
4079  * Kill an event dead; while event:refcount will preserve the event
4080  * object, it will not preserve its functionality. Once the last 'user'
4081  * gives up the object, we'll destroy the thing.
4082  */
4083 int perf_event_release_kernel(struct perf_event *event)
4084 {
4085         struct perf_event_context *ctx = event->ctx;
4086         struct perf_event *child, *tmp;
4087 
4088         /*
4089          * If we got here through err_file: fput(event_file); we will not have
4090          * attached to a context yet.
4091          */
4092         if (!ctx) {
4093                 WARN_ON_ONCE(event->attach_state &
4094                                 (PERF_ATTACH_CONTEXT|PERF_ATTACH_GROUP));
4095                 goto no_ctx;
4096         }
4097 
4098         if (!is_kernel_event(event))
4099                 perf_remove_from_owner(event);
4100 
4101         ctx = perf_event_ctx_lock(event);
4102         WARN_ON_ONCE(ctx->parent_ctx);
4103         perf_remove_from_context(event, DETACH_GROUP);
4104 
4105         raw_spin_lock_irq(&ctx->lock);
4106         /*
4107          * Mark this even as STATE_DEAD, there is no external reference to it
4108          * anymore.
4109          *
4110          * Anybody acquiring event->child_mutex after the below loop _must_
4111          * also see this, most importantly inherit_event() which will avoid
4112          * placing more children on the list.
4113          *
4114          * Thus this guarantees that we will in fact observe and kill _ALL_
4115          * child events.
4116          */
4117         event->state = PERF_EVENT_STATE_DEAD;
4118         raw_spin_unlock_irq(&ctx->lock);
4119 
4120         perf_event_ctx_unlock(event, ctx);
4121 
4122 again:
4123         mutex_lock(&event->child_mutex);
4124         list_for_each_entry(child, &event->child_list, child_list) {
4125 
4126                 /*
4127                  * Cannot change, child events are not migrated, see the
4128                  * comment with perf_event_ctx_lock_nested().
4129                  */
4130                 ctx = lockless_dereference(child->ctx);
4131                 /*
4132                  * Since child_mutex nests inside ctx::mutex, we must jump
4133                  * through hoops. We start by grabbing a reference on the ctx.
4134                  *
4135                  * Since the event cannot get freed while we hold the
4136                  * child_mutex, the context must also exist and have a !0
4137                  * reference count.
4138                  */
4139                 get_ctx(ctx);
4140 
4141                 /*
4142                  * Now that we have a ctx ref, we can drop child_mutex, and
4143                  * acquire ctx::mutex without fear of it going away. Then we
4144                  * can re-acquire child_mutex.
4145                  */
4146                 mutex_unlock(&event->child_mutex);
4147                 mutex_lock(&ctx->mutex);
4148                 mutex_lock(&event->child_mutex);
4149 
4150                 /*
4151                  * Now that we hold ctx::mutex and child_mutex, revalidate our
4152                  * state, if child is still the first entry, it didn't get freed
4153                  * and we can continue doing so.
4154                  */
4155                 tmp = list_first_entry_or_null(&event->child_list,
4156                                                struct perf_event, child_list);
4157                 if (tmp == child) {
4158                         perf_remove_from_context(child, DETACH_GROUP);
4159                         list_del(&child->child_list);
4160                         free_event(child);
4161                         /*
4162                          * This matches the refcount bump in inherit_event();
4163                          * this can't be the last reference.
4164                          */
4165                         put_event(event);
4166                 }
4167 
4168                 mutex_unlock(&event->child_mutex);
4169                 mutex_unlock(&ctx->mutex);
4170                 put_ctx(ctx);
4171                 goto again;
4172         }
4173         mutex_unlock(&event->child_mutex);
4174 
4175 no_ctx:
4176         put_event(event); /* Must be the 'last' reference */
4177         return 0;
4178 }
4179 EXPORT_SYMBOL_GPL(perf_event_release_kernel);
4180 
4181 /*
4182  * Called when the last reference to the file is gone.
4183  */
4184 static int perf_release(struct inode *inode, struct file *file)
4185 {
4186         perf_event_release_kernel(file->private_data);
4187         return 0;
4188 }
4189 
4190 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
4191 {
4192         struct perf_event *child;
4193         u64 total = 0;
4194 
4195         *enabled = 0;
4196         *running = 0;
4197 
4198         mutex_lock(&event->child_mutex);
4199 
4200         (void)perf_event_read(event, false);
4201         total += perf_event_count(event);
4202 
4203         *enabled += event->total_time_enabled +
4204                         atomic64_read(&event->child_total_time_enabled);
4205         *running += event->total_time_running +
4206                         atomic64_read(&event->child_total_time_running);
4207 
4208         list_for_each_entry(child, &event->child_list, child_list) {
4209                 (void)perf_event_read(child, false);
4210                 total += perf_event_count(child);
4211                 *enabled += child->total_time_enabled;
4212                 *running += child->total_time_running;
4213         }
4214         mutex_unlock(&event->child_mutex);
4215 
4216         return total;
4217 }
4218 EXPORT_SYMBOL_GPL(perf_event_read_value);
4219 
4220 static int __perf_read_group_add(struct perf_event *leader,
4221                                         u64 read_format, u64 *values)
4222 {
4223         struct perf_event *sub;
4224         int n = 1; /* skip @nr */
4225         int ret;
4226 
4227         ret = perf_event_read(leader, true);
4228         if (ret)
4229                 return ret;
4230 
4231         /*
4232          * Since we co-schedule groups, {enabled,running} times of siblings
4233          * will be identical to those of the leader, so we only publish one
4234          * set.
4235          */
4236         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
4237                 values[n++] += leader->total_time_enabled +
4238                         atomic64_read(&leader->child_total_time_enabled);
4239         }
4240 
4241         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
4242                 values[n++] += leader->total_time_running +
4243                         atomic64_read(&leader->child_total_time_running);
4244         }
4245 
4246         /*
4247          * Write {count,id} tuples for every sibling.
4248          */
4249         values[n++] += perf_event_count(leader);
4250         if (read_format & PERF_FORMAT_ID)
4251                 values[n++] = primary_event_id(leader);
4252 
4253         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
4254                 values[n++] += perf_event_count(sub);
4255                 if (read_format & PERF_FORMAT_ID)
4256                         values[n++] = primary_event_id(sub);
4257         }
4258 
4259         return 0;
4260 }
4261 
4262 static int perf_read_group(struct perf_event *event,
4263                                    u64 read_format, char __user *buf)
4264 {
4265         struct perf_event *leader = event->group_leader, *child;
4266         struct perf_event_context *ctx = leader->ctx;
4267         int ret;
4268         u64 *values;
4269 
4270         lockdep_assert_held(&ctx->mutex);
4271 
4272         values = kzalloc(event->read_size, GFP_KERNEL);
4273         if (!values)
4274                 return -ENOMEM;
4275 
4276         values[0] = 1 + leader->nr_siblings;
4277 
4278         /*
4279          * By locking the child_mutex of the leader we effectively
4280          * lock the child list of all siblings.. XXX explain how.
4281          */
4282         mutex_lock(&leader->child_mutex);
4283 
4284         ret = __perf_read_group_add(leader, read_format, values);
4285         if (ret)
4286                 goto unlock;
4287 
4288         list_for_each_entry(child, &leader->child_list, child_list) {
4289                 ret = __perf_read_group_add(child, read_format, values);
4290                 if (ret)
4291                         goto unlock;
4292         }
4293 
4294         mutex_unlock(&leader->child_mutex);
4295 
4296         ret = event->read_size;
4297         if (copy_to_user(buf, values, event->read_size))
4298                 ret = -EFAULT;
4299         goto out;
4300 
4301 unlock:
4302         mutex_unlock(&leader->child_mutex);
4303 out:
4304         kfree(values);
4305         return ret;
4306 }
4307 
4308 static int perf_read_one(struct perf_event *event,
4309                                  u64 read_format, char __user *buf)
4310 {
4311         u64 enabled, running;
4312         u64 values[4];
4313         int n = 0;
4314 
4315         values[n++] = perf_event_read_value(event, &enabled, &running);
4316         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
4317                 values[n++] = enabled;
4318         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
4319                 values[n++] = running;
4320         if (read_format & PERF_FORMAT_ID)
4321                 values[n++] = primary_event_id(event);
4322 
4323         if (copy_to_user(buf, values, n * sizeof(u64)))
4324                 return -EFAULT;
4325 
4326         return n * sizeof(u64);
4327 }
4328 
4329 static bool is_event_hup(struct perf_event *event)
4330 {
4331         bool no_children;
4332 
4333         if (event->state > PERF_EVENT_STATE_EXIT)
4334                 return false;
4335 
4336         mutex_lock(&event->child_mutex);
4337         no_children = list_empty(&event->child_list);
4338         mutex_unlock(&event->child_mutex);
4339         return no_children;
4340 }
4341 
4342 /*
4343  * Read the performance event - simple non blocking version for now
4344  */
4345 static ssize_t
4346 __perf_read(struct perf_event *event, char __user *buf, size_t count)
4347 {
4348         u64 read_format = event->attr.read_format;
4349         int ret;
4350 
4351         /*
4352          * Return end-of-file for a read on a event that is in
4353          * error state (i.e. because it was pinned but it couldn't be
4354          * scheduled on to the CPU at some point).
4355          */
4356         if (event->state == PERF_EVENT_STATE_ERROR)
4357                 return 0;
4358 
4359         if (count < event->read_size)
4360                 return -ENOSPC;
4361 
4362         WARN_ON_ONCE(event->ctx->parent_ctx);
4363         if (read_format & PERF_FORMAT_GROUP)
4364                 ret = perf_read_group(event, read_format, buf);
4365         else
4366                 ret = perf_read_one(event, read_format, buf);
4367 
4368         return ret;
4369 }
4370 
4371 static ssize_t
4372 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
4373 {
4374         struct perf_event *event = file->private_data;
4375         struct perf_event_context *ctx;
4376         int ret;
4377 
4378         ctx = perf_event_ctx_lock(event);
4379         ret = __perf_read(event, buf, count);
4380         perf_event_ctx_unlock(event, ctx);
4381 
4382         return ret;
4383 }
4384 
4385 static unsigned int perf_poll(struct file *file, poll_table *wait)
4386 {
4387         struct perf_event *event = file->private_data;
4388         struct ring_buffer *rb;
4389         unsigned int events = POLLHUP;
4390 
4391         poll_wait(file, &event->waitq, wait);
4392 
4393         if (is_event_hup(event))
4394                 return events;
4395 
4396         /*
4397          * Pin the event->rb by taking event->mmap_mutex; otherwise
4398          * perf_event_set_output() can swizzle our rb and make us miss wakeups.
4399          */
4400         mutex_lock(&event->mmap_mutex);
4401         rb = event->rb;
4402         if (rb)
4403                 events = atomic_xchg(&rb->poll, 0);
4404         mutex_unlock(&event->mmap_mutex);
4405         return events;
4406 }
4407 
4408 static void _perf_event_reset(struct perf_event *event)
4409 {
4410         (void)perf_event_read(event, false);
4411         local64_set(&event->count, 0);
4412         perf_event_update_userpage(event);
4413 }
4414 
4415 /*
4416  * Holding the top-level event's child_mutex means that any
4417  * descendant process that has inherited this event will block
4418  * in perf_event_exit_event() if it goes to exit, thus satisfying the
4419  * task existence requirements of perf_event_enable/disable.
4420  */
4421 static void perf_event_for_each_child(struct perf_event *event,
4422                                         void (*func)(struct perf_event *))
4423 {
4424         struct perf_event *child;
4425 
4426         WARN_ON_ONCE(event->ctx->parent_ctx);
4427 
4428         mutex_lock(&event->child_mutex);
4429         func(event);
4430         list_for_each_entry(child, &event->child_list, child_list)
4431                 func(child);
4432         mutex_unlock(&event->child_mutex);
4433 }
4434 
4435 static void perf_event_for_each(struct perf_event *event,
4436                                   void (*func)(struct perf_event *))
4437 {
4438         struct perf_event_context *ctx = event->ctx;
4439         struct perf_event *sibling;
4440 
4441         lockdep_assert_held(&ctx->mutex);
4442 
4443         event = event->group_leader;
4444 
4445         perf_event_for_each_child(event, func);
4446         list_for_each_entry(sibling, &event->sibling_list, group_entry)
4447                 perf_event_for_each_child(sibling, func);
4448 }
4449 
4450 static void __perf_event_period(struct perf_event *event,
4451                                 struct perf_cpu_context *cpuctx,
4452                                 struct perf_event_context *ctx,
4453                                 void *info)
4454 {
4455         u64 value = *((u64 *)info);
4456         bool active;
4457 
4458         if (event->attr.freq) {
4459                 event->attr.sample_freq = value;
4460         } else {
4461                 event->attr.sample_period = value;
4462                 event->hw.sample_period = value;
4463         }
4464 
4465         active = (event->state == PERF_EVENT_STATE_ACTIVE);
4466         if (active) {
4467                 perf_pmu_disable(ctx->pmu);
4468                 /*
4469                  * We could be throttled; unthrottle now to avoid the tick
4470                  * trying to unthrottle while we already re-started the event.
4471                  */
4472                 if (event->hw.interrupts == MAX_INTERRUPTS) {
4473                         event->hw.interrupts = 0;
4474                         perf_log_throttle(event, 1);
4475                 }
4476                 event->pmu->stop(event, PERF_EF_UPDATE);
4477         }
4478 
4479         local64_set(&event->hw.period_left, 0);
4480 
4481         if (active) {
4482                 event->pmu->start(event, PERF_EF_RELOAD);
4483                 perf_pmu_enable(ctx->pmu);
4484         }
4485 }
4486 
4487 static int perf_event_period(struct perf_event *event, u64 __user *arg)
4488 {
4489         u64 value;
4490 
4491         if (!is_sampling_event(event))
4492                 return -EINVAL;
4493 
4494         if (copy_from_user(&value, arg, sizeof(value)))
4495                 return -EFAULT;
4496 
4497         if (!value)
4498                 return -EINVAL;
4499 
4500         if (event->attr.freq && value > sysctl_perf_event_sample_rate)
4501                 return -EINVAL;
4502 
4503         event_function_call(event, __perf_event_period, &value);
4504 
4505         return 0;
4506 }
4507 
4508 static const struct file_operations perf_fops;
4509 
4510 static inline int perf_fget_light(int fd, struct fd *p)
4511 {
4512         struct fd f = fdget(fd);
4513         if (!f.file)
4514                 return -EBADF;
4515 
4516         if (f.file->f_op != &perf_fops) {
4517                 fdput(f);
4518                 return -EBADF;
4519         }
4520         *p = f;
4521         return 0;
4522 }
4523 
4524 static int perf_event_set_output(struct perf_event *event,
4525                                  struct perf_event *output_event);
4526 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
4527 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd);
4528 
4529 static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)
4530 {
4531         void (*func)(struct perf_event *);
4532         u32 flags = arg;
4533 
4534         switch (cmd) {
4535         case PERF_EVENT_IOC_ENABLE:
4536                 func = _perf_event_enable;
4537                 break;
4538         case PERF_EVENT_IOC_DISABLE:
4539                 func = _perf_event_disable;
4540                 break;
4541         case PERF_EVENT_IOC_RESET:
4542                 func = _perf_event_reset;
4543                 break;
4544 
4545         case PERF_EVENT_IOC_REFRESH:
4546                 return _perf_event_refresh(event, arg);
4547 
4548         case PERF_EVENT_IOC_PERIOD:
4549                 return perf_event_period(event, (u64 __user *)arg);
4550 
4551         case PERF_EVENT_IOC_ID:
4552         {
4553                 u64 id = primary_event_id(event);
4554 
4555                 if (copy_to_user((void __user *)arg, &id, sizeof(id)))
4556                         return -EFAULT;
4557                 return 0;
4558         }
4559 
4560         case PERF_EVENT_IOC_SET_OUTPUT:
4561         {
4562                 int ret;
4563                 if (arg != -1) {
4564                         struct perf_event *output_event;
4565                         struct fd output;
4566                         ret = perf_fget_light(arg, &output);
4567                         if (ret)
4568                                 return ret;
4569                         output_event = output.file->private_data;
4570                         ret = perf_event_set_output(event, output_event);
4571                         fdput(output);
4572                 } else {
4573                         ret = perf_event_set_output(event, NULL);
4574                 }
4575                 return ret;
4576         }
4577 
4578         case PERF_EVENT_IOC_SET_FILTER:
4579                 return perf_event_set_filter(event, (void __user *)arg);
4580 
4581         case PERF_EVENT_IOC_SET_BPF:
4582                 return perf_event_set_bpf_prog(event, arg);
4583 
4584         case PERF_EVENT_IOC_PAUSE_OUTPUT: {
4585                 struct ring_buffer *rb;
4586 
4587                 rcu_read_lock();
4588                 rb = rcu_dereference(event->rb);
4589                 if (!rb || !rb->nr_pages) {
4590                         rcu_read_unlock();
4591                         return -EINVAL;
4592                 }
4593                 rb_toggle_paused(rb, !!arg);
4594                 rcu_read_unlock();
4595                 return 0;
4596         }
4597         default:
4598                 return -ENOTTY;
4599         }
4600 
4601         if (flags & PERF_IOC_FLAG_GROUP)
4602                 perf_event_for_each(event, func);
4603         else
4604                 perf_event_for_each_child(event, func);
4605 
4606         return 0;
4607 }
4608 
4609 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
4610 {
4611         struct perf_event *event = file->private_data;
4612         struct perf_event_context *ctx;
4613         long ret;
4614 
4615         ctx = perf_event_ctx_lock(event);
4616         ret = _perf_ioctl(event, cmd, arg);
4617         perf_event_ctx_unlock(event, ctx);
4618 
4619         return ret;
4620 }
4621 
4622 #ifdef CONFIG_COMPAT
4623 static long perf_compat_ioctl(struct file *file, unsigned int cmd,
4624                                 unsigned long arg)
4625 {
4626         switch (_IOC_NR(cmd)) {
4627         case _IOC_NR(PERF_EVENT_IOC_SET_FILTER):
4628         case _IOC_NR(PERF_EVENT_IOC_ID):
4629                 /* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */
4630                 if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) {
4631                         cmd &= ~IOCSIZE_MASK;
4632                         cmd |= sizeof(void *) << IOCSIZE_SHIFT;
4633                 }
4634                 break;
4635         }
4636         return perf_ioctl(file, cmd, arg);
4637 }
4638 #else
4639 # define perf_compat_ioctl NULL
4640 #endif
4641 
4642 int perf_event_task_enable(void)
4643 {
4644         struct perf_event_context *ctx;
4645         struct perf_event *event;
4646 
4647         mutex_lock(&current->perf_event_mutex);
4648         list_for_each_entry(event, &current->perf_event_list, owner_entry) {
4649                 ctx = perf_event_ctx_lock(event);
4650                 perf_event_for_each_child(event, _perf_event_enable);
4651                 perf_event_ctx_unlock(event, ctx);
4652         }
4653         mutex_unlock(&current->perf_event_mutex);
4654 
4655         return 0;
4656 }
4657 
4658 int perf_event_task_disable(void)
4659 {
4660         struct perf_event_context *ctx;
4661         struct perf_event *event;
4662 
4663         mutex_lock(&current->perf_event_mutex);
4664         list_for_each_entry(event, &current->perf_event_list, owner_entry) {
4665                 ctx = perf_event_ctx_lock(event);
4666                 perf_event_for_each_child(event, _perf_event_disable);
4667                 perf_event_ctx_unlock(event, ctx);
4668         }
4669         mutex_unlock(&current->perf_event_mutex);
4670 
4671         return 0;
4672 }
4673 
4674 static int perf_event_index(struct perf_event *event)
4675 {
4676         if (event->hw.state & PERF_HES_STOPPED)
4677                 return 0;
4678 
4679         if (event->state != PERF_EVENT_STATE_ACTIVE)
4680                 return 0;
4681 
4682         return event->pmu->event_idx(event);
4683 }
4684 
4685 static void calc_timer_values(struct perf_event *event,
4686                                 u64 *now,
4687                                 u64 *enabled,
4688                                 u64 *running)
4689 {
4690         u64 ctx_time;
4691 
4692         *now = perf_clock();
4693         ctx_time = event->shadow_ctx_time + *now;
4694         *enabled = ctx_time - event->tstamp_enabled;
4695         *running = ctx_time - event->tstamp_running;
4696 }
4697 
4698 static void perf_event_init_userpage(struct perf_event *event)
4699 {
4700         struct perf_event_mmap_page *userpg;
4701         struct ring_buffer *rb;
4702 
4703         rcu_read_lock();
4704         rb = rcu_dereference(event->rb);
4705         if (!rb)
4706                 goto unlock;
4707 
4708         userpg = rb->user_page;
4709 
4710         /* Allow new userspace to detect that bit 0 is deprecated */
4711         userpg->cap_bit0_is_deprecated = 1;
4712         userpg->size = offsetof(struct perf_event_mmap_page, __reserved);
4713         userpg->data_offset = PAGE_SIZE;
4714         userpg->data_size = perf_data_size(rb);
4715 
4716 unlock:
4717         rcu_read_unlock();
4718 }
4719 
4720 void __weak arch_perf_update_userpage(
4721         struct perf_event *event, struct perf_event_mmap_page *userpg, u64 now)
4722 {
4723 }
4724 
4725 /*
4726  * Callers need to ensure there can be no nesting of this function, otherwise
4727  * the seqlock logic goes bad. We can not serialize this because the arch
4728  * code calls this from NMI context.
4729  */
4730 void perf_event_update_userpage(struct perf_event *event)
4731 {
4732         struct perf_event_mmap_page *userpg;
4733         struct ring_buffer *rb;
4734         u64 enabled, running, now;
4735 
4736         rcu_read_lock();
4737         rb = rcu_dereference(event->rb);
4738         if (!rb)
4739                 goto unlock;
4740 
4741         /*
4742          * compute total_time_enabled, total_time_running
4743          * based on snapshot values taken when the event
4744          * was last scheduled in.
4745          *
4746          * we cannot simply called update_context_time()
4747          * because of locking issue as we can be called in
4748          * NMI context
4749          */
4750         calc_timer_values(event, &now, &enabled, &running);
4751 
4752         userpg = rb->user_page;
4753         /*
4754          * Disable preemption so as to not let the corresponding user-space
4755          * spin too long if we get preempted.
4756          */
4757         preempt_disable();
4758         ++userpg->lock;
4759         barrier();
4760         userpg->index = perf_event_index(event);
4761         userpg->offset = perf_event_count(event);
4762         if (userpg->index)
4763                 userpg->offset -= local64_read(&event->hw.prev_count);
4764 
4765         userpg->time_enabled = enabled +
4766                         atomic64_read(&event->child_total_time_enabled);
4767 
4768         userpg->time_running = running +
4769                         atomic64_read(&event->child_total_time_running);
4770 
4771         arch_perf_update_userpage(event, userpg, now);
4772 
4773         barrier();
4774         ++userpg->lock;
4775         preempt_enable();
4776 unlock:
4777         rcu_read_unlock();
4778 }
4779 
4780 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
4781 {
4782         struct perf_event *event = vma->vm_file->private_data;
4783         struct ring_buffer *rb;
4784         int ret = VM_FAULT_SIGBUS;
4785 
4786         if (vmf->flags & FAULT_FLAG_MKWRITE) {
4787                 if (vmf->pgoff == 0)
4788                         ret = 0;
4789                 return ret;
4790         }
4791 
4792         rcu_read_lock();
4793         rb = rcu_dereference(event->rb);
4794         if (!rb)
4795                 goto unlock;
4796 
4797         if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
4798                 goto unlock;
4799 
4800         vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
4801         if (!vmf->page)
4802                 goto unlock;
4803 
4804         get_page(vmf->page);
4805         vmf->page->mapping = vma->vm_file->f_mapping;
4806         vmf->page->index   = vmf->pgoff;
4807 
4808         ret = 0;
4809 unlock:
4810         rcu_read_unlock();
4811 
4812         return ret;
4813 }
4814 
4815 static void ring_buffer_attach(struct perf_event *event,
4816                                struct ring_buffer *rb)
4817 {
4818         struct ring_buffer *old_rb = NULL;
4819         unsigned long flags;
4820 
4821         if (event->rb) {
4822                 /*
4823                  * Should be impossible, we set this when removing
4824                  * event->rb_entry and wait/clear when adding event->rb_entry.
4825                  */
4826                 WARN_ON_ONCE(event->rcu_pending);
4827 
4828                 old_rb = event->rb;
4829                 spin_lock_irqsave(&old_rb->event_lock, flags);
4830                 list_del_rcu(&event->rb_entry);
4831                 spin_unlock_irqrestore(&old_rb->event_lock, flags);
4832 
4833                 event->rcu_batches = get_state_synchronize_rcu();
4834                 event->rcu_pending = 1;
4835         }
4836 
4837         if (rb) {
4838                 if (event->rcu_pending) {
4839                         cond_synchronize_rcu(event->rcu_batches);
4840                         event->rcu_pending = 0;
4841                 }
4842 
4843                 spin_lock_irqsave(&rb->event_lock, flags);
4844                 list_add_rcu(&event->rb_entry, &rb->event_list);
4845                 spin_unlock_irqrestore(&rb->event_lock, flags);
4846         }
4847 
4848         /*
4849          * Avoid racing with perf_mmap_close(AUX): stop the event
4850          * before swizzling the event::rb pointer; if it's getting
4851          * unmapped, its aux_mmap_count will be 0 and it won't
4852          * restart. See the comment in __perf_pmu_output_stop().
4853          *
4854          * Data will inevitably be lost when set_output is done in
4855          * mid-air, but then again, whoever does it like this is
4856          * not in for the data anyway.
4857          */
4858         if (has_aux(event))
4859                 perf_event_stop(event, 0);
4860 
4861         rcu_assign_pointer(event->rb, rb);
4862 
4863         if (old_rb) {
4864                 ring_buffer_put(old_rb);
4865                 /*
4866                  * Since we detached before setting the new rb, so that we
4867                  * could attach the new rb, we could have missed a wakeup.
4868                  * Provide it now.
4869                  */
4870                 wake_up_all(&event->waitq);
4871         }
4872 }
4873 
4874 static void ring_buffer_wakeup(struct perf_event *event)
4875 {
4876         struct ring_buffer *rb;
4877 
4878         rcu_read_lock();
4879         rb = rcu_dereference(event->rb);
4880         if (rb) {
4881                 list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
4882                         wake_up_all(&event->waitq);
4883         }
4884         rcu_read_unlock();
4885 }
4886 
4887 struct ring_buffer *ring_buffer_get(struct perf_event *event)
4888 {
4889         struct ring_buffer *rb;
4890 
4891         rcu_read_lock();
4892         rb = rcu_dereference(event->rb);
4893         if (rb) {
4894                 if (!atomic_inc_not_zero(&rb->refcount))
4895                         rb = NULL;
4896         }
4897         rcu_read_unlock();
4898 
4899         return rb;
4900 }
4901 
4902 void ring_buffer_put(struct ring_buffer *rb)
4903 {
4904         if (!atomic_dec_and_test(&rb->refcount))
4905                 return;
4906 
4907         WARN_ON_ONCE(!list_empty(&rb->event_list));
4908 
4909         call_rcu(&rb->rcu_head, rb_free_rcu);
4910 }
4911 
4912 static void perf_mmap_open(struct vm_area_struct *vma)
4913 {
4914         struct perf_event *event = vma->vm_file->private_data;
4915 
4916         atomic_inc(&event->mmap_count);
4917         atomic_inc(&event->rb->mmap_count);
4918 
4919         if (vma->vm_pgoff)
4920                 atomic_inc(&event->rb->aux_mmap_count);
4921 
4922         if (event->pmu->event_mapped)
4923                 event->pmu->event_mapped(event);
4924 }
4925 
4926 static void perf_pmu_output_stop(struct perf_event *event);
4927 
4928 /*
4929  * A buffer can be mmap()ed multiple times; either directly through the same
4930  * event, or through other events by use of perf_event_set_output().
4931  *
4932  * In order to undo the VM accounting done by perf_mmap() we need to destroy
4933  * the buffer here, where we still have a VM context. This means we need
4934  * to detach all events redirecting to us.
4935  */
4936 static void perf_mmap_close(struct vm_area_struct *vma)
4937 {
4938         struct perf_event *event = vma->vm_file->private_data;
4939 
4940         struct ring_buffer *rb = ring_buffer_get(event);
4941         struct user_struct *mmap_user = rb->mmap_user;
4942         int mmap_locked = rb->mmap_locked;
4943         unsigned long size = perf_data_size(rb);
4944 
4945         if (event->pmu->event_unmapped)
4946                 event->pmu->event_unmapped(event);
4947 
4948         /*
4949          * rb->aux_mmap_count will always drop before rb->mmap_count and
4950          * event->mmap_count, so it is ok to use event->mmap_mutex to
4951          * serialize with perf_mmap here.
4952          */
4953         if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff &&
4954             atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &event->mmap_mutex)) {
4955                 /*
4956                  * Stop all AUX events that are writing to this buffer,
4957                  * so that we can free its AUX pages and corresponding PMU
4958                  * data. Note that after rb::aux_mmap_count dropped to zero,
4959                  * they won't start any more (see perf_aux_output_begin()).
4960                  */
4961                 perf_pmu_output_stop(event);
4962 
4963                 /* now it's safe to free the pages */
4964                 atomic_long_sub(rb->aux_nr_pages, &mmap_user->locked_vm);
4965                 vma->vm_mm->pinned_vm -= rb->aux_mmap_locked;
4966 
4967                 /* this has to be the last one */
4968                 rb_free_aux(rb);
4969                 WARN_ON_ONCE(atomic_read(&rb->aux_refcount));
4970 
4971                 mutex_unlock(&event->mmap_mutex);
4972         }
4973 
4974         atomic_dec(&rb->mmap_count);
4975 
4976         if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
4977                 goto out_put;
4978 
4979         ring_buffer_attach(event, NULL);
4980         mutex_unlock(&event->mmap_mutex);
4981 
4982         /* If there's still other mmap()s of this buffer, we're done. */
4983         if (atomic_read(&rb->mmap_count))
4984                 goto out_put;
4985 
4986         /*
4987          * No other mmap()s, detach from all other events that might redirect
4988          * into the now unreachable buffer. Somewhat complicated by the
4989          * fact that rb::event_lock otherwise nests inside mmap_mutex.
4990          */
4991 again:
4992         rcu_read_lock();
4993         list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
4994                 if (!atomic_long_inc_not_zero(&event->refcount)) {
4995                         /*
4996                          * This event is en-route to free_event() which will
4997                          * detach it and remove it from the list.
4998                          */
4999                         continue;
5000                 }
5001                 rcu_read_unlock();
5002 
5003                 mutex_lock(&event->mmap_mutex);
5004                 /*
5005                  * Check we didn't race with perf_event_set_output() which can
5006                  * swizzle the rb from under us while we were waiting to
5007                  * acquire mmap_mutex.
5008                  *
5009                  * If we find a different rb; ignore this event, a next
5010                  * iteration will no longer find it on the list. We have to
5011                  * still restart the iteration to make sure we're not now
5012                  * iterating the wrong list.
5013                  */
5014                 if (event->rb == rb)
5015                         ring_buffer_attach(event, NULL);
5016 
5017                 mutex_unlock(&event->mmap_mutex);
5018                 put_event(event);
5019 
5020                 /*
5021                  * Restart the iteration; either we're on the wrong list or
5022                  * destroyed its integrity by doing a deletion.
5023                  */
5024                 goto again;
5025         }
5026         rcu_read_unlock();
5027 
5028         /*
5029          * It could be there's still a few 0-ref events on the list; they'll
5030          * get cleaned up by free_event() -- they'll also still have their
5031          * ref on the rb and will free it whenever they are done with it.
5032          *
5033          * Aside from that, this buffer is 'fully' detached and unmapped,
5034          * undo the VM accounting.
5035          */
5036 
5037         atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm);
5038         vma->vm_mm->pinned_vm -= mmap_locked;
5039         free_uid(mmap_user);
5040 
5041 out_put:
5042         ring_buffer_put(rb); /* could be last */
5043 }
5044 
5045 static const struct vm_operations_struct perf_mmap_vmops = {
5046         .open           = perf_mmap_open,
5047         .close          = perf_mmap_close, /* non mergable */
5048         .fault          = perf_mmap_fault,
5049         .page_mkwrite   = perf_mmap_fault,
5050 };
5051 
5052 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
5053 {
5054         struct perf_event *event = file->private_data;
5055         unsigned long user_locked, user_lock_limit;
5056         struct user_struct *user = current_user();
5057         unsigned long locked, lock_limit;
5058         struct ring_buffer *rb = NULL;
5059         unsigned long vma_size;
5060         unsigned long nr_pages;
5061         long user_extra = 0, extra = 0;
5062         int ret = 0, flags = 0;
5063 
5064         /*
5065          * Don't allow mmap() of inherited per-task counters. This would
5066          * create a performance issue due to all children writing to the
5067          * same rb.
5068          */
5069         if (event->cpu == -1 && event->attr.inherit)
5070                 return -EINVAL;
5071 
5072         if (!(vma->vm_flags & VM_SHARED))
5073                 return -EINVAL;
5074 
5075         vma_size = vma->vm_end - vma->vm_start;
5076 
5077         if (vma->vm_pgoff == 0) {
5078                 nr_pages = (vma_size / PAGE_SIZE) - 1;
5079         } else {
5080                 /*
5081                  * AUX area mapping: if rb->aux_nr_pages != 0, it's already
5082                  * mapped, all subsequent mappings should have the same size
5083                  * and offset. Must be above the normal perf buffer.
5084                  */
5085                 u64 aux_offset, aux_size;
5086 
5087                 if (!event->rb)
5088                         return -EINVAL;
5089 
5090                 nr_pages = vma_size / PAGE_SIZE;
5091 
5092                 mutex_lock(&event->mmap_mutex);
5093                 ret = -EINVAL;
5094 
5095                 rb = event->rb;
5096                 if (!rb)
5097                         goto aux_unlock;
5098 
5099                 aux_offset = ACCESS_ONCE(rb->user_page->aux_offset);
5100                 aux_size = ACCESS_ONCE(rb->user_page->aux_size);
5101 
5102                 if (aux_offset < perf_data_size(rb) + PAGE_SIZE)
5103                         goto aux_unlock;
5104 
5105                 if (aux_offset != vma->vm_pgoff << PAGE_SHIFT)
5106                         goto aux_unlock;
5107 
5108                 /* already mapped with a different offset */
5109                 if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff)
5110                         goto aux_unlock;
5111 
5112                 if (aux_size != vma_size || aux_size != nr_pages * PAGE_SIZE)
5113                         goto aux_unlock;
5114 
5115                 /* already mapped with a different size */
5116                 if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages)
5117                         goto aux_unlock;
5118 
5119                 if (!is_power_of_2(nr_pages))
5120                         goto aux_unlock;
5121 
5122                 if (!atomic_inc_not_zero(&rb->mmap_count))
5123                         goto aux_unlock;
5124 
5125                 if (rb_has_aux(rb)) {
5126                         atomic_inc(&rb->aux_mmap_count);
5127                         ret = 0;
5128                         goto unlock;
5129                 }
5130 
5131                 atomic_set(&rb->aux_mmap_count, 1);
5132                 user_extra = nr_pages;
5133 
5134                 goto accounting;
5135         }
5136 
5137         /*
5138          * If we have rb pages ensure they're a power-of-two number, so we
5139          * can do bitmasks instead of modulo.
5140          */
5141         if (nr_pages != 0 && !is_power_of_2(nr_pages))
5142                 return -EINVAL;
5143 
5144         if (vma_size != PAGE_SIZE * (1 + nr_pages))
5145                 return -EINVAL;
5146 
5147         WARN_ON_ONCE(event->ctx->parent_ctx);
5148 again:
5149         mutex_lock(&event->mmap_mutex);
5150         if (event->rb) {
5151                 if (event->rb->nr_pages != nr_pages) {
5152                         ret = -EINVAL;
5153                         goto unlock;
5154                 }
5155 
5156                 if (!atomic_inc_not_zero(&event->rb->mmap_count)) {
5157                         /*
5158                          * Raced against perf_mmap_close() through
5159                          * perf_event_set_output(). Try again, hope for better
5160                          * luck.
5161                          */
5162                         mutex_unlock(&event->mmap_mutex);
5163                         goto again;
5164                 }
5165 
5166                 goto unlock;
5167         }
5168 
5169         user_extra = nr_pages + 1;
5170 
5171 accounting:
5172         user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
5173 
5174         /*
5175          * Increase the limit linearly with more CPUs:
5176          */
5177         user_lock_limit *= num_online_cpus();
5178 
5179         user_locked = atomic_long_read(&user->locked_vm) + user_extra;
5180 
5181         if (user_locked > user_lock_limit)
5182                 extra = user_locked - user_lock_limit;
5183 
5184         lock_limit = rlimit(RLIMIT_MEMLOCK);
5185         lock_limit >>= PAGE_SHIFT;
5186         locked = vma->vm_mm->pinned_vm + extra;
5187 
5188         if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
5189                 !capable(CAP_IPC_LOCK)) {
5190                 ret = -EPERM;
5191                 goto unlock;
5192         }
5193 
5194         WARN_ON(!rb && event->rb);
5195 
5196         if (vma->vm_flags & VM_WRITE)
5197                 flags |= RING_BUFFER_WRITABLE;
5198 
5199         if (!rb) {
5200                 rb = rb_alloc(nr_pages,
5201                               event->attr.watermark ? event->attr.wakeup_watermark : 0,
5202                               event->cpu, flags);
5203 
5204                 if (!rb) {
5205                         ret = -ENOMEM;
5206                         goto unlock;
5207                 }
5208 
5209                 atomic_set(&rb->mmap_count, 1);
5210                 rb->mmap_user = get_current_user();
5211                 rb->mmap_locked = extra;
5212 
5213                 ring_buffer_attach(event, rb);
5214 
5215                 perf_event_init_userpage(event);
5216                 perf_event_update_userpage(event);
5217         } else {
5218                 ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages,
5219                                    event->attr.aux_watermark, flags);
5220                 if (!ret)
5221                         rb->aux_mmap_locked = extra;
5222         }
5223 
5224 unlock:
5225         if (!ret) {
5226                 atomic_long_add(user_extra, &user->locked_vm);
5227                 vma->vm_mm->pinned_vm += extra;
5228 
5229                 atomic_inc(&event->mmap_count);
5230         } else if (rb) {
5231                 atomic_dec(&rb->mmap_count);
5232         }
5233 aux_unlock:
5234         mutex_unlock(&event->mmap_mutex);
5235 
5236         /*
5237          * Since pinned accounting is per vm we cannot allow fork() to copy our
5238          * vma.
5239          */
5240         vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;
5241         vma->vm_ops = &perf_mmap_vmops;
5242 
5243         if (event->pmu->event_mapped)
5244                 event->pmu->event_mapped(event);
5245 
5246         return ret;
5247 }
5248 
5249 static int perf_fasync(int fd, struct file *filp, int on)
5250 {
5251         struct inode *inode = file_inode(filp);
5252         struct perf_event *event = filp->private_data;
5253         int retval;
5254 
5255         inode_lock(inode);
5256         retval = fasync_helper(fd, filp, on, &event->fasync);
5257         inode_unlock(inode);
5258 
5259         if (retval < 0)
5260                 return retval;
5261 
5262         return 0;
5263 }
5264 
5265 static const struct file_operations perf_fops = {
5266         .llseek                 = no_llseek,
5267         .release                = perf_release,
5268         .read                   = perf_read,
5269         .poll                   = perf_poll,
5270         .unlocked_ioctl         = perf_ioctl,
5271         .compat_ioctl           = perf_compat_ioctl,
5272         .mmap                   = perf_mmap,
5273         .fasync                 = perf_fasync,
5274 };
5275 
5276 /*
5277  * Perf event wakeup
5278  *
5279  * If there's data, ensure we set the poll() state and publish everything
5280  * to user-space before waking everybody up.
5281  */
5282 
5283 static inline struct fasync_struct **perf_event_fasync(struct perf_event *event)
5284 {
5285         /* only the parent has fasync state */
5286         if (event->parent)
5287                 event = event->parent;
5288         return &event->fasync;
5289 }
5290 
5291 void perf_event_wakeup(struct perf_event *event)
5292 {
5293         ring_buffer_wakeup(event);
5294 
5295         if (event->pending_kill) {
5296                 kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill);
5297                 event->pending_kill = 0;
5298         }
5299 }
5300 
5301 static void perf_pending_event(struct irq_work *entry)
5302 {
5303         struct perf_event *event = container_of(entry,
5304                         struct perf_event, pending);
5305         int rctx;
5306 
5307         rctx = perf_swevent_get_recursion_context();
5308         /*
5309          * If we 'fail' here, that's OK, it means recursion is already disabled
5310          * and we won't recurse 'further'.
5311          */
5312 
5313         if (event->pending_disable) {
5314                 event->pending_disable = 0;
5315                 perf_event_disable_local(event);
5316         }
5317 
5318         if (event->pending_wakeup) {
5319                 event->pending_wakeup = 0;
5320                 perf_event_wakeup(event);
5321         }
5322 
5323         if (rctx >= 0)
5324                 perf_swevent_put_recursion_context(rctx);
5325 }
5326 
5327 /*
5328  * We assume there is only KVM supporting the callbacks.
5329  * Later on, we might change it to a list if there is
5330  * another virtualization implementation supporting the callbacks.
5331  */
5332 struct perf_guest_info_callbacks *perf_guest_cbs;
5333 
5334 int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
5335 {
5336         perf_guest_cbs = cbs;
5337         return 0;
5338 }
5339 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
5340 
5341 int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
5342 {
5343         perf_guest_cbs = NULL;
5344         return 0;
5345 }
5346 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
5347 
5348 static void
5349 perf_output_sample_regs(struct perf_output_handle *handle,
5350                         struct pt_regs *regs, u64 mask)
5351 {
5352         int bit;
5353 
5354         for_each_set_bit(bit, (const unsigned long *) &mask,
5355                          sizeof(mask) * BITS_PER_BYTE) {
5356                 u64 val;
5357 
5358                 val = perf_reg_value(regs, bit);
5359                 perf_output_put(handle, val);
5360         }
5361 }
5362 
5363 static void perf_sample_regs_user(struct perf_regs *regs_user,
5364                                   struct pt_regs *regs,
5365                                   struct pt_regs *regs_user_copy)
5366 {
5367         if (user_mode(regs)) {
5368                 regs_user->abi = perf_reg_abi(current);
5369                 regs_user->regs = regs;
5370         } else if (current->mm) {
5371                 perf_get_regs_user(regs_user, regs, regs_user_copy);
5372         } else {
5373                 regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
5374                 regs_user->regs = NULL;
5375         }
5376 }
5377 
5378 static void perf_sample_regs_intr(struct perf_regs *regs_intr,
5379                                   struct pt_regs *regs)
5380 {
5381         regs_intr->regs = regs;
5382         regs_intr->abi  = perf_reg_abi(current);
5383 }
5384 
5385 
5386 /*
5387  * Get remaining task size from user stack pointer.
5388  *
5389  * It'd be better to take stack vma map and limit this more
5390  * precisly, but there's no way to get it safely under interrupt,
5391  * so using TASK_SIZE as limit.
5392  */
5393 static u64 perf_ustack_task_size(struct pt_regs *regs)
5394 {
5395         unsigned long addr = perf_user_stack_pointer(regs);
5396 
5397         if (!addr || addr >= TASK_SIZE)
5398                 return 0;
5399 
5400         return TASK_SIZE - addr;
5401 }
5402 
5403 static u16
5404 perf_sample_ustack_size(u16 stack_size, u16 header_size,
5405                         struct pt_regs *regs)
5406 {
5407         u64 task_size;
5408 
5409         /* No regs, no stack pointer, no dump. */
5410         if (!regs)
5411                 return 0;
5412 
5413         /*
5414          * Check if we fit in with the requested stack size into the:
5415          * - TASK_SIZE
5416          *   If we don't, we limit the size to the TASK_SIZE.
5417          *
5418          * - remaining sample size
5419          *   If we don't, we customize the stack size to
5420          *   fit in to the remaining sample size.
5421          */
5422 
5423         task_size  = min((u64) USHRT_MAX, perf_ustack_task_size(regs));
5424         stack_size = min(stack_size, (u16) task_size);
5425 
5426         /* Current header size plus static size and dynamic size. */
5427         header_size += 2 * sizeof(u64);
5428 
5429         /* Do we fit in with the current stack dump size? */
5430         if ((u16) (header_size + stack_size) < header_size) {
5431                 /*
5432                  * If we overflow the maximum size for the sample,
5433                  * we customize the stack dump size to fit in.
5434                  */
5435                 stack_size = USHRT_MAX - header_size - sizeof(u64);
5436                 stack_size = round_up(stack_size, sizeof(u64));
5437         }
5438 
5439         return stack_size;
5440 }
5441 
5442 static void
5443 perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size,
5444                           struct pt_regs *regs)
5445 {
5446         /* Case of a kernel thread, nothing to dump */
5447         if (!regs) {
5448                 u64 size = 0;
5449                 perf_output_put(handle, size);
5450         } else {
5451                 unsigned long sp;
5452                 unsigned int rem;
5453                 u64 dyn_size;
5454 
5455                 /*
5456                  * We dump:
5457                  * static size
5458                  *   - the size requested by user or the best one we can fit
5459                  *     in to the sample max size
5460                  * data
5461                  *   - user stack dump data
5462                  * dynamic size
5463                  *   - the actual dumped size
5464                  */
5465 
5466                 /* Static size. */
5467                 perf_output_put(handle, dump_size);
5468 
5469                 /* Data. */
5470                 sp = perf_user_stack_pointer(regs);
5471                 rem = __output_copy_user(handle, (void *) sp, dump_size);
5472                 dyn_size = dump_size - rem;
5473 
5474                 perf_output_skip(handle, rem);
5475 
5476                 /* Dynamic size. */
5477                 perf_output_put(handle, dyn_size);
5478         }
5479 }
5480 
5481 static void __perf_event_header__init_id(struct perf_event_header *header,
5482                                          struct perf_sample_data *data,
5483                                          struct perf_event *event)
5484 {
5485         u64 sample_type = event->attr.sample_type;
5486 
5487         data->type = sample_type;
5488         header->size += event->id_header_size;
5489 
5490         if (sample_type & PERF_SAMPLE_TID) {
5491                 /* namespace issues */
5492                 data->tid_entry.pid = perf_event_pid(event, current);
5493                 data->tid_entry.tid = perf_event_tid(event, current);
5494         }
5495 
5496         if (sample_type & PERF_SAMPLE_TIME)
5497                 data->time = perf_event_clock(event);
5498 
5499         if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
5500                 data->id = primary_event_id(event);
5501 
5502         if (sample_type & PERF_SAMPLE_STREAM_ID)
5503                 data->stream_id = event->id;
5504 
5505         if (sample_type & PERF_SAMPLE_CPU) {
5506                 data->cpu_entry.cpu      = raw_smp_processor_id();
5507                 data->cpu_entry.reserved = 0;
5508         }
5509 }
5510 
5511 void perf_event_header__init_id(struct perf_event_header *header,
5512                                 struct perf_sample_data *data,
5513                                 struct perf_event *event)
5514 {
5515         if (event->attr.sample_id_all)
5516                 __perf_event_header__init_id(header, data, event);
5517 }
5518 
5519 static void __perf_event__output_id_sample(struct perf_output_handle *handle,
5520                                            struct perf_sample_data *data)
5521 {
5522         u64 sample_type = data->type;
5523 
5524         if (sample_type & PERF_SAMPLE_TID)
5525                 perf_output_put(handle, data->tid_entry);
5526 
5527         if (sample_type & PERF_SAMPLE_TIME)
5528                 perf_output_put(handle, data->time);
5529 
5530         if (sample_type & PERF_SAMPLE_ID)
5531                 perf_output_put(handle, data->id);
5532 
5533         if (sample_type & PERF_SAMPLE_STREAM_ID)
5534                 perf_output_put(handle, data->stream_id);
5535 
5536         if (sample_type & PERF_SAMPLE_CPU)
5537                 perf_output_put(handle, data->cpu_entry);
5538 
5539         if (sample_type & PERF_SAMPLE_IDENTIFIER)
5540                 perf_output_put(handle, data->id);
5541 }
5542 
5543 void perf_event__output_id_sample(struct perf_event *event,
5544                                   struct perf_output_handle *handle,
5545                                   struct perf_sample_data *sample)
5546 {
5547         if (event->attr.sample_id_all)
5548                 __perf_event__output_id_sample(handle, sample);
5549 }
5550 
5551 static void perf_output_read_one(struct perf_output_handle *handle,
5552                                  struct perf_event *event,
5553                                  u64 enabled, u64 running)
5554 {
5555         u64 read_format = event->attr.read_format;
5556         u64 values[4];
5557         int n = 0;
5558 
5559         values[n++] = perf_event_count(event);
5560         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
5561                 values[n++] = enabled +
5562                         atomic64_read(&event->child_total_time_enabled);
5563         }
5564         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
5565                 values[n++] = running +
5566                         atomic64_read(&event->child_total_time_running);
5567         }
5568         if (read_format & PERF_FORMAT_ID)
5569                 values[n++] = primary_event_id(event);
5570 
5571         __output_copy(handle, values, n * sizeof(u64));
5572 }
5573 
5574 /*
5575  * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
5576  */
5577 static void perf_output_read_group(struct perf_output_handle *handle,
5578                             struct perf_event *event,
5579                             u64 enabled, u64 running)
5580 {
5581         struct perf_event *leader = event->group_leader, *sub;
5582         u64 read_format = event->attr.read_format;
5583         u64 values[5];
5584         int n = 0;
5585 
5586         values[n++] = 1 + leader->nr_siblings;
5587 
5588         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
5589                 values[n++] = enabled;
5590 
5591         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
5592                 values[n++] = running;
5593 
5594         if (leader != event)
5595                 leader->pmu->read(leader);
5596 
5597         values[n++] = perf_event_count(leader);
5598         if (read_format & PERF_FORMAT_ID)
5599                 values[n++] = primary_event_id(leader);
5600 
5601         __output_copy(handle, values, n * sizeof(u64));
5602 
5603         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
5604                 n = 0;
5605 
5606                 if ((sub != event) &&
5607                     (sub->state == PERF_EVENT_STATE_ACTIVE))
5608                         sub->pmu->read(sub);
5609 
5610                 values[n++] = perf_event_count(sub);
5611                 if (read_format & PERF_FORMAT_ID)
5612                         values[n++] = primary_event_id(sub);
5613 
5614                 __output_copy(handle, values, n * sizeof(u64));
5615         }
5616 }
5617 
5618 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
5619                                  PERF_FORMAT_TOTAL_TIME_RUNNING)
5620 
5621 static void perf_output_read(struct perf_output_handle *handle,
5622                              struct perf_event *event)
5623 {
5624         u64 enabled = 0, running = 0, now;
5625         u64 read_format = event->attr.read_format;
5626 
5627         /*
5628          * compute total_time_enabled, total_time_running
5629          * based on snapshot values taken when the event
5630          * was last scheduled in.
5631          *
5632          * we cannot simply called update_context_time()
5633          * because of locking issue as we are called in
5634          * NMI context
5635          */
5636         if (read_format & PERF_FORMAT_TOTAL_TIMES)
5637                 calc_timer_values(event, &now, &enabled, &running);
5638 
5639         if (event->attr.read_format & PERF_FORMAT_GROUP)
5640                 perf_output_read_group(handle, event, enabled, running);
5641         else
5642                 perf_output_read_one(handle, event, enabled, running);
5643 }
5644 
5645 void perf_output_sample(struct perf_output_handle *handle,
5646                         struct perf_event_header *header,
5647                         struct perf_sample_data *data,
5648                         struct perf_event *event)
5649 {
5650         u64 sample_type = data->type;
5651 
5652         perf_output_put(handle, *header);
5653 
5654         if (sample_type & PERF_SAMPLE_IDENTIFIER)
5655                 perf_output_put(handle, data->id);
5656 
5657         if (sample_type & PERF_SAMPLE_IP)
5658                 perf_output_put(handle, data->ip);
5659 
5660         if (sample_type & PERF_SAMPLE_TID)
5661                 perf_output_put(handle, data->tid_entry);
5662 
5663         if (sample_type & PERF_SAMPLE_TIME)
5664                 perf_output_put(handle, data->time);
5665 
5666         if (sample_type & PERF_SAMPLE_ADDR)
5667                 perf_output_put(handle, data->addr);
5668 
5669         if (sample_type & PERF_SAMPLE_ID)
5670                 perf_output_put(handle, data->id);
5671 
5672         if (sample_type & PERF_SAMPLE_STREAM_ID)
5673                 perf_output_put(handle, data->stream_id);
5674 
5675         if (sample_type & PERF_SAMPLE_CPU)
5676                 perf_output_put(handle, data->cpu_entry);
5677 
5678         if (sample_type & PERF_SAMPLE_PERIOD)
5679                 perf_output_put(handle, data->period);
5680 
5681         if (sample_type & PERF_SAMPLE_READ)
5682                 perf_output_read(handle, event);
5683 
5684         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
5685                 if (data->callchain) {
5686                         int size = 1;
5687 
5688                         if (data->callchain)
5689                                 size += data->callchain->nr;
5690 
5691                         size *= sizeof(u64);
5692 
5693                         __output_copy(handle, data->callchain, size);
5694                 } else {
5695                         u64 nr = 0;
5696                         perf_output_put(handle, nr);
5697                 }
5698         }
5699 
5700         if (sample_type & PERF_SAMPLE_RAW) {
5701                 struct perf_raw_record *raw = data->raw;
5702 
5703                 if (raw) {
5704                         struct perf_raw_frag *frag = &raw->frag;
5705 
5706                         perf_output_put(handle, raw->size);
5707                         do {
5708                                 if (frag->copy) {
5709                                         __output_custom(handle, frag->copy,
5710                                                         frag->data, frag->size);
5711                                 } else {
5712                                         __output_copy(handle, frag->data,
5713                                                       frag->size);
5714                                 }
5715                                 if (perf_raw_frag_last(frag))
5716                                         break;
5717                                 frag = frag->next;
5718                         } while (1);
5719                         if (frag->pad)
5720                                 __output_skip(handle, NULL, frag->pad);
5721                 } else {
5722                         struct {
5723                                 u32     size;
5724                                 u32     data;
5725                         } raw = {
5726                                 .size = sizeof(u32),
5727                                 .data = 0,
5728                         };
5729                         perf_output_put(handle, raw);
5730                 }
5731         }
5732 
5733         if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
5734                 if (data->br_stack) {
5735                         size_t size;
5736 
5737                         size = data->br_stack->nr
5738                              * sizeof(struct perf_branch_entry);
5739 
5740                         perf_output_put(handle, data->br_stack->nr);
5741                         perf_output_copy(handle, data->br_stack->entries, size);
5742                 } else {
5743                         /*
5744                          * we always store at least the value of nr
5745                          */
5746                         u64 nr = 0;
5747                         perf_output_put(handle, nr);
5748                 }
5749         }
5750 
5751         if (sample_type & PERF_SAMPLE_REGS_USER) {
5752                 u64 abi = data->regs_user.abi;
5753 
5754                 /*
5755                  * If there are no regs to dump, notice it through
5756                  * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
5757                  */
5758                 perf_output_put(handle, abi);
5759 
5760                 if (abi) {
5761                         u64 mask = event->attr.sample_regs_user;
5762                         perf_output_sample_regs(handle,
5763                                                 data->regs_user.regs,
5764                                                 mask);
5765                 }
5766         }
5767 
5768         if (sample_type & PERF_SAMPLE_STACK_USER) {
5769                 perf_output_sample_ustack(handle,
5770                                           data->stack_user_size,
5771                                           data->regs_user.regs);
5772         }
5773 
5774         if (sample_type & PERF_SAMPLE_WEIGHT)
5775                 perf_output_put(handle, data->weight);
5776 
5777         if (sample_type & PERF_SAMPLE_DATA_SRC)
5778                 perf_output_put(handle, data->data_src.val);
5779 
5780         if (sample_type & PERF_SAMPLE_TRANSACTION)
5781                 perf_output_put(handle, data->txn);
5782 
5783         if (sample_type & PERF_SAMPLE_REGS_INTR) {
5784                 u64 abi = data->regs_intr.abi;
5785                 /*
5786                  * If there are no regs to dump, notice it through
5787                  * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
5788                  */
5789                 perf_output_put(handle, abi);
5790 
5791                 if (abi) {
5792                         u64 mask = event->attr.sample_regs_intr;
5793 
5794                         perf_output_sample_regs(handle,
5795                                                 data->regs_intr.regs,
5796                                                 mask);
5797                 }
5798         }
5799 
5800         if (!event->attr.watermark) {
5801                 int wakeup_events = event->attr.wakeup_events;
5802 
5803                 if (wakeup_events) {
5804                         struct ring_buffer *rb = handle->rb;
5805                         int events = local_inc_return(&rb->events);
5806 
5807                         if (events >= wakeup_events) {
5808                                 local_sub(wakeup_events, &rb->events);
5809                                 local_inc(&rb->wakeup);
5810                         }
5811                 }
5812         }
5813 }
5814 
5815 void perf_prepare_sample(struct perf_event_header *header,
5816                          struct perf_sample_data *data,
5817                          struct perf_event *event,
5818                          struct pt_regs *regs)
5819 {
5820         u64 sample_type = event->attr.sample_type;
5821 
5822         header->type = PERF_RECORD_SAMPLE;
5823         header->size = sizeof(*header) + event->header_size;
5824 
5825         header->misc = 0;
5826         header->misc |= perf_misc_flags(regs);
5827 
5828         __perf_event_header__init_id(header, data, event);
5829 
5830         if (sample_type & PERF_SAMPLE_IP)
5831                 data->ip = perf_instruction_pointer(regs);
5832 
5833         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
5834                 int size = 1;
5835 
5836                 data->callchain = perf_callchain(event, regs);
5837 
5838                 if (data->callchain)
5839                         size += data->callchain->nr;
5840 
5841                 header->size += size * sizeof(u64);
5842         }
5843 
5844         if (sample_type & PERF_SAMPLE_RAW) {
5845                 struct perf_raw_record *raw = data->raw;
5846                 int size;
5847 
5848                 if (raw) {
5849                         struct perf_raw_frag *frag = &raw->frag;
5850                         u32 sum = 0;
5851 
5852                         do {
5853                                 sum += frag->size;
5854                                 if (perf_raw_frag_last(frag))
5855                                         break;
5856                                 frag = frag->next;
5857                         } while (1);
5858 
5859                         size = round_up(sum + sizeof(u32), sizeof(u64));
5860                         raw->size = size - sizeof(u32);
5861                         frag->pad = raw->size - sum;
5862                 } else {
5863                         size = sizeof(u64);
5864                 }
5865 
5866                 header->size += size;
5867         }
5868 
5869         if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
5870                 int size = sizeof(u64); /* nr */
5871                 if (data->br_stack) {
5872                         size += data->br_stack->nr
5873                               * sizeof(struct perf_branch_entry);
5874                 }
5875                 header->size += size;
5876         }
5877 
5878         if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER))
5879                 perf_sample_regs_user(&data->regs_user, regs,
5880                                       &data->regs_user_copy);
5881 
5882         if (sample_type & PERF_SAMPLE_REGS_USER) {
5883                 /* regs dump ABI info */
5884                 int size = sizeof(u64);
5885 
5886                 if (data->regs_user.regs) {
5887                         u64 mask = event->attr.sample_regs_user;
5888                         size += hweight64(mask) * sizeof(u64);
5889                 }
5890 
5891                 header->size += size;
5892         }
5893 
5894         if (sample_type & PERF_SAMPLE_STACK_USER) {
5895                 /*
5896                  * Either we need PERF_SAMPLE_STACK_USER bit to be allways
5897                  * processed as the last one or have additional check added
5898                  * in case new sample type is added, because we could eat
5899                  * up the rest of the sample size.
5900                  */
5901                 u16 stack_size = event->attr.sample_stack_user;
5902                 u16 size = sizeof(u64);
5903 
5904                 stack_size = perf_sample_ustack_size(stack_size, header->size,
5905                                                      data->regs_user.regs);
5906 
5907                 /*
5908                  * If there is something to dump, add space for the dump
5909                  * itself and for the field that tells the dynamic size,
5910                  * which is how many have been actually dumped.
5911                  */
5912                 if (stack_size)
5913                         size += sizeof(u64) + stack_size;
5914 
5915                 data->stack_user_size = stack_size;
5916                 header->size += size;
5917         }
5918 
5919         if (sample_type & PERF_SAMPLE_REGS_INTR) {
5920                 /* regs dump ABI info */
5921                 int size = sizeof(u64);
5922 
5923                 perf_sample_regs_intr(&data->regs_intr, regs);
5924 
5925                 if (data->regs_intr.regs) {
5926                         u64 mask = event->attr.sample_regs_intr;
5927 
5928                         size += hweight64(mask) * sizeof(u64);
5929                 }
5930 
5931                 header->size += size;
5932         }
5933 }
5934 
5935 static void __always_inline
5936 __perf_event_output(struct perf_event *event,
5937                     struct perf_sample_data *data,
5938                     struct pt_regs *regs,
5939                     int (*output_begin)(struct perf_output_handle *,
5940                                         struct perf_event *,
5941                                         unsigned int))
5942 {
5943         struct perf_output_handle handle;
5944         struct perf_event_header header;
5945 
5946         /* protect the callchain buffers */
5947         rcu_read_lock();
5948 
5949         perf_prepare_sample(&header, data, event, regs);
5950 
5951         if (output_begin(&handle, event, header.size))
5952                 goto exit;
5953 
5954         perf_output_sample(&handle, &header, data, event);
5955 
5956         perf_output_end(&handle);
5957 
5958 exit:
5959         rcu_read_unlock();
5960 }
5961 
5962 void
5963 perf_event_output_forward(struct perf_event *event,
5964                          struct perf_sample_data *data,
5965                          struct pt_regs *regs)
5966 {
5967         __perf_event_output(event, data, regs, perf_output_begin_forward);
5968 }
5969 
5970 void
5971 perf_event_output_backward(struct perf_event *event,
5972                            struct perf_sample_data *data,
5973                            struct pt_regs *regs)
5974 {
5975         __perf_event_output(event, data, regs, perf_output_begin_backward);
5976 }
5977 
5978 void
5979 perf_event_output(struct perf_event *event,
5980                   struct perf_sample_data *data,
5981                   struct pt_regs *regs)
5982 {
5983         __perf_event_output(event, data, regs, perf_output_begin);
5984 }
5985 
5986 /*
5987  * read event_id
5988  */
5989 
5990 struct perf_read_event {