1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Performance events core code: 4 * 5 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 6 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar 7 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra 8 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 9 */ 10 11 #include <linux/fs.h> 12 #include <linux/mm.h> 13 #include <linux/cpu.h> 14 #include <linux/smp.h> 15 #include <linux/idr.h> 16 #include <linux/file.h> 17 #include <linux/poll.h> 18 #include <linux/slab.h> 19 #include <linux/hash.h> 20 #include <linux/tick.h> 21 #include <linux/sysfs.h> 22 #include <linux/dcache.h> 23 #include <linux/percpu.h> 24 #include <linux/ptrace.h> 25 #include <linux/reboot.h> 26 #include <linux/vmstat.h> 27 #include <linux/device.h> 28 #include <linux/export.h> 29 #include <linux/vmalloc.h> 30 #include <linux/hardirq.h> 31 #include <linux/hugetlb.h> 32 #include <linux/rculist.h> 33 #include <linux/uaccess.h> 34 #include <linux/syscalls.h> 35 #include <linux/anon_inodes.h> 36 #include <linux/kernel_stat.h> 37 #include <linux/cgroup.h> 38 #include <linux/perf_event.h> 39 #include <linux/trace_events.h> 40 #include <linux/hw_breakpoint.h> 41 #include <linux/mm_types.h> 42 #include <linux/module.h> 43 #include <linux/mman.h> 44 #include <linux/compat.h> 45 #include <linux/bpf.h> 46 #include <linux/filter.h> 47 #include <linux/namei.h> 48 #include <linux/parser.h> 49 #include <linux/sched/clock.h> 50 #include <linux/sched/mm.h> 51 #include <linux/proc_ns.h> 52 #include <linux/mount.h> 53 #include <linux/min_heap.h> 54 #include <linux/highmem.h> 55 #include <linux/pgtable.h> 56 #include <linux/buildid.h> 57 58 #include "internal.h" 59 60 #include <asm/irq_regs.h> 61 62 typedef int (*remote_function_f)(void *); 63 64 struct remote_function_call { 65 struct task_struct *p; 66 remote_function_f func; 67 void *info; 68 int ret; 69 }; 70 71 static void remote_function(void *data) 72 { 73 struct remote_function_call *tfc = data; 74 struct task_struct *p = tfc->p; 75 76 if (p) { 77 /* -EAGAIN */ 78 if (task_cpu(p) != smp_processor_id()) 79 return; 80 81 /* 82 * Now that we're on right CPU with IRQs disabled, we can test 83 * if we hit the right task without races. 84 */ 85 86 tfc->ret = -ESRCH; /* No such (running) process */ 87 if (p != current) 88 return; 89 } 90 91 tfc->ret = tfc->func(tfc->info); 92 } 93 94 /** 95 * task_function_call - call a function on the cpu on which a task runs 96 * @p: the task to evaluate 97 * @func: the function to be called 98 * @info: the function call argument 99 * 100 * Calls the function @func when the task is currently running. This might 101 * be on the current CPU, which just calls the function directly. This will 102 * retry due to any failures in smp_call_function_single(), such as if the 103 * task_cpu() goes offline concurrently. 104 * 105 * returns @func return value or -ESRCH or -ENXIO when the process isn't running 106 */ 107 static int 108 task_function_call(struct task_struct *p, remote_function_f func, void *info) 109 { 110 struct remote_function_call data = { 111 .p = p, 112 .func = func, 113 .info = info, 114 .ret = -EAGAIN, 115 }; 116 int ret; 117 118 for (;;) { 119 ret = smp_call_function_single(task_cpu(p), remote_function, 120 &data, 1); 121 if (!ret) 122 ret = data.ret; 123 124 if (ret != -EAGAIN) 125 break; 126 127 cond_resched(); 128 } 129 130 return ret; 131 } 132 133 /** 134 * cpu_function_call - call a function on the cpu 135 * @cpu: target cpu to queue this function 136 * @func: the function to be called 137 * @info: the function call argument 138 * 139 * Calls the function @func on the remote cpu. 140 * 141 * returns: @func return value or -ENXIO when the cpu is offline 142 */ 143 static int cpu_function_call(int cpu, remote_function_f func, void *info) 144 { 145 struct remote_function_call data = { 146 .p = NULL, 147 .func = func, 148 .info = info, 149 .ret = -ENXIO, /* No such CPU */ 150 }; 151 152 smp_call_function_single(cpu, remote_function, &data, 1); 153 154 return data.ret; 155 } 156 157 static inline struct perf_cpu_context * 158 __get_cpu_context(struct perf_event_context *ctx) 159 { 160 return this_cpu_ptr(ctx->pmu->pmu_cpu_context); 161 } 162 163 static void perf_ctx_lock(struct perf_cpu_context *cpuctx, 164 struct perf_event_context *ctx) 165 { 166 raw_spin_lock(&cpuctx->ctx.lock); 167 if (ctx) 168 raw_spin_lock(&ctx->lock); 169 } 170 171 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx, 172 struct perf_event_context *ctx) 173 { 174 if (ctx) 175 raw_spin_unlock(&ctx->lock); 176 raw_spin_unlock(&cpuctx->ctx.lock); 177 } 178 179 #define TASK_TOMBSTONE ((void *)-1L) 180 181 static bool is_kernel_event(struct perf_event *event) 182 { 183 return READ_ONCE(event->owner) == TASK_TOMBSTONE; 184 } 185 186 /* 187 * On task ctx scheduling... 188 * 189 * When !ctx->nr_events a task context will not be scheduled. This means 190 * we can disable the scheduler hooks (for performance) without leaving 191 * pending task ctx state. 192 * 193 * This however results in two special cases: 194 * 195 * - removing the last event from a task ctx; this is relatively straight 196 * forward and is done in __perf_remove_from_context. 197 * 198 * - adding the first event to a task ctx; this is tricky because we cannot 199 * rely on ctx->is_active and therefore cannot use event_function_call(). 200 * See perf_install_in_context(). 201 * 202 * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set. 203 */ 204 205 typedef void (*event_f)(struct perf_event *, struct perf_cpu_context *, 206 struct perf_event_context *, void *); 207 208 struct event_function_struct { 209 struct perf_event *event; 210 event_f func; 211 void *data; 212 }; 213 214 static int event_function(void *info) 215 { 216 struct event_function_struct *efs = info; 217 struct perf_event *event = efs->event; 218 struct perf_event_context *ctx = event->ctx; 219 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 220 struct perf_event_context *task_ctx = cpuctx->task_ctx; 221 int ret = 0; 222 223 lockdep_assert_irqs_disabled(); 224 225 perf_ctx_lock(cpuctx, task_ctx); 226 /* 227 * Since we do the IPI call without holding ctx->lock things can have 228 * changed, double check we hit the task we set out to hit. 229 */ 230 if (ctx->task) { 231 if (ctx->task != current) { 232 ret = -ESRCH; 233 goto unlock; 234 } 235 236 /* 237 * We only use event_function_call() on established contexts, 238 * and event_function() is only ever called when active (or 239 * rather, we'll have bailed in task_function_call() or the 240 * above ctx->task != current test), therefore we must have 241 * ctx->is_active here. 242 */ 243 WARN_ON_ONCE(!ctx->is_active); 244 /* 245 * And since we have ctx->is_active, cpuctx->task_ctx must 246 * match. 247 */ 248 WARN_ON_ONCE(task_ctx != ctx); 249 } else { 250 WARN_ON_ONCE(&cpuctx->ctx != ctx); 251 } 252 253 efs->func(event, cpuctx, ctx, efs->data); 254 unlock: 255 perf_ctx_unlock(cpuctx, task_ctx); 256 257 return ret; 258 } 259 260 static void event_function_call(struct perf_event *event, event_f func, void *data) 261 { 262 struct perf_event_context *ctx = event->ctx; 263 struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */ 264 struct event_function_struct efs = { 265 .event = event, 266 .func = func, 267 .data = data, 268 }; 269 270 if (!event->parent) { 271 /* 272 * If this is a !child event, we must hold ctx::mutex to 273 * stabilize the event->ctx relation. See 274 * perf_event_ctx_lock(). 275 */ 276 lockdep_assert_held(&ctx->mutex); 277 } 278 279 if (!task) { 280 cpu_function_call(event->cpu, event_function, &efs); 281 return; 282 } 283 284 if (task == TASK_TOMBSTONE) 285 return; 286 287 again: 288 if (!task_function_call(task, event_function, &efs)) 289 return; 290 291 raw_spin_lock_irq(&ctx->lock); 292 /* 293 * Reload the task pointer, it might have been changed by 294 * a concurrent perf_event_context_sched_out(). 295 */ 296 task = ctx->task; 297 if (task == TASK_TOMBSTONE) { 298 raw_spin_unlock_irq(&ctx->lock); 299 return; 300 } 301 if (ctx->is_active) { 302 raw_spin_unlock_irq(&ctx->lock); 303 goto again; 304 } 305 func(event, NULL, ctx, data); 306 raw_spin_unlock_irq(&ctx->lock); 307 } 308 309 /* 310 * Similar to event_function_call() + event_function(), but hard assumes IRQs 311 * are already disabled and we're on the right CPU. 312 */ 313 static void event_function_local(struct perf_event *event, event_f func, void *data) 314 { 315 struct perf_event_context *ctx = event->ctx; 316 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 317 struct task_struct *task = READ_ONCE(ctx->task); 318 struct perf_event_context *task_ctx = NULL; 319 320 lockdep_assert_irqs_disabled(); 321 322 if (task) { 323 if (task == TASK_TOMBSTONE) 324 return; 325 326 task_ctx = ctx; 327 } 328 329 perf_ctx_lock(cpuctx, task_ctx); 330 331 task = ctx->task; 332 if (task == TASK_TOMBSTONE) 333 goto unlock; 334 335 if (task) { 336 /* 337 * We must be either inactive or active and the right task, 338 * otherwise we're screwed, since we cannot IPI to somewhere 339 * else. 340 */ 341 if (ctx->is_active) { 342 if (WARN_ON_ONCE(task != current)) 343 goto unlock; 344 345 if (WARN_ON_ONCE(cpuctx->task_ctx != ctx)) 346 goto unlock; 347 } 348 } else { 349 WARN_ON_ONCE(&cpuctx->ctx != ctx); 350 } 351 352 func(event, cpuctx, ctx, data); 353 unlock: 354 perf_ctx_unlock(cpuctx, task_ctx); 355 } 356 357 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\ 358 PERF_FLAG_FD_OUTPUT |\ 359 PERF_FLAG_PID_CGROUP |\ 360 PERF_FLAG_FD_CLOEXEC) 361 362 /* 363 * branch priv levels that need permission checks 364 */ 365 #define PERF_SAMPLE_BRANCH_PERM_PLM \ 366 (PERF_SAMPLE_BRANCH_KERNEL |\ 367 PERF_SAMPLE_BRANCH_HV) 368 369 enum event_type_t { 370 EVENT_FLEXIBLE = 0x1, 371 EVENT_PINNED = 0x2, 372 EVENT_TIME = 0x4, 373 /* see ctx_resched() for details */ 374 EVENT_CPU = 0x8, 375 EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED, 376 }; 377 378 /* 379 * perf_sched_events : >0 events exist 380 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu 381 */ 382 383 static void perf_sched_delayed(struct work_struct *work); 384 DEFINE_STATIC_KEY_FALSE(perf_sched_events); 385 static DECLARE_DELAYED_WORK(perf_sched_work, perf_sched_delayed); 386 static DEFINE_MUTEX(perf_sched_mutex); 387 static atomic_t perf_sched_count; 388 389 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events); 390 static DEFINE_PER_CPU(int, perf_sched_cb_usages); 391 static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events); 392 393 static atomic_t nr_mmap_events __read_mostly; 394 static atomic_t nr_comm_events __read_mostly; 395 static atomic_t nr_namespaces_events __read_mostly; 396 static atomic_t nr_task_events __read_mostly; 397 static atomic_t nr_freq_events __read_mostly; 398 static atomic_t nr_switch_events __read_mostly; 399 static atomic_t nr_ksymbol_events __read_mostly; 400 static atomic_t nr_bpf_events __read_mostly; 401 static atomic_t nr_cgroup_events __read_mostly; 402 static atomic_t nr_text_poke_events __read_mostly; 403 static atomic_t nr_build_id_events __read_mostly; 404 405 static LIST_HEAD(pmus); 406 static DEFINE_MUTEX(pmus_lock); 407 static struct srcu_struct pmus_srcu; 408 static cpumask_var_t perf_online_mask; 409 static struct kmem_cache *perf_event_cache; 410 411 /* 412 * perf event paranoia level: 413 * -1 - not paranoid at all 414 * 0 - disallow raw tracepoint access for unpriv 415 * 1 - disallow cpu events for unpriv 416 * 2 - disallow kernel profiling for unpriv 417 */ 418 int sysctl_perf_event_paranoid __read_mostly = 2; 419 420 /* Minimum for 512 kiB + 1 user control page */ 421 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */ 422 423 /* 424 * max perf event sample rate 425 */ 426 #define DEFAULT_MAX_SAMPLE_RATE 100000 427 #define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE) 428 #define DEFAULT_CPU_TIME_MAX_PERCENT 25 429 430 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE; 431 432 static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ); 433 static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS; 434 435 static int perf_sample_allowed_ns __read_mostly = 436 DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100; 437 438 static void update_perf_cpu_limits(void) 439 { 440 u64 tmp = perf_sample_period_ns; 441 442 tmp *= sysctl_perf_cpu_time_max_percent; 443 tmp = div_u64(tmp, 100); 444 if (!tmp) 445 tmp = 1; 446 447 WRITE_ONCE(perf_sample_allowed_ns, tmp); 448 } 449 450 static bool perf_rotate_context(struct perf_cpu_context *cpuctx); 451 452 int perf_proc_update_handler(struct ctl_table *table, int write, 453 void *buffer, size_t *lenp, loff_t *ppos) 454 { 455 int ret; 456 int perf_cpu = sysctl_perf_cpu_time_max_percent; 457 /* 458 * If throttling is disabled don't allow the write: 459 */ 460 if (write && (perf_cpu == 100 || perf_cpu == 0)) 461 return -EINVAL; 462 463 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 464 if (ret || !write) 465 return ret; 466 467 max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ); 468 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; 469 update_perf_cpu_limits(); 470 471 return 0; 472 } 473 474 int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT; 475 476 int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write, 477 void *buffer, size_t *lenp, loff_t *ppos) 478 { 479 int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 480 481 if (ret || !write) 482 return ret; 483 484 if (sysctl_perf_cpu_time_max_percent == 100 || 485 sysctl_perf_cpu_time_max_percent == 0) { 486 printk(KERN_WARNING 487 "perf: Dynamic interrupt throttling disabled, can hang your system!\n"); 488 WRITE_ONCE(perf_sample_allowed_ns, 0); 489 } else { 490 update_perf_cpu_limits(); 491 } 492 493 return 0; 494 } 495 496 /* 497 * perf samples are done in some very critical code paths (NMIs). 498 * If they take too much CPU time, the system can lock up and not 499 * get any real work done. This will drop the sample rate when 500 * we detect that events are taking too long. 501 */ 502 #define NR_ACCUMULATED_SAMPLES 128 503 static DEFINE_PER_CPU(u64, running_sample_length); 504 505 static u64 __report_avg; 506 static u64 __report_allowed; 507 508 static void perf_duration_warn(struct irq_work *w) 509 { 510 printk_ratelimited(KERN_INFO 511 "perf: interrupt took too long (%lld > %lld), lowering " 512 "kernel.perf_event_max_sample_rate to %d\n", 513 __report_avg, __report_allowed, 514 sysctl_perf_event_sample_rate); 515 } 516 517 static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn); 518 519 void perf_sample_event_took(u64 sample_len_ns) 520 { 521 u64 max_len = READ_ONCE(perf_sample_allowed_ns); 522 u64 running_len; 523 u64 avg_len; 524 u32 max; 525 526 if (max_len == 0) 527 return; 528 529 /* Decay the counter by 1 average sample. */ 530 running_len = __this_cpu_read(running_sample_length); 531 running_len -= running_len/NR_ACCUMULATED_SAMPLES; 532 running_len += sample_len_ns; 533 __this_cpu_write(running_sample_length, running_len); 534 535 /* 536 * Note: this will be biased artifically low until we have 537 * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us 538 * from having to maintain a count. 539 */ 540 avg_len = running_len/NR_ACCUMULATED_SAMPLES; 541 if (avg_len <= max_len) 542 return; 543 544 __report_avg = avg_len; 545 __report_allowed = max_len; 546 547 /* 548 * Compute a throttle threshold 25% below the current duration. 549 */ 550 avg_len += avg_len / 4; 551 max = (TICK_NSEC / 100) * sysctl_perf_cpu_time_max_percent; 552 if (avg_len < max) 553 max /= (u32)avg_len; 554 else 555 max = 1; 556 557 WRITE_ONCE(perf_sample_allowed_ns, avg_len); 558 WRITE_ONCE(max_samples_per_tick, max); 559 560 sysctl_perf_event_sample_rate = max * HZ; 561 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; 562 563 if (!irq_work_queue(&perf_duration_work)) { 564 early_printk("perf: interrupt took too long (%lld > %lld), lowering " 565 "kernel.perf_event_max_sample_rate to %d\n", 566 __report_avg, __report_allowed, 567 sysctl_perf_event_sample_rate); 568 } 569 } 570 571 static atomic64_t perf_event_id; 572 573 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, 574 enum event_type_t event_type); 575 576 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, 577 enum event_type_t event_type, 578 struct task_struct *task); 579 580 static void update_context_time(struct perf_event_context *ctx); 581 static u64 perf_event_time(struct perf_event *event); 582 583 void __weak perf_event_print_debug(void) { } 584 585 static inline u64 perf_clock(void) 586 { 587 return local_clock(); 588 } 589 590 static inline u64 perf_event_clock(struct perf_event *event) 591 { 592 return event->clock(); 593 } 594 595 /* 596 * State based event timekeeping... 597 * 598 * The basic idea is to use event->state to determine which (if any) time 599 * fields to increment with the current delta. This means we only need to 600 * update timestamps when we change state or when they are explicitly requested 601 * (read). 602 * 603 * Event groups make things a little more complicated, but not terribly so. The 604 * rules for a group are that if the group leader is OFF the entire group is 605 * OFF, irrespecive of what the group member states are. This results in 606 * __perf_effective_state(). 607 * 608 * A futher ramification is that when a group leader flips between OFF and 609 * !OFF, we need to update all group member times. 610 * 611 * 612 * NOTE: perf_event_time() is based on the (cgroup) context time, and thus we 613 * need to make sure the relevant context time is updated before we try and 614 * update our timestamps. 615 */ 616 617 static __always_inline enum perf_event_state 618 __perf_effective_state(struct perf_event *event) 619 { 620 struct perf_event *leader = event->group_leader; 621 622 if (leader->state <= PERF_EVENT_STATE_OFF) 623 return leader->state; 624 625 return event->state; 626 } 627 628 static __always_inline void 629 __perf_update_times(struct perf_event *event, u64 now, u64 *enabled, u64 *running) 630 { 631 enum perf_event_state state = __perf_effective_state(event); 632 u64 delta = now - event->tstamp; 633 634 *enabled = event->total_time_enabled; 635 if (state >= PERF_EVENT_STATE_INACTIVE) 636 *enabled += delta; 637 638 *running = event->total_time_running; 639 if (state >= PERF_EVENT_STATE_ACTIVE) 640 *running += delta; 641 } 642 643 static void perf_event_update_time(struct perf_event *event) 644 { 645 u64 now = perf_event_time(event); 646 647 __perf_update_times(event, now, &event->total_time_enabled, 648 &event->total_time_running); 649 event->tstamp = now; 650 } 651 652 static void perf_event_update_sibling_time(struct perf_event *leader) 653 { 654 struct perf_event *sibling; 655 656 for_each_sibling_event(sibling, leader) 657 perf_event_update_time(sibling); 658 } 659 660 static void 661 perf_event_set_state(struct perf_event *event, enum perf_event_state state) 662 { 663 if (event->state == state) 664 return; 665 666 perf_event_update_time(event); 667 /* 668 * If a group leader gets enabled/disabled all its siblings 669 * are affected too. 670 */ 671 if ((event->state < 0) ^ (state < 0)) 672 perf_event_update_sibling_time(event); 673 674 WRITE_ONCE(event->state, state); 675 } 676 677 /* 678 * UP store-release, load-acquire 679 */ 680 681 #define __store_release(ptr, val) \ 682 do { \ 683 barrier(); \ 684 WRITE_ONCE(*(ptr), (val)); \ 685 } while (0) 686 687 #define __load_acquire(ptr) \ 688 ({ \ 689 __unqual_scalar_typeof(*(ptr)) ___p = READ_ONCE(*(ptr)); \ 690 barrier(); \ 691 ___p; \ 692 }) 693 694 #ifdef CONFIG_CGROUP_PERF 695 696 static inline bool 697 perf_cgroup_match(struct perf_event *event) 698 { 699 struct perf_event_context *ctx = event->ctx; 700 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 701 702 /* @event doesn't care about cgroup */ 703 if (!event->cgrp) 704 return true; 705 706 /* wants specific cgroup scope but @cpuctx isn't associated with any */ 707 if (!cpuctx->cgrp) 708 return false; 709 710 /* 711 * Cgroup scoping is recursive. An event enabled for a cgroup is 712 * also enabled for all its descendant cgroups. If @cpuctx's 713 * cgroup is a descendant of @event's (the test covers identity 714 * case), it's a match. 715 */ 716 return cgroup_is_descendant(cpuctx->cgrp->css.cgroup, 717 event->cgrp->css.cgroup); 718 } 719 720 static inline void perf_detach_cgroup(struct perf_event *event) 721 { 722 css_put(&event->cgrp->css); 723 event->cgrp = NULL; 724 } 725 726 static inline int is_cgroup_event(struct perf_event *event) 727 { 728 return event->cgrp != NULL; 729 } 730 731 static inline u64 perf_cgroup_event_time(struct perf_event *event) 732 { 733 struct perf_cgroup_info *t; 734 735 t = per_cpu_ptr(event->cgrp->info, event->cpu); 736 return t->time; 737 } 738 739 static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now) 740 { 741 struct perf_cgroup_info *t; 742 743 t = per_cpu_ptr(event->cgrp->info, event->cpu); 744 if (!__load_acquire(&t->active)) 745 return t->time; 746 now += READ_ONCE(t->timeoffset); 747 return now; 748 } 749 750 static inline void __update_cgrp_time(struct perf_cgroup_info *info, u64 now, bool adv) 751 { 752 if (adv) 753 info->time += now - info->timestamp; 754 info->timestamp = now; 755 /* 756 * see update_context_time() 757 */ 758 WRITE_ONCE(info->timeoffset, info->time - info->timestamp); 759 } 760 761 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, bool final) 762 { 763 struct perf_cgroup *cgrp = cpuctx->cgrp; 764 struct cgroup_subsys_state *css; 765 struct perf_cgroup_info *info; 766 767 if (cgrp) { 768 u64 now = perf_clock(); 769 770 for (css = &cgrp->css; css; css = css->parent) { 771 cgrp = container_of(css, struct perf_cgroup, css); 772 info = this_cpu_ptr(cgrp->info); 773 774 __update_cgrp_time(info, now, true); 775 if (final) 776 __store_release(&info->active, 0); 777 } 778 } 779 } 780 781 static inline void update_cgrp_time_from_event(struct perf_event *event) 782 { 783 struct perf_cgroup_info *info; 784 struct perf_cgroup *cgrp; 785 786 /* 787 * ensure we access cgroup data only when needed and 788 * when we know the cgroup is pinned (css_get) 789 */ 790 if (!is_cgroup_event(event)) 791 return; 792 793 cgrp = perf_cgroup_from_task(current, event->ctx); 794 /* 795 * Do not update time when cgroup is not active 796 */ 797 if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup)) { 798 info = this_cpu_ptr(event->cgrp->info); 799 __update_cgrp_time(info, perf_clock(), true); 800 } 801 } 802 803 static inline void 804 perf_cgroup_set_timestamp(struct task_struct *task, 805 struct perf_event_context *ctx) 806 { 807 struct perf_cgroup *cgrp; 808 struct perf_cgroup_info *info; 809 struct cgroup_subsys_state *css; 810 811 /* 812 * ctx->lock held by caller 813 * ensure we do not access cgroup data 814 * unless we have the cgroup pinned (css_get) 815 */ 816 if (!task || !ctx->nr_cgroups) 817 return; 818 819 cgrp = perf_cgroup_from_task(task, ctx); 820 821 for (css = &cgrp->css; css; css = css->parent) { 822 cgrp = container_of(css, struct perf_cgroup, css); 823 info = this_cpu_ptr(cgrp->info); 824 __update_cgrp_time(info, ctx->timestamp, false); 825 __store_release(&info->active, 1); 826 } 827 } 828 829 static DEFINE_PER_CPU(struct list_head, cgrp_cpuctx_list); 830 831 #define PERF_CGROUP_SWOUT 0x1 /* cgroup switch out every event */ 832 #define PERF_CGROUP_SWIN 0x2 /* cgroup switch in events based on task */ 833 834 /* 835 * reschedule events based on the cgroup constraint of task. 836 * 837 * mode SWOUT : schedule out everything 838 * mode SWIN : schedule in based on cgroup for next 839 */ 840 static void perf_cgroup_switch(struct task_struct *task, int mode) 841 { 842 struct perf_cpu_context *cpuctx, *tmp; 843 struct list_head *list; 844 unsigned long flags; 845 846 /* 847 * Disable interrupts and preemption to avoid this CPU's 848 * cgrp_cpuctx_entry to change under us. 849 */ 850 local_irq_save(flags); 851 852 list = this_cpu_ptr(&cgrp_cpuctx_list); 853 list_for_each_entry_safe(cpuctx, tmp, list, cgrp_cpuctx_entry) { 854 WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0); 855 856 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 857 perf_pmu_disable(cpuctx->ctx.pmu); 858 859 if (mode & PERF_CGROUP_SWOUT) { 860 cpu_ctx_sched_out(cpuctx, EVENT_ALL); 861 /* 862 * must not be done before ctxswout due 863 * to event_filter_match() in event_sched_out() 864 */ 865 cpuctx->cgrp = NULL; 866 } 867 868 if (mode & PERF_CGROUP_SWIN) { 869 WARN_ON_ONCE(cpuctx->cgrp); 870 /* 871 * set cgrp before ctxsw in to allow 872 * event_filter_match() to not have to pass 873 * task around 874 * we pass the cpuctx->ctx to perf_cgroup_from_task() 875 * because cgorup events are only per-cpu 876 */ 877 cpuctx->cgrp = perf_cgroup_from_task(task, 878 &cpuctx->ctx); 879 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task); 880 } 881 perf_pmu_enable(cpuctx->ctx.pmu); 882 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 883 } 884 885 local_irq_restore(flags); 886 } 887 888 static inline void perf_cgroup_sched_out(struct task_struct *task, 889 struct task_struct *next) 890 { 891 struct perf_cgroup *cgrp1; 892 struct perf_cgroup *cgrp2 = NULL; 893 894 rcu_read_lock(); 895 /* 896 * we come here when we know perf_cgroup_events > 0 897 * we do not need to pass the ctx here because we know 898 * we are holding the rcu lock 899 */ 900 cgrp1 = perf_cgroup_from_task(task, NULL); 901 cgrp2 = perf_cgroup_from_task(next, NULL); 902 903 /* 904 * only schedule out current cgroup events if we know 905 * that we are switching to a different cgroup. Otherwise, 906 * do no touch the cgroup events. 907 */ 908 if (cgrp1 != cgrp2) 909 perf_cgroup_switch(task, PERF_CGROUP_SWOUT); 910 911 rcu_read_unlock(); 912 } 913 914 static inline void perf_cgroup_sched_in(struct task_struct *prev, 915 struct task_struct *task) 916 { 917 struct perf_cgroup *cgrp1; 918 struct perf_cgroup *cgrp2 = NULL; 919 920 rcu_read_lock(); 921 /* 922 * we come here when we know perf_cgroup_events > 0 923 * we do not need to pass the ctx here because we know 924 * we are holding the rcu lock 925 */ 926 cgrp1 = perf_cgroup_from_task(task, NULL); 927 cgrp2 = perf_cgroup_from_task(prev, NULL); 928 929 /* 930 * only need to schedule in cgroup events if we are changing 931 * cgroup during ctxsw. Cgroup events were not scheduled 932 * out of ctxsw out if that was not the case. 933 */ 934 if (cgrp1 != cgrp2) 935 perf_cgroup_switch(task, PERF_CGROUP_SWIN); 936 937 rcu_read_unlock(); 938 } 939 940 static int perf_cgroup_ensure_storage(struct perf_event *event, 941 struct cgroup_subsys_state *css) 942 { 943 struct perf_cpu_context *cpuctx; 944 struct perf_event **storage; 945 int cpu, heap_size, ret = 0; 946 947 /* 948 * Allow storage to have sufficent space for an iterator for each 949 * possibly nested cgroup plus an iterator for events with no cgroup. 950 */ 951 for (heap_size = 1; css; css = css->parent) 952 heap_size++; 953 954 for_each_possible_cpu(cpu) { 955 cpuctx = per_cpu_ptr(event->pmu->pmu_cpu_context, cpu); 956 if (heap_size <= cpuctx->heap_size) 957 continue; 958 959 storage = kmalloc_node(heap_size * sizeof(struct perf_event *), 960 GFP_KERNEL, cpu_to_node(cpu)); 961 if (!storage) { 962 ret = -ENOMEM; 963 break; 964 } 965 966 raw_spin_lock_irq(&cpuctx->ctx.lock); 967 if (cpuctx->heap_size < heap_size) { 968 swap(cpuctx->heap, storage); 969 if (storage == cpuctx->heap_default) 970 storage = NULL; 971 cpuctx->heap_size = heap_size; 972 } 973 raw_spin_unlock_irq(&cpuctx->ctx.lock); 974 975 kfree(storage); 976 } 977 978 return ret; 979 } 980 981 static inline int perf_cgroup_connect(int fd, struct perf_event *event, 982 struct perf_event_attr *attr, 983 struct perf_event *group_leader) 984 { 985 struct perf_cgroup *cgrp; 986 struct cgroup_subsys_state *css; 987 struct fd f = fdget(fd); 988 int ret = 0; 989 990 if (!f.file) 991 return -EBADF; 992 993 css = css_tryget_online_from_dir(f.file->f_path.dentry, 994 &perf_event_cgrp_subsys); 995 if (IS_ERR(css)) { 996 ret = PTR_ERR(css); 997 goto out; 998 } 999 1000 ret = perf_cgroup_ensure_storage(event, css); 1001 if (ret) 1002 goto out; 1003 1004 cgrp = container_of(css, struct perf_cgroup, css); 1005 event->cgrp = cgrp; 1006 1007 /* 1008 * all events in a group must monitor 1009 * the same cgroup because a task belongs 1010 * to only one perf cgroup at a time 1011 */ 1012 if (group_leader && group_leader->cgrp != cgrp) { 1013 perf_detach_cgroup(event); 1014 ret = -EINVAL; 1015 } 1016 out: 1017 fdput(f); 1018 return ret; 1019 } 1020 1021 static inline void 1022 perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx) 1023 { 1024 struct perf_cpu_context *cpuctx; 1025 1026 if (!is_cgroup_event(event)) 1027 return; 1028 1029 /* 1030 * Because cgroup events are always per-cpu events, 1031 * @ctx == &cpuctx->ctx. 1032 */ 1033 cpuctx = container_of(ctx, struct perf_cpu_context, ctx); 1034 1035 /* 1036 * Since setting cpuctx->cgrp is conditional on the current @cgrp 1037 * matching the event's cgroup, we must do this for every new event, 1038 * because if the first would mismatch, the second would not try again 1039 * and we would leave cpuctx->cgrp unset. 1040 */ 1041 if (ctx->is_active && !cpuctx->cgrp) { 1042 struct perf_cgroup *cgrp = perf_cgroup_from_task(current, ctx); 1043 1044 if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup)) 1045 cpuctx->cgrp = cgrp; 1046 } 1047 1048 if (ctx->nr_cgroups++) 1049 return; 1050 1051 list_add(&cpuctx->cgrp_cpuctx_entry, 1052 per_cpu_ptr(&cgrp_cpuctx_list, event->cpu)); 1053 } 1054 1055 static inline void 1056 perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx) 1057 { 1058 struct perf_cpu_context *cpuctx; 1059 1060 if (!is_cgroup_event(event)) 1061 return; 1062 1063 /* 1064 * Because cgroup events are always per-cpu events, 1065 * @ctx == &cpuctx->ctx. 1066 */ 1067 cpuctx = container_of(ctx, struct perf_cpu_context, ctx); 1068 1069 if (--ctx->nr_cgroups) 1070 return; 1071 1072 if (ctx->is_active && cpuctx->cgrp) 1073 cpuctx->cgrp = NULL; 1074 1075 list_del(&cpuctx->cgrp_cpuctx_entry); 1076 } 1077 1078 #else /* !CONFIG_CGROUP_PERF */ 1079 1080 static inline bool 1081 perf_cgroup_match(struct perf_event *event) 1082 { 1083 return true; 1084 } 1085 1086 static inline void perf_detach_cgroup(struct perf_event *event) 1087 {} 1088 1089 static inline int is_cgroup_event(struct perf_event *event) 1090 { 1091 return 0; 1092 } 1093 1094 static inline void update_cgrp_time_from_event(struct perf_event *event) 1095 { 1096 } 1097 1098 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, 1099 bool final) 1100 { 1101 } 1102 1103 static inline void perf_cgroup_sched_out(struct task_struct *task, 1104 struct task_struct *next) 1105 { 1106 } 1107 1108 static inline void perf_cgroup_sched_in(struct task_struct *prev, 1109 struct task_struct *task) 1110 { 1111 } 1112 1113 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event, 1114 struct perf_event_attr *attr, 1115 struct perf_event *group_leader) 1116 { 1117 return -EINVAL; 1118 } 1119 1120 static inline void 1121 perf_cgroup_set_timestamp(struct task_struct *task, 1122 struct perf_event_context *ctx) 1123 { 1124 } 1125 1126 static inline void 1127 perf_cgroup_switch(struct task_struct *task, struct task_struct *next) 1128 { 1129 } 1130 1131 static inline u64 perf_cgroup_event_time(struct perf_event *event) 1132 { 1133 return 0; 1134 } 1135 1136 static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now) 1137 { 1138 return 0; 1139 } 1140 1141 static inline void 1142 perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx) 1143 { 1144 } 1145 1146 static inline void 1147 perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx) 1148 { 1149 } 1150 #endif 1151 1152 /* 1153 * set default to be dependent on timer tick just 1154 * like original code 1155 */ 1156 #define PERF_CPU_HRTIMER (1000 / HZ) 1157 /* 1158 * function must be called with interrupts disabled 1159 */ 1160 static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr) 1161 { 1162 struct perf_cpu_context *cpuctx; 1163 bool rotations; 1164 1165 lockdep_assert_irqs_disabled(); 1166 1167 cpuctx = container_of(hr, struct perf_cpu_context, hrtimer); 1168 rotations = perf_rotate_context(cpuctx); 1169 1170 raw_spin_lock(&cpuctx->hrtimer_lock); 1171 if (rotations) 1172 hrtimer_forward_now(hr, cpuctx->hrtimer_interval); 1173 else 1174 cpuctx->hrtimer_active = 0; 1175 raw_spin_unlock(&cpuctx->hrtimer_lock); 1176 1177 return rotations ? HRTIMER_RESTART : HRTIMER_NORESTART; 1178 } 1179 1180 static void __perf_mux_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu) 1181 { 1182 struct hrtimer *timer = &cpuctx->hrtimer; 1183 struct pmu *pmu = cpuctx->ctx.pmu; 1184 u64 interval; 1185 1186 /* no multiplexing needed for SW PMU */ 1187 if (pmu->task_ctx_nr == perf_sw_context) 1188 return; 1189 1190 /* 1191 * check default is sane, if not set then force to 1192 * default interval (1/tick) 1193 */ 1194 interval = pmu->hrtimer_interval_ms; 1195 if (interval < 1) 1196 interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER; 1197 1198 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval); 1199 1200 raw_spin_lock_init(&cpuctx->hrtimer_lock); 1201 hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD); 1202 timer->function = perf_mux_hrtimer_handler; 1203 } 1204 1205 static int perf_mux_hrtimer_restart(struct perf_cpu_context *cpuctx) 1206 { 1207 struct hrtimer *timer = &cpuctx->hrtimer; 1208 struct pmu *pmu = cpuctx->ctx.pmu; 1209 unsigned long flags; 1210 1211 /* not for SW PMU */ 1212 if (pmu->task_ctx_nr == perf_sw_context) 1213 return 0; 1214 1215 raw_spin_lock_irqsave(&cpuctx->hrtimer_lock, flags); 1216 if (!cpuctx->hrtimer_active) { 1217 cpuctx->hrtimer_active = 1; 1218 hrtimer_forward_now(timer, cpuctx->hrtimer_interval); 1219 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED_HARD); 1220 } 1221 raw_spin_unlock_irqrestore(&cpuctx->hrtimer_lock, flags); 1222 1223 return 0; 1224 } 1225 1226 void perf_pmu_disable(struct pmu *pmu) 1227 { 1228 int *count = this_cpu_ptr(pmu->pmu_disable_count); 1229 if (!(*count)++) 1230 pmu->pmu_disable(pmu); 1231 } 1232 1233 void perf_pmu_enable(struct pmu *pmu) 1234 { 1235 int *count = this_cpu_ptr(pmu->pmu_disable_count); 1236 if (!--(*count)) 1237 pmu->pmu_enable(pmu); 1238 } 1239 1240 static DEFINE_PER_CPU(struct list_head, active_ctx_list); 1241 1242 /* 1243 * perf_event_ctx_activate(), perf_event_ctx_deactivate(), and 1244 * perf_event_task_tick() are fully serialized because they're strictly cpu 1245 * affine and perf_event_ctx{activate,deactivate} are called with IRQs 1246 * disabled, while perf_event_task_tick is called from IRQ context. 1247 */ 1248 static void perf_event_ctx_activate(struct perf_event_context *ctx) 1249 { 1250 struct list_head *head = this_cpu_ptr(&active_ctx_list); 1251 1252 lockdep_assert_irqs_disabled(); 1253 1254 WARN_ON(!list_empty(&ctx->active_ctx_list)); 1255 1256 list_add(&ctx->active_ctx_list, head); 1257 } 1258 1259 static void perf_event_ctx_deactivate(struct perf_event_context *ctx) 1260 { 1261 lockdep_assert_irqs_disabled(); 1262 1263 WARN_ON(list_empty(&ctx->active_ctx_list)); 1264 1265 list_del_init(&ctx->active_ctx_list); 1266 } 1267 1268 static void get_ctx(struct perf_event_context *ctx) 1269 { 1270 refcount_inc(&ctx->refcount); 1271 } 1272 1273 static void *alloc_task_ctx_data(struct pmu *pmu) 1274 { 1275 if (pmu->task_ctx_cache) 1276 return kmem_cache_zalloc(pmu->task_ctx_cache, GFP_KERNEL); 1277 1278 return NULL; 1279 } 1280 1281 static void free_task_ctx_data(struct pmu *pmu, void *task_ctx_data) 1282 { 1283 if (pmu->task_ctx_cache && task_ctx_data) 1284 kmem_cache_free(pmu->task_ctx_cache, task_ctx_data); 1285 } 1286 1287 static void free_ctx(struct rcu_head *head) 1288 { 1289 struct perf_event_context *ctx; 1290 1291 ctx = container_of(head, struct perf_event_context, rcu_head); 1292 free_task_ctx_data(ctx->pmu, ctx->task_ctx_data); 1293 kfree(ctx); 1294 } 1295 1296 static void put_ctx(struct perf_event_context *ctx) 1297 { 1298 if (refcount_dec_and_test(&ctx->refcount)) { 1299 if (ctx->parent_ctx) 1300 put_ctx(ctx->parent_ctx); 1301 if (ctx->task && ctx->task != TASK_TOMBSTONE) 1302 put_task_struct(ctx->task); 1303 call_rcu(&ctx->rcu_head, free_ctx); 1304 } 1305 } 1306 1307 /* 1308 * Because of perf_event::ctx migration in sys_perf_event_open::move_group and 1309 * perf_pmu_migrate_context() we need some magic. 1310 * 1311 * Those places that change perf_event::ctx will hold both 1312 * perf_event_ctx::mutex of the 'old' and 'new' ctx value. 1313 * 1314 * Lock ordering is by mutex address. There are two other sites where 1315 * perf_event_context::mutex nests and those are: 1316 * 1317 * - perf_event_exit_task_context() [ child , 0 ] 1318 * perf_event_exit_event() 1319 * put_event() [ parent, 1 ] 1320 * 1321 * - perf_event_init_context() [ parent, 0 ] 1322 * inherit_task_group() 1323 * inherit_group() 1324 * inherit_event() 1325 * perf_event_alloc() 1326 * perf_init_event() 1327 * perf_try_init_event() [ child , 1 ] 1328 * 1329 * While it appears there is an obvious deadlock here -- the parent and child 1330 * nesting levels are inverted between the two. This is in fact safe because 1331 * life-time rules separate them. That is an exiting task cannot fork, and a 1332 * spawning task cannot (yet) exit. 1333 * 1334 * But remember that these are parent<->child context relations, and 1335 * migration does not affect children, therefore these two orderings should not 1336 * interact. 1337 * 1338 * The change in perf_event::ctx does not affect children (as claimed above) 1339 * because the sys_perf_event_open() case will install a new event and break 1340 * the ctx parent<->child relation, and perf_pmu_migrate_context() is only 1341 * concerned with cpuctx and that doesn't have children. 1342 * 1343 * The places that change perf_event::ctx will issue: 1344 * 1345 * perf_remove_from_context(); 1346 * synchronize_rcu(); 1347 * perf_install_in_context(); 1348 * 1349 * to affect the change. The remove_from_context() + synchronize_rcu() should 1350 * quiesce the event, after which we can install it in the new location. This 1351 * means that only external vectors (perf_fops, prctl) can perturb the event 1352 * while in transit. Therefore all such accessors should also acquire 1353 * perf_event_context::mutex to serialize against this. 1354 * 1355 * However; because event->ctx can change while we're waiting to acquire 1356 * ctx->mutex we must be careful and use the below perf_event_ctx_lock() 1357 * function. 1358 * 1359 * Lock order: 1360 * exec_update_lock 1361 * task_struct::perf_event_mutex 1362 * perf_event_context::mutex 1363 * perf_event::child_mutex; 1364 * perf_event_context::lock 1365 * perf_event::mmap_mutex 1366 * mmap_lock 1367 * perf_addr_filters_head::lock 1368 * 1369 * cpu_hotplug_lock 1370 * pmus_lock 1371 * cpuctx->mutex / perf_event_context::mutex 1372 */ 1373 static struct perf_event_context * 1374 perf_event_ctx_lock_nested(struct perf_event *event, int nesting) 1375 { 1376 struct perf_event_context *ctx; 1377 1378 again: 1379 rcu_read_lock(); 1380 ctx = READ_ONCE(event->ctx); 1381 if (!refcount_inc_not_zero(&ctx->refcount)) { 1382 rcu_read_unlock(); 1383 goto again; 1384 } 1385 rcu_read_unlock(); 1386 1387 mutex_lock_nested(&ctx->mutex, nesting); 1388 if (event->ctx != ctx) { 1389 mutex_unlock(&ctx->mutex); 1390 put_ctx(ctx); 1391 goto again; 1392 } 1393 1394 return ctx; 1395 } 1396 1397 static inline struct perf_event_context * 1398 perf_event_ctx_lock(struct perf_event *event) 1399 { 1400 return perf_event_ctx_lock_nested(event, 0); 1401 } 1402 1403 static void perf_event_ctx_unlock(struct perf_event *event, 1404 struct perf_event_context *ctx) 1405 { 1406 mutex_unlock(&ctx->mutex); 1407 put_ctx(ctx); 1408 } 1409 1410 /* 1411 * This must be done under the ctx->lock, such as to serialize against 1412 * context_equiv(), therefore we cannot call put_ctx() since that might end up 1413 * calling scheduler related locks and ctx->lock nests inside those. 1414 */ 1415 static __must_check struct perf_event_context * 1416 unclone_ctx(struct perf_event_context *ctx) 1417 { 1418 struct perf_event_context *parent_ctx = ctx->parent_ctx; 1419 1420 lockdep_assert_held(&ctx->lock); 1421 1422 if (parent_ctx) 1423 ctx->parent_ctx = NULL; 1424 ctx->generation++; 1425 1426 return parent_ctx; 1427 } 1428 1429 static u32 perf_event_pid_type(struct perf_event *event, struct task_struct *p, 1430 enum pid_type type) 1431 { 1432 u32 nr; 1433 /* 1434 * only top level events have the pid namespace they were created in 1435 */ 1436 if (event->parent) 1437 event = event->parent; 1438 1439 nr = __task_pid_nr_ns(p, type, event->ns); 1440 /* avoid -1 if it is idle thread or runs in another ns */ 1441 if (!nr && !pid_alive(p)) 1442 nr = -1; 1443 return nr; 1444 } 1445 1446 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p) 1447 { 1448 return perf_event_pid_type(event, p, PIDTYPE_TGID); 1449 } 1450 1451 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p) 1452 { 1453 return perf_event_pid_type(event, p, PIDTYPE_PID); 1454 } 1455 1456 /* 1457 * If we inherit events we want to return the parent event id 1458 * to userspace. 1459 */ 1460 static u64 primary_event_id(struct perf_event *event) 1461 { 1462 u64 id = event->id; 1463 1464 if (event->parent) 1465 id = event->parent->id; 1466 1467 return id; 1468 } 1469 1470 /* 1471 * Get the perf_event_context for a task and lock it. 1472 * 1473 * This has to cope with the fact that until it is locked, 1474 * the context could get moved to another task. 1475 */ 1476 static struct perf_event_context * 1477 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags) 1478 { 1479 struct perf_event_context *ctx; 1480 1481 retry: 1482 /* 1483 * One of the few rules of preemptible RCU is that one cannot do 1484 * rcu_read_unlock() while holding a scheduler (or nested) lock when 1485 * part of the read side critical section was irqs-enabled -- see 1486 * rcu_read_unlock_special(). 1487 * 1488 * Since ctx->lock nests under rq->lock we must ensure the entire read 1489 * side critical section has interrupts disabled. 1490 */ 1491 local_irq_save(*flags); 1492 rcu_read_lock(); 1493 ctx = rcu_dereference(task->perf_event_ctxp[ctxn]); 1494 if (ctx) { 1495 /* 1496 * If this context is a clone of another, it might 1497 * get swapped for another underneath us by 1498 * perf_event_task_sched_out, though the 1499 * rcu_read_lock() protects us from any context 1500 * getting freed. Lock the context and check if it 1501 * got swapped before we could get the lock, and retry 1502 * if so. If we locked the right context, then it 1503 * can't get swapped on us any more. 1504 */ 1505 raw_spin_lock(&ctx->lock); 1506 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) { 1507 raw_spin_unlock(&ctx->lock); 1508 rcu_read_unlock(); 1509 local_irq_restore(*flags); 1510 goto retry; 1511 } 1512 1513 if (ctx->task == TASK_TOMBSTONE || 1514 !refcount_inc_not_zero(&ctx->refcount)) { 1515 raw_spin_unlock(&ctx->lock); 1516 ctx = NULL; 1517 } else { 1518 WARN_ON_ONCE(ctx->task != task); 1519 } 1520 } 1521 rcu_read_unlock(); 1522 if (!ctx) 1523 local_irq_restore(*flags); 1524 return ctx; 1525 } 1526 1527 /* 1528 * Get the context for a task and increment its pin_count so it 1529 * can't get swapped to another task. This also increments its 1530 * reference count so that the context can't get freed. 1531 */ 1532 static struct perf_event_context * 1533 perf_pin_task_context(struct task_struct *task, int ctxn) 1534 { 1535 struct perf_event_context *ctx; 1536 unsigned long flags; 1537 1538 ctx = perf_lock_task_context(task, ctxn, &flags); 1539 if (ctx) { 1540 ++ctx->pin_count; 1541 raw_spin_unlock_irqrestore(&ctx->lock, flags); 1542 } 1543 return ctx; 1544 } 1545 1546 static void perf_unpin_context(struct perf_event_context *ctx) 1547 { 1548 unsigned long flags; 1549 1550 raw_spin_lock_irqsave(&ctx->lock, flags); 1551 --ctx->pin_count; 1552 raw_spin_unlock_irqrestore(&ctx->lock, flags); 1553 } 1554 1555 /* 1556 * Update the record of the current time in a context. 1557 */ 1558 static void __update_context_time(struct perf_event_context *ctx, bool adv) 1559 { 1560 u64 now = perf_clock(); 1561 1562 if (adv) 1563 ctx->time += now - ctx->timestamp; 1564 ctx->timestamp = now; 1565 1566 /* 1567 * The above: time' = time + (now - timestamp), can be re-arranged 1568 * into: time` = now + (time - timestamp), which gives a single value 1569 * offset to compute future time without locks on. 1570 * 1571 * See perf_event_time_now(), which can be used from NMI context where 1572 * it's (obviously) not possible to acquire ctx->lock in order to read 1573 * both the above values in a consistent manner. 1574 */ 1575 WRITE_ONCE(ctx->timeoffset, ctx->time - ctx->timestamp); 1576 } 1577 1578 static void update_context_time(struct perf_event_context *ctx) 1579 { 1580 __update_context_time(ctx, true); 1581 } 1582 1583 static u64 perf_event_time(struct perf_event *event) 1584 { 1585 struct perf_event_context *ctx = event->ctx; 1586 1587 if (unlikely(!ctx)) 1588 return 0; 1589 1590 if (is_cgroup_event(event)) 1591 return perf_cgroup_event_time(event); 1592 1593 return ctx->time; 1594 } 1595 1596 static u64 perf_event_time_now(struct perf_event *event, u64 now) 1597 { 1598 struct perf_event_context *ctx = event->ctx; 1599 1600 if (unlikely(!ctx)) 1601 return 0; 1602 1603 if (is_cgroup_event(event)) 1604 return perf_cgroup_event_time_now(event, now); 1605 1606 if (!(__load_acquire(&ctx->is_active) & EVENT_TIME)) 1607 return ctx->time; 1608 1609 now += READ_ONCE(ctx->timeoffset); 1610 return now; 1611 } 1612 1613 static enum event_type_t get_event_type(struct perf_event *event) 1614 { 1615 struct perf_event_context *ctx = event->ctx; 1616 enum event_type_t event_type; 1617 1618 lockdep_assert_held(&ctx->lock); 1619 1620 /* 1621 * It's 'group type', really, because if our group leader is 1622 * pinned, so are we. 1623 */ 1624 if (event->group_leader != event) 1625 event = event->group_leader; 1626 1627 event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE; 1628 if (!ctx->task) 1629 event_type |= EVENT_CPU; 1630 1631 return event_type; 1632 } 1633 1634 /* 1635 * Helper function to initialize event group nodes. 1636 */ 1637 static void init_event_group(struct perf_event *event) 1638 { 1639 RB_CLEAR_NODE(&event->group_node); 1640 event->group_index = 0; 1641 } 1642 1643 /* 1644 * Extract pinned or flexible groups from the context 1645 * based on event attrs bits. 1646 */ 1647 static struct perf_event_groups * 1648 get_event_groups(struct perf_event *event, struct perf_event_context *ctx) 1649 { 1650 if (event->attr.pinned) 1651 return &ctx->pinned_groups; 1652 else 1653 return &ctx->flexible_groups; 1654 } 1655 1656 /* 1657 * Helper function to initializes perf_event_group trees. 1658 */ 1659 static void perf_event_groups_init(struct perf_event_groups *groups) 1660 { 1661 groups->tree = RB_ROOT; 1662 groups->index = 0; 1663 } 1664 1665 static inline struct cgroup *event_cgroup(const struct perf_event *event) 1666 { 1667 struct cgroup *cgroup = NULL; 1668 1669 #ifdef CONFIG_CGROUP_PERF 1670 if (event->cgrp) 1671 cgroup = event->cgrp->css.cgroup; 1672 #endif 1673 1674 return cgroup; 1675 } 1676 1677 /* 1678 * Compare function for event groups; 1679 * 1680 * Implements complex key that first sorts by CPU and then by virtual index 1681 * which provides ordering when rotating groups for the same CPU. 1682 */ 1683 static __always_inline int 1684 perf_event_groups_cmp(const int left_cpu, const struct cgroup *left_cgroup, 1685 const u64 left_group_index, const struct perf_event *right) 1686 { 1687 if (left_cpu < right->cpu) 1688 return -1; 1689 if (left_cpu > right->cpu) 1690 return 1; 1691 1692 #ifdef CONFIG_CGROUP_PERF 1693 { 1694 const struct cgroup *right_cgroup = event_cgroup(right); 1695 1696 if (left_cgroup != right_cgroup) { 1697 if (!left_cgroup) { 1698 /* 1699 * Left has no cgroup but right does, no 1700 * cgroups come first. 1701 */ 1702 return -1; 1703 } 1704 if (!right_cgroup) { 1705 /* 1706 * Right has no cgroup but left does, no 1707 * cgroups come first. 1708 */ 1709 return 1; 1710 } 1711 /* Two dissimilar cgroups, order by id. */ 1712 if (cgroup_id(left_cgroup) < cgroup_id(right_cgroup)) 1713 return -1; 1714 1715 return 1; 1716 } 1717 } 1718 #endif 1719 1720 if (left_group_index < right->group_index) 1721 return -1; 1722 if (left_group_index > right->group_index) 1723 return 1; 1724 1725 return 0; 1726 } 1727 1728 #define __node_2_pe(node) \ 1729 rb_entry((node), struct perf_event, group_node) 1730 1731 static inline bool __group_less(struct rb_node *a, const struct rb_node *b) 1732 { 1733 struct perf_event *e = __node_2_pe(a); 1734 return perf_event_groups_cmp(e->cpu, event_cgroup(e), e->group_index, 1735 __node_2_pe(b)) < 0; 1736 } 1737 1738 struct __group_key { 1739 int cpu; 1740 struct cgroup *cgroup; 1741 }; 1742 1743 static inline int __group_cmp(const void *key, const struct rb_node *node) 1744 { 1745 const struct __group_key *a = key; 1746 const struct perf_event *b = __node_2_pe(node); 1747 1748 /* partial/subtree match: @cpu, @cgroup; ignore: @group_index */ 1749 return perf_event_groups_cmp(a->cpu, a->cgroup, b->group_index, b); 1750 } 1751 1752 /* 1753 * Insert @event into @groups' tree; using {@event->cpu, ++@groups->index} for 1754 * key (see perf_event_groups_less). This places it last inside the CPU 1755 * subtree. 1756 */ 1757 static void 1758 perf_event_groups_insert(struct perf_event_groups *groups, 1759 struct perf_event *event) 1760 { 1761 event->group_index = ++groups->index; 1762 1763 rb_add(&event->group_node, &groups->tree, __group_less); 1764 } 1765 1766 /* 1767 * Helper function to insert event into the pinned or flexible groups. 1768 */ 1769 static void 1770 add_event_to_groups(struct perf_event *event, struct perf_event_context *ctx) 1771 { 1772 struct perf_event_groups *groups; 1773 1774 groups = get_event_groups(event, ctx); 1775 perf_event_groups_insert(groups, event); 1776 } 1777 1778 /* 1779 * Delete a group from a tree. 1780 */ 1781 static void 1782 perf_event_groups_delete(struct perf_event_groups *groups, 1783 struct perf_event *event) 1784 { 1785 WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) || 1786 RB_EMPTY_ROOT(&groups->tree)); 1787 1788 rb_erase(&event->group_node, &groups->tree); 1789 init_event_group(event); 1790 } 1791 1792 /* 1793 * Helper function to delete event from its groups. 1794 */ 1795 static void 1796 del_event_from_groups(struct perf_event *event, struct perf_event_context *ctx) 1797 { 1798 struct perf_event_groups *groups; 1799 1800 groups = get_event_groups(event, ctx); 1801 perf_event_groups_delete(groups, event); 1802 } 1803 1804 /* 1805 * Get the leftmost event in the cpu/cgroup subtree. 1806 */ 1807 static struct perf_event * 1808 perf_event_groups_first(struct perf_event_groups *groups, int cpu, 1809 struct cgroup *cgrp) 1810 { 1811 struct __group_key key = { 1812 .cpu = cpu, 1813 .cgroup = cgrp, 1814 }; 1815 struct rb_node *node; 1816 1817 node = rb_find_first(&key, &groups->tree, __group_cmp); 1818 if (node) 1819 return __node_2_pe(node); 1820 1821 return NULL; 1822 } 1823 1824 /* 1825 * Like rb_entry_next_safe() for the @cpu subtree. 1826 */ 1827 static struct perf_event * 1828 perf_event_groups_next(struct perf_event *event) 1829 { 1830 struct __group_key key = { 1831 .cpu = event->cpu, 1832 .cgroup = event_cgroup(event), 1833 }; 1834 struct rb_node *next; 1835 1836 next = rb_next_match(&key, &event->group_node, __group_cmp); 1837 if (next) 1838 return __node_2_pe(next); 1839 1840 return NULL; 1841 } 1842 1843 /* 1844 * Iterate through the whole groups tree. 1845 */ 1846 #define perf_event_groups_for_each(event, groups) \ 1847 for (event = rb_entry_safe(rb_first(&((groups)->tree)), \ 1848 typeof(*event), group_node); event; \ 1849 event = rb_entry_safe(rb_next(&event->group_node), \ 1850 typeof(*event), group_node)) 1851 1852 /* 1853 * Add an event from the lists for its context. 1854 * Must be called with ctx->mutex and ctx->lock held. 1855 */ 1856 static void 1857 list_add_event(struct perf_event *event, struct perf_event_context *ctx) 1858 { 1859 lockdep_assert_held(&ctx->lock); 1860 1861 WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT); 1862 event->attach_state |= PERF_ATTACH_CONTEXT; 1863 1864 event->tstamp = perf_event_time(event); 1865 1866 /* 1867 * If we're a stand alone event or group leader, we go to the context 1868 * list, group events are kept attached to the group so that 1869 * perf_group_detach can, at all times, locate all siblings. 1870 */ 1871 if (event->group_leader == event) { 1872 event->group_caps = event->event_caps; 1873 add_event_to_groups(event, ctx); 1874 } 1875 1876 list_add_rcu(&event->event_entry, &ctx->event_list); 1877 ctx->nr_events++; 1878 if (event->attr.inherit_stat) 1879 ctx->nr_stat++; 1880 1881 if (event->state > PERF_EVENT_STATE_OFF) 1882 perf_cgroup_event_enable(event, ctx); 1883 1884 ctx->generation++; 1885 } 1886 1887 /* 1888 * Initialize event state based on the perf_event_attr::disabled. 1889 */ 1890 static inline void perf_event__state_init(struct perf_event *event) 1891 { 1892 event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF : 1893 PERF_EVENT_STATE_INACTIVE; 1894 } 1895 1896 static void __perf_event_read_size(struct perf_event *event, int nr_siblings) 1897 { 1898 int entry = sizeof(u64); /* value */ 1899 int size = 0; 1900 int nr = 1; 1901 1902 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 1903 size += sizeof(u64); 1904 1905 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 1906 size += sizeof(u64); 1907 1908 if (event->attr.read_format & PERF_FORMAT_ID) 1909 entry += sizeof(u64); 1910 1911 if (event->attr.read_format & PERF_FORMAT_GROUP) { 1912 nr += nr_siblings; 1913 size += sizeof(u64); 1914 } 1915 1916 size += entry * nr; 1917 event->read_size = size; 1918 } 1919 1920 static void __perf_event_header_size(struct perf_event *event, u64 sample_type) 1921 { 1922 struct perf_sample_data *data; 1923 u16 size = 0; 1924 1925 if (sample_type & PERF_SAMPLE_IP) 1926 size += sizeof(data->ip); 1927 1928 if (sample_type & PERF_SAMPLE_ADDR) 1929 size += sizeof(data->addr); 1930 1931 if (sample_type & PERF_SAMPLE_PERIOD) 1932 size += sizeof(data->period); 1933 1934 if (sample_type & PERF_SAMPLE_WEIGHT_TYPE) 1935 size += sizeof(data->weight.full); 1936 1937 if (sample_type & PERF_SAMPLE_READ) 1938 size += event->read_size; 1939 1940 if (sample_type & PERF_SAMPLE_DATA_SRC) 1941 size += sizeof(data->data_src.val); 1942 1943 if (sample_type & PERF_SAMPLE_TRANSACTION) 1944 size += sizeof(data->txn); 1945 1946 if (sample_type & PERF_SAMPLE_PHYS_ADDR) 1947 size += sizeof(data->phys_addr); 1948 1949 if (sample_type & PERF_SAMPLE_CGROUP) 1950 size += sizeof(data->cgroup); 1951 1952 if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE) 1953 size += sizeof(data->data_page_size); 1954 1955 if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) 1956 size += sizeof(data->code_page_size); 1957 1958 event->header_size = size; 1959 } 1960 1961 /* 1962 * Called at perf_event creation and when events are attached/detached from a 1963 * group. 1964 */ 1965 static void perf_event__header_size(struct perf_event *event) 1966 { 1967 __perf_event_read_size(event, 1968 event->group_leader->nr_siblings); 1969 __perf_event_header_size(event, event->attr.sample_type); 1970 } 1971 1972 static void perf_event__id_header_size(struct perf_event *event) 1973 { 1974 struct perf_sample_data *data; 1975 u64 sample_type = event->attr.sample_type; 1976 u16 size = 0; 1977 1978 if (sample_type & PERF_SAMPLE_TID) 1979 size += sizeof(data->tid_entry); 1980 1981 if (sample_type & PERF_SAMPLE_TIME) 1982 size += sizeof(data->time); 1983 1984 if (sample_type & PERF_SAMPLE_IDENTIFIER) 1985 size += sizeof(data->id); 1986 1987 if (sample_type & PERF_SAMPLE_ID) 1988 size += sizeof(data->id); 1989 1990 if (sample_type & PERF_SAMPLE_STREAM_ID) 1991 size += sizeof(data->stream_id); 1992 1993 if (sample_type & PERF_SAMPLE_CPU) 1994 size += sizeof(data->cpu_entry); 1995 1996 event->id_header_size = size; 1997 } 1998 1999 static bool perf_event_validate_size(struct perf_event *event) 2000 { 2001 /* 2002 * The values computed here will be over-written when we actually 2003 * attach the event. 2004 */ 2005 __perf_event_read_size(event, event->group_leader->nr_siblings + 1); 2006 __perf_event_header_size(event, event->attr.sample_type & ~PERF_SAMPLE_READ); 2007 perf_event__id_header_size(event); 2008 2009 /* 2010 * Sum the lot; should not exceed the 64k limit we have on records. 2011 * Conservative limit to allow for callchains and other variable fields. 2012 */ 2013 if (event->read_size + event->header_size + 2014 event->id_header_size + sizeof(struct perf_event_header) >= 16*1024) 2015 return false; 2016 2017 return true; 2018 } 2019 2020 static void perf_group_attach(struct perf_event *event) 2021 { 2022 struct perf_event *group_leader = event->group_leader, *pos; 2023 2024 lockdep_assert_held(&event->ctx->lock); 2025 2026 /* 2027 * We can have double attach due to group movement in perf_event_open. 2028 */ 2029 if (event->attach_state & PERF_ATTACH_GROUP) 2030 return; 2031 2032 event->attach_state |= PERF_ATTACH_GROUP; 2033 2034 if (group_leader == event) 2035 return; 2036 2037 WARN_ON_ONCE(group_leader->ctx != event->ctx); 2038 2039 group_leader->group_caps &= event->event_caps; 2040 2041 list_add_tail(&event->sibling_list, &group_leader->sibling_list); 2042 group_leader->nr_siblings++; 2043 2044 perf_event__header_size(group_leader); 2045 2046 for_each_sibling_event(pos, group_leader) 2047 perf_event__header_size(pos); 2048 } 2049 2050 /* 2051 * Remove an event from the lists for its context. 2052 * Must be called with ctx->mutex and ctx->lock held. 2053 */ 2054 static void 2055 list_del_event(struct perf_event *event, struct perf_event_context *ctx) 2056 { 2057 WARN_ON_ONCE(event->ctx != ctx); 2058 lockdep_assert_held(&ctx->lock); 2059 2060 /* 2061 * We can have double detach due to exit/hot-unplug + close. 2062 */ 2063 if (!(event->attach_state & PERF_ATTACH_CONTEXT)) 2064 return; 2065 2066 event->attach_state &= ~PERF_ATTACH_CONTEXT; 2067 2068 ctx->nr_events--; 2069 if (event->attr.inherit_stat) 2070 ctx->nr_stat--; 2071 2072 list_del_rcu(&event->event_entry); 2073 2074 if (event->group_leader == event) 2075 del_event_from_groups(event, ctx); 2076 2077 /* 2078 * If event was in error state, then keep it 2079 * that way, otherwise bogus counts will be 2080 * returned on read(). The only way to get out 2081 * of error state is by explicit re-enabling 2082 * of the event 2083 */ 2084 if (event->state > PERF_EVENT_STATE_OFF) { 2085 perf_cgroup_event_disable(event, ctx); 2086 perf_event_set_state(event, PERF_EVENT_STATE_OFF); 2087 } 2088 2089 ctx->generation++; 2090 } 2091 2092 static int 2093 perf_aux_output_match(struct perf_event *event, struct perf_event *aux_event) 2094 { 2095 if (!has_aux(aux_event)) 2096 return 0; 2097 2098 if (!event->pmu->aux_output_match) 2099 return 0; 2100 2101 return event->pmu->aux_output_match(aux_event); 2102 } 2103 2104 static void put_event(struct perf_event *event); 2105 static void event_sched_out(struct perf_event *event, 2106 struct perf_cpu_context *cpuctx, 2107 struct perf_event_context *ctx); 2108 2109 static void perf_put_aux_event(struct perf_event *event) 2110 { 2111 struct perf_event_context *ctx = event->ctx; 2112 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 2113 struct perf_event *iter; 2114 2115 /* 2116 * If event uses aux_event tear down the link 2117 */ 2118 if (event->aux_event) { 2119 iter = event->aux_event; 2120 event->aux_event = NULL; 2121 put_event(iter); 2122 return; 2123 } 2124 2125 /* 2126 * If the event is an aux_event, tear down all links to 2127 * it from other events. 2128 */ 2129 for_each_sibling_event(iter, event->group_leader) { 2130 if (iter->aux_event != event) 2131 continue; 2132 2133 iter->aux_event = NULL; 2134 put_event(event); 2135 2136 /* 2137 * If it's ACTIVE, schedule it out and put it into ERROR 2138 * state so that we don't try to schedule it again. Note 2139 * that perf_event_enable() will clear the ERROR status. 2140 */ 2141 event_sched_out(iter, cpuctx, ctx); 2142 perf_event_set_state(event, PERF_EVENT_STATE_ERROR); 2143 } 2144 } 2145 2146 static bool perf_need_aux_event(struct perf_event *event) 2147 { 2148 return !!event->attr.aux_output || !!event->attr.aux_sample_size; 2149 } 2150 2151 static int perf_get_aux_event(struct perf_event *event, 2152 struct perf_event *group_leader) 2153 { 2154 /* 2155 * Our group leader must be an aux event if we want to be 2156 * an aux_output. This way, the aux event will precede its 2157 * aux_output events in the group, and therefore will always 2158 * schedule first. 2159 */ 2160 if (!group_leader) 2161 return 0; 2162 2163 /* 2164 * aux_output and aux_sample_size are mutually exclusive. 2165 */ 2166 if (event->attr.aux_output && event->attr.aux_sample_size) 2167 return 0; 2168 2169 if (event->attr.aux_output && 2170 !perf_aux_output_match(event, group_leader)) 2171 return 0; 2172 2173 if (event->attr.aux_sample_size && !group_leader->pmu->snapshot_aux) 2174 return 0; 2175 2176 if (!atomic_long_inc_not_zero(&group_leader->refcount)) 2177 return 0; 2178 2179 /* 2180 * Link aux_outputs to their aux event; this is undone in 2181 * perf_group_detach() by perf_put_aux_event(). When the 2182 * group in torn down, the aux_output events loose their 2183 * link to the aux_event and can't schedule any more. 2184 */ 2185 event->aux_event = group_leader; 2186 2187 return 1; 2188 } 2189 2190 static inline struct list_head *get_event_list(struct perf_event *event) 2191 { 2192 struct perf_event_context *ctx = event->ctx; 2193 return event->attr.pinned ? &ctx->pinned_active : &ctx->flexible_active; 2194 } 2195 2196 /* 2197 * Events that have PERF_EV_CAP_SIBLING require being part of a group and 2198 * cannot exist on their own, schedule them out and move them into the ERROR 2199 * state. Also see _perf_event_enable(), it will not be able to recover 2200 * this ERROR state. 2201 */ 2202 static inline void perf_remove_sibling_event(struct perf_event *event) 2203 { 2204 struct perf_event_context *ctx = event->ctx; 2205 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 2206 2207 event_sched_out(event, cpuctx, ctx); 2208 perf_event_set_state(event, PERF_EVENT_STATE_ERROR); 2209 } 2210 2211 static void perf_group_detach(struct perf_event *event) 2212 { 2213 struct perf_event *leader = event->group_leader; 2214 struct perf_event *sibling, *tmp; 2215 struct perf_event_context *ctx = event->ctx; 2216 2217 lockdep_assert_held(&ctx->lock); 2218 2219 /* 2220 * We can have double detach due to exit/hot-unplug + close. 2221 */ 2222 if (!(event->attach_state & PERF_ATTACH_GROUP)) 2223 return; 2224 2225 event->attach_state &= ~PERF_ATTACH_GROUP; 2226 2227 perf_put_aux_event(event); 2228 2229 /* 2230 * If this is a sibling, remove it from its group. 2231 */ 2232 if (leader != event) { 2233 list_del_init(&event->sibling_list); 2234 event->group_leader->nr_siblings--; 2235 goto out; 2236 } 2237 2238 /* 2239 * If this was a group event with sibling events then 2240 * upgrade the siblings to singleton events by adding them 2241 * to whatever list we are on. 2242 */ 2243 list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) { 2244 2245 if (sibling->event_caps & PERF_EV_CAP_SIBLING) 2246 perf_remove_sibling_event(sibling); 2247 2248 sibling->group_leader = sibling; 2249 list_del_init(&sibling->sibling_list); 2250 2251 /* Inherit group flags from the previous leader */ 2252 sibling->group_caps = event->group_caps; 2253 2254 if (!RB_EMPTY_NODE(&event->group_node)) { 2255 add_event_to_groups(sibling, event->ctx); 2256 2257 if (sibling->state == PERF_EVENT_STATE_ACTIVE) 2258 list_add_tail(&sibling->active_list, get_event_list(sibling)); 2259 } 2260 2261 WARN_ON_ONCE(sibling->ctx != event->ctx); 2262 } 2263 2264 out: 2265 for_each_sibling_event(tmp, leader) 2266 perf_event__header_size(tmp); 2267 2268 perf_event__header_size(leader); 2269 } 2270 2271 static void sync_child_event(struct perf_event *child_event); 2272 2273 static void perf_child_detach(struct perf_event *event) 2274 { 2275 struct perf_event *parent_event = event->parent; 2276 2277 if (!(event->attach_state & PERF_ATTACH_CHILD)) 2278 return; 2279 2280 event->attach_state &= ~PERF_ATTACH_CHILD; 2281 2282 if (WARN_ON_ONCE(!parent_event)) 2283 return; 2284 2285 lockdep_assert_held(&parent_event->child_mutex); 2286 2287 sync_child_event(event); 2288 list_del_init(&event->child_list); 2289 } 2290 2291 static bool is_orphaned_event(struct perf_event *event) 2292 { 2293 return event->state == PERF_EVENT_STATE_DEAD; 2294 } 2295 2296 static inline int __pmu_filter_match(struct perf_event *event) 2297 { 2298 struct pmu *pmu = event->pmu; 2299 return pmu->filter_match ? pmu->filter_match(event) : 1; 2300 } 2301 2302 /* 2303 * Check whether we should attempt to schedule an event group based on 2304 * PMU-specific filtering. An event group can consist of HW and SW events, 2305 * potentially with a SW leader, so we must check all the filters, to 2306 * determine whether a group is schedulable: 2307 */ 2308 static inline int pmu_filter_match(struct perf_event *event) 2309 { 2310 struct perf_event *sibling; 2311 2312 if (!__pmu_filter_match(event)) 2313 return 0; 2314 2315 for_each_sibling_event(sibling, event) { 2316 if (!__pmu_filter_match(sibling)) 2317 return 0; 2318 } 2319 2320 return 1; 2321 } 2322 2323 static inline int 2324 event_filter_match(struct perf_event *event) 2325 { 2326 return (event->cpu == -1 || event->cpu == smp_processor_id()) && 2327 perf_cgroup_match(event) && pmu_filter_match(event); 2328 } 2329 2330 static void 2331 event_sched_out(struct perf_event *event, 2332 struct perf_cpu_context *cpuctx, 2333 struct perf_event_context *ctx) 2334 { 2335 enum perf_event_state state = PERF_EVENT_STATE_INACTIVE; 2336 2337 WARN_ON_ONCE(event->ctx != ctx); 2338 lockdep_assert_held(&ctx->lock); 2339 2340 if (event->state != PERF_EVENT_STATE_ACTIVE) 2341 return; 2342 2343 /* 2344 * Asymmetry; we only schedule events _IN_ through ctx_sched_in(), but 2345 * we can schedule events _OUT_ individually through things like 2346 * __perf_remove_from_context(). 2347 */ 2348 list_del_init(&event->active_list); 2349 2350 perf_pmu_disable(event->pmu); 2351 2352 event->pmu->del(event, 0); 2353 event->oncpu = -1; 2354 2355 if (READ_ONCE(event->pending_disable) >= 0) { 2356 WRITE_ONCE(event->pending_disable, -1); 2357 perf_cgroup_event_disable(event, ctx); 2358 state = PERF_EVENT_STATE_OFF; 2359 } 2360 perf_event_set_state(event, state); 2361 2362 if (!is_software_event(event)) 2363 cpuctx->active_oncpu--; 2364 if (!--ctx->nr_active) 2365 perf_event_ctx_deactivate(ctx); 2366 if (event->attr.freq && event->attr.sample_freq) 2367 ctx->nr_freq--; 2368 if (event->attr.exclusive || !cpuctx->active_oncpu) 2369 cpuctx->exclusive = 0; 2370 2371 perf_pmu_enable(event->pmu); 2372 } 2373 2374 static void 2375 group_sched_out(struct perf_event *group_event, 2376 struct perf_cpu_context *cpuctx, 2377 struct perf_event_context *ctx) 2378 { 2379 struct perf_event *event; 2380 2381 if (group_event->state != PERF_EVENT_STATE_ACTIVE) 2382 return; 2383 2384 perf_pmu_disable(ctx->pmu); 2385 2386 event_sched_out(group_event, cpuctx, ctx); 2387 2388 /* 2389 * Schedule out siblings (if any): 2390 */ 2391 for_each_sibling_event(event, group_event) 2392 event_sched_out(event, cpuctx, ctx); 2393 2394 perf_pmu_enable(ctx->pmu); 2395 } 2396 2397 #define DETACH_GROUP 0x01UL 2398 #define DETACH_CHILD 0x02UL 2399 2400 /* 2401 * Cross CPU call to remove a performance event 2402 * 2403 * We disable the event on the hardware level first. After that we 2404 * remove it from the context list. 2405 */ 2406 static void 2407 __perf_remove_from_context(struct perf_event *event, 2408 struct perf_cpu_context *cpuctx, 2409 struct perf_event_context *ctx, 2410 void *info) 2411 { 2412 unsigned long flags = (unsigned long)info; 2413 2414 if (ctx->is_active & EVENT_TIME) { 2415 update_context_time(ctx); 2416 update_cgrp_time_from_cpuctx(cpuctx, false); 2417 } 2418 2419 event_sched_out(event, cpuctx, ctx); 2420 if (flags & DETACH_GROUP) 2421 perf_group_detach(event); 2422 if (flags & DETACH_CHILD) 2423 perf_child_detach(event); 2424 list_del_event(event, ctx); 2425 2426 if (!ctx->nr_events && ctx->is_active) { 2427 if (ctx == &cpuctx->ctx) 2428 update_cgrp_time_from_cpuctx(cpuctx, true); 2429 2430 ctx->is_active = 0; 2431 ctx->rotate_necessary = 0; 2432 if (ctx->task) { 2433 WARN_ON_ONCE(cpuctx->task_ctx != ctx); 2434 cpuctx->task_ctx = NULL; 2435 } 2436 } 2437 } 2438 2439 /* 2440 * Remove the event from a task's (or a CPU's) list of events. 2441 * 2442 * If event->ctx is a cloned context, callers must make sure that 2443 * every task struct that event->ctx->task could possibly point to 2444 * remains valid. This is OK when called from perf_release since 2445 * that only calls us on the top-level context, which can't be a clone. 2446 * When called from perf_event_exit_task, it's OK because the 2447 * context has been detached from its task. 2448 */ 2449 static void perf_remove_from_context(struct perf_event *event, unsigned long flags) 2450 { 2451 struct perf_event_context *ctx = event->ctx; 2452 2453 lockdep_assert_held(&ctx->mutex); 2454 2455 /* 2456 * Because of perf_event_exit_task(), perf_remove_from_context() ought 2457 * to work in the face of TASK_TOMBSTONE, unlike every other 2458 * event_function_call() user. 2459 */ 2460 raw_spin_lock_irq(&ctx->lock); 2461 /* 2462 * Cgroup events are per-cpu events, and must IPI because of 2463 * cgrp_cpuctx_list. 2464 */ 2465 if (!ctx->is_active && !is_cgroup_event(event)) { 2466 __perf_remove_from_context(event, __get_cpu_context(ctx), 2467 ctx, (void *)flags); 2468 raw_spin_unlock_irq(&ctx->lock); 2469 return; 2470 } 2471 raw_spin_unlock_irq(&ctx->lock); 2472 2473 event_function_call(event, __perf_remove_from_context, (void *)flags); 2474 } 2475 2476 /* 2477 * Cross CPU call to disable a performance event 2478 */ 2479 static void __perf_event_disable(struct perf_event *event, 2480 struct perf_cpu_context *cpuctx, 2481 struct perf_event_context *ctx, 2482 void *info) 2483 { 2484 if (event->state < PERF_EVENT_STATE_INACTIVE) 2485 return; 2486 2487 if (ctx->is_active & EVENT_TIME) { 2488 update_context_time(ctx); 2489 update_cgrp_time_from_event(event); 2490 } 2491 2492 if (event == event->group_leader) 2493 group_sched_out(event, cpuctx, ctx); 2494 else 2495 event_sched_out(event, cpuctx, ctx); 2496 2497 perf_event_set_state(event, PERF_EVENT_STATE_OFF); 2498 perf_cgroup_event_disable(event, ctx); 2499 } 2500 2501 /* 2502 * Disable an event. 2503 * 2504 * If event->ctx is a cloned context, callers must make sure that 2505 * every task struct that event->ctx->task could possibly point to 2506 * remains valid. This condition is satisfied when called through 2507 * perf_event_for_each_child or perf_event_for_each because they 2508 * hold the top-level event's child_mutex, so any descendant that 2509 * goes to exit will block in perf_event_exit_event(). 2510 * 2511 * When called from perf_pending_event it's OK because event->ctx 2512 * is the current context on this CPU and preemption is disabled, 2513 * hence we can't get into perf_event_task_sched_out for this context. 2514 */ 2515 static void _perf_event_disable(struct perf_event *event) 2516 { 2517 struct perf_event_context *ctx = event->ctx; 2518 2519 raw_spin_lock_irq(&ctx->lock); 2520 if (event->state <= PERF_EVENT_STATE_OFF) { 2521 raw_spin_unlock_irq(&ctx->lock); 2522 return; 2523 } 2524 raw_spin_unlock_irq(&ctx->lock); 2525 2526 event_function_call(event, __perf_event_disable, NULL); 2527 } 2528 2529 void perf_event_disable_local(struct perf_event *event) 2530 { 2531 event_function_local(event, __perf_event_disable, NULL); 2532 } 2533 2534 /* 2535 * Strictly speaking kernel users cannot create groups and therefore this 2536 * interface does not need the perf_event_ctx_lock() magic. 2537 */ 2538 void perf_event_disable(struct perf_event *event) 2539 { 2540 struct perf_event_context *ctx; 2541 2542 ctx = perf_event_ctx_lock(event); 2543 _perf_event_disable(event); 2544 perf_event_ctx_unlock(event, ctx); 2545 } 2546 EXPORT_SYMBOL_GPL(perf_event_disable); 2547 2548 void perf_event_disable_inatomic(struct perf_event *event) 2549 { 2550 WRITE_ONCE(event->pending_disable, smp_processor_id()); 2551 /* can fail, see perf_pending_event_disable() */ 2552 irq_work_queue(&event->pending); 2553 } 2554 2555 #define MAX_INTERRUPTS (~0ULL) 2556 2557 static void perf_log_throttle(struct perf_event *event, int enable); 2558 static void perf_log_itrace_start(struct perf_event *event); 2559 2560 static int 2561 event_sched_in(struct perf_event *event, 2562 struct perf_cpu_context *cpuctx, 2563 struct perf_event_context *ctx) 2564 { 2565 int ret = 0; 2566 2567 WARN_ON_ONCE(event->ctx != ctx); 2568 2569 lockdep_assert_held(&ctx->lock); 2570 2571 if (event->state <= PERF_EVENT_STATE_OFF) 2572 return 0; 2573 2574 WRITE_ONCE(event->oncpu, smp_processor_id()); 2575 /* 2576 * Order event::oncpu write to happen before the ACTIVE state is 2577 * visible. This allows perf_event_{stop,read}() to observe the correct 2578 * ->oncpu if it sees ACTIVE. 2579 */ 2580 smp_wmb(); 2581 perf_event_set_state(event, PERF_EVENT_STATE_ACTIVE); 2582 2583 /* 2584 * Unthrottle events, since we scheduled we might have missed several 2585 * ticks already, also for a heavily scheduling task there is little 2586 * guarantee it'll get a tick in a timely manner. 2587 */ 2588 if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) { 2589 perf_log_throttle(event, 1); 2590 event->hw.interrupts = 0; 2591 } 2592 2593 perf_pmu_disable(event->pmu); 2594 2595 perf_log_itrace_start(event); 2596 2597 if (event->pmu->add(event, PERF_EF_START)) { 2598 perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE); 2599 event->oncpu = -1; 2600 ret = -EAGAIN; 2601 goto out; 2602 } 2603 2604 if (!is_software_event(event)) 2605 cpuctx->active_oncpu++; 2606 if (!ctx->nr_active++) 2607 perf_event_ctx_activate(ctx); 2608 if (event->attr.freq && event->attr.sample_freq) 2609 ctx->nr_freq++; 2610 2611 if (event->attr.exclusive) 2612 cpuctx->exclusive = 1; 2613 2614 out: 2615 perf_pmu_enable(event->pmu); 2616 2617 return ret; 2618 } 2619 2620 static int 2621 group_sched_in(struct perf_event *group_event, 2622 struct perf_cpu_context *cpuctx, 2623 struct perf_event_context *ctx) 2624 { 2625 struct perf_event *event, *partial_group = NULL; 2626 struct pmu *pmu = ctx->pmu; 2627 2628 if (group_event->state == PERF_EVENT_STATE_OFF) 2629 return 0; 2630 2631 pmu->start_txn(pmu, PERF_PMU_TXN_ADD); 2632 2633 if (event_sched_in(group_event, cpuctx, ctx)) 2634 goto error; 2635 2636 /* 2637 * Schedule in siblings as one group (if any): 2638 */ 2639 for_each_sibling_event(event, group_event) { 2640 if (event_sched_in(event, cpuctx, ctx)) { 2641 partial_group = event; 2642 goto group_error; 2643 } 2644 } 2645 2646 if (!pmu->commit_txn(pmu)) 2647 return 0; 2648 2649 group_error: 2650 /* 2651 * Groups can be scheduled in as one unit only, so undo any 2652 * partial group before returning: 2653 * The events up to the failed event are scheduled out normally. 2654 */ 2655 for_each_sibling_event(event, group_event) { 2656 if (event == partial_group) 2657 break; 2658 2659 event_sched_out(event, cpuctx, ctx); 2660 } 2661 event_sched_out(group_event, cpuctx, ctx); 2662 2663 error: 2664 pmu->cancel_txn(pmu); 2665 return -EAGAIN; 2666 } 2667 2668 /* 2669 * Work out whether we can put this event group on the CPU now. 2670 */ 2671 static int group_can_go_on(struct perf_event *event, 2672 struct perf_cpu_context *cpuctx, 2673 int can_add_hw) 2674 { 2675 /* 2676 * Groups consisting entirely of software events can always go on. 2677 */ 2678 if (event->group_caps & PERF_EV_CAP_SOFTWARE) 2679 return 1; 2680 /* 2681 * If an exclusive group is already on, no other hardware 2682 * events can go on. 2683 */ 2684 if (cpuctx->exclusive) 2685 return 0; 2686 /* 2687 * If this group is exclusive and there are already 2688 * events on the CPU, it can't go on. 2689 */ 2690 if (event->attr.exclusive && !list_empty(get_event_list(event))) 2691 return 0; 2692 /* 2693 * Otherwise, try to add it if all previous groups were able 2694 * to go on. 2695 */ 2696 return can_add_hw; 2697 } 2698 2699 static void add_event_to_ctx(struct perf_event *event, 2700 struct perf_event_context *ctx) 2701 { 2702 list_add_event(event, ctx); 2703 perf_group_attach(event); 2704 } 2705 2706 static void ctx_sched_out(struct perf_event_context *ctx, 2707 struct perf_cpu_context *cpuctx, 2708 enum event_type_t event_type); 2709 static void 2710 ctx_sched_in(struct perf_event_context *ctx, 2711 struct perf_cpu_context *cpuctx, 2712 enum event_type_t event_type, 2713 struct task_struct *task); 2714 2715 static void task_ctx_sched_out(struct perf_cpu_context *cpuctx, 2716 struct perf_event_context *ctx, 2717 enum event_type_t event_type) 2718 { 2719 if (!cpuctx->task_ctx) 2720 return; 2721 2722 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx)) 2723 return; 2724 2725 ctx_sched_out(ctx, cpuctx, event_type); 2726 } 2727 2728 static void perf_event_sched_in(struct perf_cpu_context *cpuctx, 2729 struct perf_event_context *ctx, 2730 struct task_struct *task) 2731 { 2732 cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task); 2733 if (ctx) 2734 ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task); 2735 cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task); 2736 if (ctx) 2737 ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task); 2738 } 2739 2740 /* 2741 * We want to maintain the following priority of scheduling: 2742 * - CPU pinned (EVENT_CPU | EVENT_PINNED) 2743 * - task pinned (EVENT_PINNED) 2744 * - CPU flexible (EVENT_CPU | EVENT_FLEXIBLE) 2745 * - task flexible (EVENT_FLEXIBLE). 2746 * 2747 * In order to avoid unscheduling and scheduling back in everything every 2748 * time an event is added, only do it for the groups of equal priority and 2749 * below. 2750 * 2751 * This can be called after a batch operation on task events, in which case 2752 * event_type is a bit mask of the types of events involved. For CPU events, 2753 * event_type is only either EVENT_PINNED or EVENT_FLEXIBLE. 2754 */ 2755 static void ctx_resched(struct perf_cpu_context *cpuctx, 2756 struct perf_event_context *task_ctx, 2757 enum event_type_t event_type) 2758 { 2759 enum event_type_t ctx_event_type; 2760 bool cpu_event = !!(event_type & EVENT_CPU); 2761 2762 /* 2763 * If pinned groups are involved, flexible groups also need to be 2764 * scheduled out. 2765 */ 2766 if (event_type & EVENT_PINNED) 2767 event_type |= EVENT_FLEXIBLE; 2768 2769 ctx_event_type = event_type & EVENT_ALL; 2770 2771 perf_pmu_disable(cpuctx->ctx.pmu); 2772 if (task_ctx) 2773 task_ctx_sched_out(cpuctx, task_ctx, event_type); 2774 2775 /* 2776 * Decide which cpu ctx groups to schedule out based on the types 2777 * of events that caused rescheduling: 2778 * - EVENT_CPU: schedule out corresponding groups; 2779 * - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups; 2780 * - otherwise, do nothing more. 2781 */ 2782 if (cpu_event) 2783 cpu_ctx_sched_out(cpuctx, ctx_event_type); 2784 else if (ctx_event_type & EVENT_PINNED) 2785 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE); 2786 2787 perf_event_sched_in(cpuctx, task_ctx, current); 2788 perf_pmu_enable(cpuctx->ctx.pmu); 2789 } 2790 2791 void perf_pmu_resched(struct pmu *pmu) 2792 { 2793 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 2794 struct perf_event_context *task_ctx = cpuctx->task_ctx; 2795 2796 perf_ctx_lock(cpuctx, task_ctx); 2797 ctx_resched(cpuctx, task_ctx, EVENT_ALL|EVENT_CPU); 2798 perf_ctx_unlock(cpuctx, task_ctx); 2799 } 2800 2801 /* 2802 * Cross CPU call to install and enable a performance event 2803 * 2804 * Very similar to remote_function() + event_function() but cannot assume that 2805 * things like ctx->is_active and cpuctx->task_ctx are set. 2806 */ 2807 static int __perf_install_in_context(void *info) 2808 { 2809 struct perf_event *event = info; 2810 struct perf_event_context *ctx = event->ctx; 2811 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 2812 struct perf_event_context *task_ctx = cpuctx->task_ctx; 2813 bool reprogram = true; 2814 int ret = 0; 2815 2816 raw_spin_lock(&cpuctx->ctx.lock); 2817 if (ctx->task) { 2818 raw_spin_lock(&ctx->lock); 2819 task_ctx = ctx; 2820 2821 reprogram = (ctx->task == current); 2822 2823 /* 2824 * If the task is running, it must be running on this CPU, 2825 * otherwise we cannot reprogram things. 2826 * 2827 * If its not running, we don't care, ctx->lock will 2828 * serialize against it becoming runnable. 2829 */ 2830 if (task_curr(ctx->task) && !reprogram) { 2831 ret = -ESRCH; 2832 goto unlock; 2833 } 2834 2835 WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx); 2836 } else if (task_ctx) { 2837 raw_spin_lock(&task_ctx->lock); 2838 } 2839 2840 #ifdef CONFIG_CGROUP_PERF 2841 if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) { 2842 /* 2843 * If the current cgroup doesn't match the event's 2844 * cgroup, we should not try to schedule it. 2845 */ 2846 struct perf_cgroup *cgrp = perf_cgroup_from_task(current, ctx); 2847 reprogram = cgroup_is_descendant(cgrp->css.cgroup, 2848 event->cgrp->css.cgroup); 2849 } 2850 #endif 2851 2852 if (reprogram) { 2853 ctx_sched_out(ctx, cpuctx, EVENT_TIME); 2854 add_event_to_ctx(event, ctx); 2855 ctx_resched(cpuctx, task_ctx, get_event_type(event)); 2856 } else { 2857 add_event_to_ctx(event, ctx); 2858 } 2859 2860 unlock: 2861 perf_ctx_unlock(cpuctx, task_ctx); 2862 2863 return ret; 2864 } 2865 2866 static bool exclusive_event_installable(struct perf_event *event, 2867 struct perf_event_context *ctx); 2868 2869 /* 2870 * Attach a performance event to a context. 2871 * 2872 * Very similar to event_function_call, see comment there. 2873 */ 2874 static void 2875 perf_install_in_context(struct perf_event_context *ctx, 2876 struct perf_event *event, 2877 int cpu) 2878 { 2879 struct task_struct *task = READ_ONCE(ctx->task); 2880 2881 lockdep_assert_held(&ctx->mutex); 2882 2883 WARN_ON_ONCE(!exclusive_event_installable(event, ctx)); 2884 2885 if (event->cpu != -1) 2886 event->cpu = cpu; 2887 2888 /* 2889 * Ensures that if we can observe event->ctx, both the event and ctx 2890 * will be 'complete'. See perf_iterate_sb_cpu(). 2891 */ 2892 smp_store_release(&event->ctx, ctx); 2893 2894 /* 2895 * perf_event_attr::disabled events will not run and can be initialized 2896 * without IPI. Except when this is the first event for the context, in 2897 * that case we need the magic of the IPI to set ctx->is_active. 2898 * Similarly, cgroup events for the context also needs the IPI to 2899 * manipulate the cgrp_cpuctx_list. 2900 * 2901 * The IOC_ENABLE that is sure to follow the creation of a disabled 2902 * event will issue the IPI and reprogram the hardware. 2903 */ 2904 if (__perf_effective_state(event) == PERF_EVENT_STATE_OFF && 2905 ctx->nr_events && !is_cgroup_event(event)) { 2906 raw_spin_lock_irq(&ctx->lock); 2907 if (ctx->task == TASK_TOMBSTONE) { 2908 raw_spin_unlock_irq(&ctx->lock); 2909 return; 2910 } 2911 add_event_to_ctx(event, ctx); 2912 raw_spin_unlock_irq(&ctx->lock); 2913 return; 2914 } 2915 2916 if (!task) { 2917 cpu_function_call(cpu, __perf_install_in_context, event); 2918 return; 2919 } 2920 2921 /* 2922 * Should not happen, we validate the ctx is still alive before calling. 2923 */ 2924 if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) 2925 return; 2926 2927 /* 2928 * Installing events is tricky because we cannot rely on ctx->is_active 2929 * to be set in case this is the nr_events 0 -> 1 transition. 2930 * 2931 * Instead we use task_curr(), which tells us if the task is running. 2932 * However, since we use task_curr() outside of rq::lock, we can race 2933 * against the actual state. This means the result can be wrong. 2934 * 2935 * If we get a false positive, we retry, this is harmless. 2936 * 2937 * If we get a false negative, things are complicated. If we are after 2938 * perf_event_context_sched_in() ctx::lock will serialize us, and the 2939 * value must be correct. If we're before, it doesn't matter since 2940 * perf_event_context_sched_in() will program the counter. 2941 * 2942 * However, this hinges on the remote context switch having observed 2943 * our task->perf_event_ctxp[] store, such that it will in fact take 2944 * ctx::lock in perf_event_context_sched_in(). 2945 * 2946 * We do this by task_function_call(), if the IPI fails to hit the task 2947 * we know any future context switch of task must see the 2948 * perf_event_ctpx[] store. 2949 */ 2950 2951 /* 2952 * This smp_mb() orders the task->perf_event_ctxp[] store with the 2953 * task_cpu() load, such that if the IPI then does not find the task 2954 * running, a future context switch of that task must observe the 2955 * store. 2956 */ 2957 smp_mb(); 2958 again: 2959 if (!task_function_call(task, __perf_install_in_context, event)) 2960 return; 2961 2962 raw_spin_lock_irq(&ctx->lock); 2963 task = ctx->task; 2964 if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) { 2965 /* 2966 * Cannot happen because we already checked above (which also 2967 * cannot happen), and we hold ctx->mutex, which serializes us 2968 * against perf_event_exit_task_context(). 2969 */ 2970 raw_spin_unlock_irq(&ctx->lock); 2971 return; 2972 } 2973 /* 2974 * If the task is not running, ctx->lock will avoid it becoming so, 2975 * thus we can safely install the event. 2976 */ 2977 if (task_curr(task)) { 2978 raw_spin_unlock_irq(&ctx->lock); 2979 goto again; 2980 } 2981 add_event_to_ctx(event, ctx); 2982 raw_spin_unlock_irq(&ctx->lock); 2983 } 2984 2985 /* 2986 * Cross CPU call to enable a performance event 2987 */ 2988 static void __perf_event_enable(struct perf_event *event, 2989 struct perf_cpu_context *cpuctx, 2990 struct perf_event_context *ctx, 2991 void *info) 2992 { 2993 struct perf_event *leader = event->group_leader; 2994 struct perf_event_context *task_ctx; 2995 2996 if (event->state >= PERF_EVENT_STATE_INACTIVE || 2997 event->state <= PERF_EVENT_STATE_ERROR) 2998 return; 2999 3000 if (ctx->is_active) 3001 ctx_sched_out(ctx, cpuctx, EVENT_TIME); 3002 3003 perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE); 3004 perf_cgroup_event_enable(event, ctx); 3005 3006 if (!ctx->is_active) 3007 return; 3008 3009 if (!event_filter_match(event)) { 3010 ctx_sched_in(ctx, cpuctx, EVENT_TIME, current); 3011 return; 3012 } 3013 3014 /* 3015 * If the event is in a group and isn't the group leader, 3016 * then don't put it on unless the group is on. 3017 */ 3018 if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) { 3019 ctx_sched_in(ctx, cpuctx, EVENT_TIME, current); 3020 return; 3021 } 3022 3023 task_ctx = cpuctx->task_ctx; 3024 if (ctx->task) 3025 WARN_ON_ONCE(task_ctx != ctx); 3026 3027 ctx_resched(cpuctx, task_ctx, get_event_type(event)); 3028 } 3029 3030 /* 3031 * Enable an event. 3032 * 3033 * If event->ctx is a cloned context, callers must make sure that 3034 * every task struct that event->ctx->task could possibly point to 3035 * remains valid. This condition is satisfied when called through 3036 * perf_event_for_each_child or perf_event_for_each as described 3037 * for perf_event_disable. 3038 */ 3039 static void _perf_event_enable(struct perf_event *event) 3040 { 3041 struct perf_event_context *ctx = event->ctx; 3042 3043 raw_spin_lock_irq(&ctx->lock); 3044 if (event->state >= PERF_EVENT_STATE_INACTIVE || 3045 event->state < PERF_EVENT_STATE_ERROR) { 3046 out: 3047 raw_spin_unlock_irq(&ctx->lock); 3048 return; 3049 } 3050 3051 /* 3052 * If the event is in error state, clear that first. 3053 * 3054 * That way, if we see the event in error state below, we know that it 3055 * has gone back into error state, as distinct from the task having 3056 * been scheduled away before the cross-call arrived. 3057 */ 3058 if (event->state == PERF_EVENT_STATE_ERROR) { 3059 /* 3060 * Detached SIBLING events cannot leave ERROR state. 3061 */ 3062 if (event->event_caps & PERF_EV_CAP_SIBLING && 3063 event->group_leader == event) 3064 goto out; 3065 3066 event->state = PERF_EVENT_STATE_OFF; 3067 } 3068 raw_spin_unlock_irq(&ctx->lock); 3069 3070 event_function_call(event, __perf_event_enable, NULL); 3071 } 3072 3073 /* 3074 * See perf_event_disable(); 3075 */ 3076 void perf_event_enable(struct perf_event *event) 3077 { 3078 struct perf_event_context *ctx; 3079 3080 ctx = perf_event_ctx_lock(event); 3081 _perf_event_enable(event); 3082 perf_event_ctx_unlock(event, ctx); 3083 } 3084 EXPORT_SYMBOL_GPL(perf_event_enable); 3085 3086 struct stop_event_data { 3087 struct perf_event *event; 3088 unsigned int restart; 3089 }; 3090 3091 static int __perf_event_stop(void *info) 3092 { 3093 struct stop_event_data *sd = info; 3094 struct perf_event *event = sd->event; 3095 3096 /* if it's already INACTIVE, do nothing */ 3097 if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE) 3098 return 0; 3099 3100 /* matches smp_wmb() in event_sched_in() */ 3101 smp_rmb(); 3102 3103 /* 3104 * There is a window with interrupts enabled before we get here, 3105 * so we need to check again lest we try to stop another CPU's event. 3106 */ 3107 if (READ_ONCE(event->oncpu) != smp_processor_id()) 3108 return -EAGAIN; 3109 3110 event->pmu->stop(event, PERF_EF_UPDATE); 3111 3112 /* 3113 * May race with the actual stop (through perf_pmu_output_stop()), 3114 * but it is only used for events with AUX ring buffer, and such 3115 * events will refuse to restart because of rb::aux_mmap_count==0, 3116 * see comments in perf_aux_output_begin(). 3117 * 3118 * Since this is happening on an event-local CPU, no trace is lost 3119 * while restarting. 3120 */ 3121 if (sd->restart) 3122 event->pmu->start(event, 0); 3123 3124 return 0; 3125 } 3126 3127 static int perf_event_stop(struct perf_event *event, int restart) 3128 { 3129 struct stop_event_data sd = { 3130 .event = event, 3131 .restart = restart, 3132 }; 3133 int ret = 0; 3134 3135 do { 3136 if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE) 3137 return 0; 3138 3139 /* matches smp_wmb() in event_sched_in() */ 3140 smp_rmb(); 3141 3142 /* 3143 * We only want to restart ACTIVE events, so if the event goes 3144 * inactive here (event->oncpu==-1), there's nothing more to do; 3145 * fall through with ret==-ENXIO. 3146 */ 3147 ret = cpu_function_call(READ_ONCE(event->oncpu), 3148 __perf_event_stop, &sd); 3149 } while (ret == -EAGAIN); 3150 3151 return ret; 3152 } 3153 3154 /* 3155 * In order to contain the amount of racy and tricky in the address filter 3156 * configuration management, it is a two part process: 3157 * 3158 * (p1) when userspace mappings change as a result of (1) or (2) or (3) below, 3159 * we update the addresses of corresponding vmas in 3160 * event::addr_filter_ranges array and bump the event::addr_filters_gen; 3161 * (p2) when an event is scheduled in (pmu::add), it calls 3162 * perf_event_addr_filters_sync() which calls pmu::addr_filters_sync() 3163 * if the generation has changed since the previous call. 3164 * 3165 * If (p1) happens while the event is active, we restart it to force (p2). 3166 * 3167 * (1) perf_addr_filters_apply(): adjusting filters' offsets based on 3168 * pre-existing mappings, called once when new filters arrive via SET_FILTER 3169 * ioctl; 3170 * (2) perf_addr_filters_adjust(): adjusting filters' offsets based on newly 3171 * registered mapping, called for every new mmap(), with mm::mmap_lock down 3172 * for reading; 3173 * (3) perf_event_addr_filters_exec(): clearing filters' offsets in the process 3174 * of exec. 3175 */ 3176 void perf_event_addr_filters_sync(struct perf_event *event) 3177 { 3178 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 3179 3180 if (!has_addr_filter(event)) 3181 return; 3182 3183 raw_spin_lock(&ifh->lock); 3184 if (event->addr_filters_gen != event->hw.addr_filters_gen) { 3185 event->pmu->addr_filters_sync(event); 3186 event->hw.addr_filters_gen = event->addr_filters_gen; 3187 } 3188 raw_spin_unlock(&ifh->lock); 3189 } 3190 EXPORT_SYMBOL_GPL(perf_event_addr_filters_sync); 3191 3192 static int _perf_event_refresh(struct perf_event *event, int refresh) 3193 { 3194 /* 3195 * not supported on inherited events 3196 */ 3197 if (event->attr.inherit || !is_sampling_event(event)) 3198 return -EINVAL; 3199 3200 atomic_add(refresh, &event->event_limit); 3201 _perf_event_enable(event); 3202 3203 return 0; 3204 } 3205 3206 /* 3207 * See perf_event_disable() 3208 */ 3209 int perf_event_refresh(struct perf_event *event, int refresh) 3210 { 3211 struct perf_event_context *ctx; 3212 int ret; 3213 3214 ctx = perf_event_ctx_lock(event); 3215 ret = _perf_event_refresh(event, refresh); 3216 perf_event_ctx_unlock(event, ctx); 3217 3218 return ret; 3219 } 3220 EXPORT_SYMBOL_GPL(perf_event_refresh); 3221 3222 static int perf_event_modify_breakpoint(struct perf_event *bp, 3223 struct perf_event_attr *attr) 3224 { 3225 int err; 3226 3227 _perf_event_disable(bp); 3228 3229 err = modify_user_hw_breakpoint_check(bp, attr, true); 3230 3231 if (!bp->attr.disabled) 3232 _perf_event_enable(bp); 3233 3234 return err; 3235 } 3236 3237 /* 3238 * Copy event-type-independent attributes that may be modified. 3239 */ 3240 static void perf_event_modify_copy_attr(struct perf_event_attr *to, 3241 const struct perf_event_attr *from) 3242 { 3243 to->sig_data = from->sig_data; 3244 } 3245 3246 static int perf_event_modify_attr(struct perf_event *event, 3247 struct perf_event_attr *attr) 3248 { 3249 int (*func)(struct perf_event *, struct perf_event_attr *); 3250 struct perf_event *child; 3251 int err; 3252 3253 if (event->attr.type != attr->type) 3254 return -EINVAL; 3255 3256 switch (event->attr.type) { 3257 case PERF_TYPE_BREAKPOINT: 3258 func = perf_event_modify_breakpoint; 3259 break; 3260 default: 3261 /* Place holder for future additions. */ 3262 return -EOPNOTSUPP; 3263 } 3264 3265 WARN_ON_ONCE(event->ctx->parent_ctx); 3266 3267 mutex_lock(&event->child_mutex); 3268 /* 3269 * Event-type-independent attributes must be copied before event-type 3270 * modification, which will validate that final attributes match the 3271 * source attributes after all relevant attributes have been copied. 3272 */ 3273 perf_event_modify_copy_attr(&event->attr, attr); 3274 err = func(event, attr); 3275 if (err) 3276 goto out; 3277 list_for_each_entry(child, &event->child_list, child_list) { 3278 perf_event_modify_copy_attr(&child->attr, attr); 3279 err = func(child, attr); 3280 if (err) 3281 goto out; 3282 } 3283 out: 3284 mutex_unlock(&event->child_mutex); 3285 return err; 3286 } 3287 3288 static void ctx_sched_out(struct perf_event_context *ctx, 3289 struct perf_cpu_context *cpuctx, 3290 enum event_type_t event_type) 3291 { 3292 struct perf_event *event, *tmp; 3293 int is_active = ctx->is_active; 3294 3295 lockdep_assert_held(&ctx->lock); 3296 3297 if (likely(!ctx->nr_events)) { 3298 /* 3299 * See __perf_remove_from_context(). 3300 */ 3301 WARN_ON_ONCE(ctx->is_active); 3302 if (ctx->task) 3303 WARN_ON_ONCE(cpuctx->task_ctx); 3304 return; 3305 } 3306 3307 /* 3308 * Always update time if it was set; not only when it changes. 3309 * Otherwise we can 'forget' to update time for any but the last 3310 * context we sched out. For example: 3311 * 3312 * ctx_sched_out(.event_type = EVENT_FLEXIBLE) 3313 * ctx_sched_out(.event_type = EVENT_PINNED) 3314 * 3315 * would only update time for the pinned events. 3316 */ 3317 if (is_active & EVENT_TIME) { 3318 /* update (and stop) ctx time */ 3319 update_context_time(ctx); 3320 update_cgrp_time_from_cpuctx(cpuctx, ctx == &cpuctx->ctx); 3321 /* 3322 * CPU-release for the below ->is_active store, 3323 * see __load_acquire() in perf_event_time_now() 3324 */ 3325 barrier(); 3326 } 3327 3328 ctx->is_active &= ~event_type; 3329 if (!(ctx->is_active & EVENT_ALL)) 3330 ctx->is_active = 0; 3331 3332 if (ctx->task) { 3333 WARN_ON_ONCE(cpuctx->task_ctx != ctx); 3334 if (!ctx->is_active) 3335 cpuctx->task_ctx = NULL; 3336 } 3337 3338 is_active ^= ctx->is_active; /* changed bits */ 3339 3340 if (!ctx->nr_active || !(is_active & EVENT_ALL)) 3341 return; 3342 3343 perf_pmu_disable(ctx->pmu); 3344 if (is_active & EVENT_PINNED) { 3345 list_for_each_entry_safe(event, tmp, &ctx->pinned_active, active_list) 3346 group_sched_out(event, cpuctx, ctx); 3347 } 3348 3349 if (is_active & EVENT_FLEXIBLE) { 3350 list_for_each_entry_safe(event, tmp, &ctx->flexible_active, active_list) 3351 group_sched_out(event, cpuctx, ctx); 3352 3353 /* 3354 * Since we cleared EVENT_FLEXIBLE, also clear 3355 * rotate_necessary, is will be reset by 3356 * ctx_flexible_sched_in() when needed. 3357 */ 3358 ctx->rotate_necessary = 0; 3359 } 3360 perf_pmu_enable(ctx->pmu); 3361 } 3362 3363 /* 3364 * Test whether two contexts are equivalent, i.e. whether they have both been 3365 * cloned from the same version of the same context. 3366 * 3367 * Equivalence is measured using a generation number in the context that is 3368 * incremented on each modification to it; see unclone_ctx(), list_add_event() 3369 * and list_del_event(). 3370 */ 3371 static int context_equiv(struct perf_event_context *ctx1, 3372 struct perf_event_context *ctx2) 3373 { 3374 lockdep_assert_held(&ctx1->lock); 3375 lockdep_assert_held(&ctx2->lock); 3376 3377 /* Pinning disables the swap optimization */ 3378 if (ctx1->pin_count || ctx2->pin_count) 3379 return 0; 3380 3381 /* If ctx1 is the parent of ctx2 */ 3382 if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen) 3383 return 1; 3384 3385 /* If ctx2 is the parent of ctx1 */ 3386 if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation) 3387 return 1; 3388 3389 /* 3390 * If ctx1 and ctx2 have the same parent; we flatten the parent 3391 * hierarchy, see perf_event_init_context(). 3392 */ 3393 if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx && 3394 ctx1->parent_gen == ctx2->parent_gen) 3395 return 1; 3396 3397 /* Unmatched */ 3398 return 0; 3399 } 3400 3401 static void __perf_event_sync_stat(struct perf_event *event, 3402 struct perf_event *next_event) 3403 { 3404 u64 value; 3405 3406 if (!event->attr.inherit_stat) 3407 return; 3408 3409 /* 3410 * Update the event value, we cannot use perf_event_read() 3411 * because we're in the middle of a context switch and have IRQs 3412 * disabled, which upsets smp_call_function_single(), however 3413 * we know the event must be on the current CPU, therefore we 3414 * don't need to use it. 3415 */ 3416 if (event->state == PERF_EVENT_STATE_ACTIVE) 3417 event->pmu->read(event); 3418 3419 perf_event_update_time(event); 3420 3421 /* 3422 * In order to keep per-task stats reliable we need to flip the event 3423 * values when we flip the contexts. 3424 */ 3425 value = local64_read(&next_event->count); 3426 value = local64_xchg(&event->count, value); 3427 local64_set(&next_event->count, value); 3428 3429 swap(event->total_time_enabled, next_event->total_time_enabled); 3430 swap(event->total_time_running, next_event->total_time_running); 3431 3432 /* 3433 * Since we swizzled the values, update the user visible data too. 3434 */ 3435 perf_event_update_userpage(event); 3436 perf_event_update_userpage(next_event); 3437 } 3438 3439 static void perf_event_sync_stat(struct perf_event_context *ctx, 3440 struct perf_event_context *next_ctx) 3441 { 3442 struct perf_event *event, *next_event; 3443 3444 if (!ctx->nr_stat) 3445 return; 3446 3447 update_context_time(ctx); 3448 3449 event = list_first_entry(&ctx->event_list, 3450 struct perf_event, event_entry); 3451 3452 next_event = list_first_entry(&next_ctx->event_list, 3453 struct perf_event, event_entry); 3454 3455 while (&event->event_entry != &ctx->event_list && 3456 &next_event->event_entry != &next_ctx->event_list) { 3457 3458 __perf_event_sync_stat(event, next_event); 3459 3460 event = list_next_entry(event, event_entry); 3461 next_event = list_next_entry(next_event, event_entry); 3462 } 3463 } 3464 3465 static void perf_event_context_sched_out(struct task_struct *task, int ctxn, 3466 struct task_struct *next) 3467 { 3468 struct perf_event_context *ctx = task->perf_event_ctxp[ctxn]; 3469 struct perf_event_context *next_ctx; 3470 struct perf_event_context *parent, *next_parent; 3471 struct perf_cpu_context *cpuctx; 3472 int do_switch = 1; 3473 struct pmu *pmu; 3474 3475 if (likely(!ctx)) 3476 return; 3477 3478 pmu = ctx->pmu; 3479 cpuctx = __get_cpu_context(ctx); 3480 if (!cpuctx->task_ctx) 3481 return; 3482 3483 rcu_read_lock(); 3484 next_ctx = next->perf_event_ctxp[ctxn]; 3485 if (!next_ctx) 3486 goto unlock; 3487 3488 parent = rcu_dereference(ctx->parent_ctx); 3489 next_parent = rcu_dereference(next_ctx->parent_ctx); 3490 3491 /* If neither context have a parent context; they cannot be clones. */ 3492 if (!parent && !next_parent) 3493 goto unlock; 3494 3495 if (next_parent == ctx || next_ctx == parent || next_parent == parent) { 3496 /* 3497 * Looks like the two contexts are clones, so we might be 3498 * able to optimize the context switch. We lock both 3499 * contexts and check that they are clones under the 3500 * lock (including re-checking that neither has been 3501 * uncloned in the meantime). It doesn't matter which 3502 * order we take the locks because no other cpu could 3503 * be trying to lock both of these tasks. 3504 */ 3505 raw_spin_lock(&ctx->lock); 3506 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); 3507 if (context_equiv(ctx, next_ctx)) { 3508 3509 WRITE_ONCE(ctx->task, next); 3510 WRITE_ONCE(next_ctx->task, task); 3511 3512 perf_pmu_disable(pmu); 3513 3514 if (cpuctx->sched_cb_usage && pmu->sched_task) 3515 pmu->sched_task(ctx, false); 3516 3517 /* 3518 * PMU specific parts of task perf context can require 3519 * additional synchronization. As an example of such 3520 * synchronization see implementation details of Intel 3521 * LBR call stack data profiling; 3522 */ 3523 if (pmu->swap_task_ctx) 3524 pmu->swap_task_ctx(ctx, next_ctx); 3525 else 3526 swap(ctx->task_ctx_data, next_ctx->task_ctx_data); 3527 3528 perf_pmu_enable(pmu); 3529 3530 /* 3531 * RCU_INIT_POINTER here is safe because we've not 3532 * modified the ctx and the above modification of 3533 * ctx->task and ctx->task_ctx_data are immaterial 3534 * since those values are always verified under 3535 * ctx->lock which we're now holding. 3536 */ 3537 RCU_INIT_POINTER(task->perf_event_ctxp[ctxn], next_ctx); 3538 RCU_INIT_POINTER(next->perf_event_ctxp[ctxn], ctx); 3539 3540 do_switch = 0; 3541 3542 perf_event_sync_stat(ctx, next_ctx); 3543 } 3544 raw_spin_unlock(&next_ctx->lock); 3545 raw_spin_unlock(&ctx->lock); 3546 } 3547 unlock: 3548 rcu_read_unlock(); 3549 3550 if (do_switch) { 3551 raw_spin_lock(&ctx->lock); 3552 perf_pmu_disable(pmu); 3553 3554 if (cpuctx->sched_cb_usage && pmu->sched_task) 3555 pmu->sched_task(ctx, false); 3556 task_ctx_sched_out(cpuctx, ctx, EVENT_ALL); 3557 3558 perf_pmu_enable(pmu); 3559 raw_spin_unlock(&ctx->lock); 3560 } 3561 } 3562 3563 static DEFINE_PER_CPU(struct list_head, sched_cb_list); 3564 3565 void perf_sched_cb_dec(struct pmu *pmu) 3566 { 3567 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 3568 3569 this_cpu_dec(perf_sched_cb_usages); 3570 3571 if (!--cpuctx->sched_cb_usage) 3572 list_del(&cpuctx->sched_cb_entry); 3573 } 3574 3575 3576 void perf_sched_cb_inc(struct pmu *pmu) 3577 { 3578 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 3579 3580 if (!cpuctx->sched_cb_usage++) 3581 list_add(&cpuctx->sched_cb_entry, this_cpu_ptr(&sched_cb_list)); 3582 3583 this_cpu_inc(perf_sched_cb_usages); 3584 } 3585 3586 /* 3587 * This function provides the context switch callback to the lower code 3588 * layer. It is invoked ONLY when the context switch callback is enabled. 3589 * 3590 * This callback is relevant even to per-cpu events; for example multi event 3591 * PEBS requires this to provide PID/TID information. This requires we flush 3592 * all queued PEBS records before we context switch to a new task. 3593 */ 3594 static void __perf_pmu_sched_task(struct perf_cpu_context *cpuctx, bool sched_in) 3595 { 3596 struct pmu *pmu; 3597 3598 pmu = cpuctx->ctx.pmu; /* software PMUs will not have sched_task */ 3599 3600 if (WARN_ON_ONCE(!pmu->sched_task)) 3601 return; 3602 3603 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 3604 perf_pmu_disable(pmu); 3605 3606 pmu->sched_task(cpuctx->task_ctx, sched_in); 3607 3608 perf_pmu_enable(pmu); 3609 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 3610 } 3611 3612 static void perf_pmu_sched_task(struct task_struct *prev, 3613 struct task_struct *next, 3614 bool sched_in) 3615 { 3616 struct perf_cpu_context *cpuctx; 3617 3618 if (prev == next) 3619 return; 3620 3621 list_for_each_entry(cpuctx, this_cpu_ptr(&sched_cb_list), sched_cb_entry) { 3622 /* will be handled in perf_event_context_sched_in/out */ 3623 if (cpuctx->task_ctx) 3624 continue; 3625 3626 __perf_pmu_sched_task(cpuctx, sched_in); 3627 } 3628 } 3629 3630 static void perf_event_switch(struct task_struct *task, 3631 struct task_struct *next_prev, bool sched_in); 3632 3633 #define for_each_task_context_nr(ctxn) \ 3634 for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++) 3635 3636 /* 3637 * Called from scheduler to remove the events of the current task, 3638 * with interrupts disabled. 3639 * 3640 * We stop each event and update the event value in event->count. 3641 * 3642 * This does not protect us against NMI, but disable() 3643 * sets the disabled bit in the control field of event _before_ 3644 * accessing the event control register. If a NMI hits, then it will 3645 * not restart the event. 3646 */ 3647 void __perf_event_task_sched_out(struct task_struct *task, 3648 struct task_struct *next) 3649 { 3650 int ctxn; 3651 3652 if (__this_cpu_read(perf_sched_cb_usages)) 3653 perf_pmu_sched_task(task, next, false); 3654 3655 if (atomic_read(&nr_switch_events)) 3656 perf_event_switch(task, next, false); 3657 3658 for_each_task_context_nr(ctxn) 3659 perf_event_context_sched_out(task, ctxn, next); 3660 3661 /* 3662 * if cgroup events exist on this CPU, then we need 3663 * to check if we have to switch out PMU state. 3664 * cgroup event are system-wide mode only 3665 */ 3666 if (atomic_read(this_cpu_ptr(&perf_cgroup_events))) 3667 perf_cgroup_sched_out(task, next); 3668 } 3669 3670 /* 3671 * Called with IRQs disabled 3672 */ 3673 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, 3674 enum event_type_t event_type) 3675 { 3676 ctx_sched_out(&cpuctx->ctx, cpuctx, event_type); 3677 } 3678 3679 static bool perf_less_group_idx(const void *l, const void *r) 3680 { 3681 const struct perf_event *le = *(const struct perf_event **)l; 3682 const struct perf_event *re = *(const struct perf_event **)r; 3683 3684 return le->group_index < re->group_index; 3685 } 3686 3687 static void swap_ptr(void *l, void *r) 3688 { 3689 void **lp = l, **rp = r; 3690 3691 swap(*lp, *rp); 3692 } 3693 3694 static const struct min_heap_callbacks perf_min_heap = { 3695 .elem_size = sizeof(struct perf_event *), 3696 .less = perf_less_group_idx, 3697 .swp = swap_ptr, 3698 }; 3699 3700 static void __heap_add(struct min_heap *heap, struct perf_event *event) 3701 { 3702 struct perf_event **itrs = heap->data; 3703 3704 if (event) { 3705 itrs[heap->nr] = event; 3706 heap->nr++; 3707 } 3708 } 3709 3710 static noinline int visit_groups_merge(struct perf_cpu_context *cpuctx, 3711 struct perf_event_groups *groups, int cpu, 3712 int (*func)(struct perf_event *, void *), 3713 void *data) 3714 { 3715 #ifdef CONFIG_CGROUP_PERF 3716 struct cgroup_subsys_state *css = NULL; 3717 #endif 3718 /* Space for per CPU and/or any CPU event iterators. */ 3719 struct perf_event *itrs[2]; 3720 struct min_heap event_heap; 3721 struct perf_event **evt; 3722 int ret; 3723 3724 if (cpuctx) { 3725 event_heap = (struct min_heap){ 3726 .data = cpuctx->heap, 3727 .nr = 0, 3728 .size = cpuctx->heap_size, 3729 }; 3730 3731 lockdep_assert_held(&cpuctx->ctx.lock); 3732 3733 #ifdef CONFIG_CGROUP_PERF 3734 if (cpuctx->cgrp) 3735 css = &cpuctx->cgrp->css; 3736 #endif 3737 } else { 3738 event_heap = (struct min_heap){ 3739 .data = itrs, 3740 .nr = 0, 3741 .size = ARRAY_SIZE(itrs), 3742 }; 3743 /* Events not within a CPU context may be on any CPU. */ 3744 __heap_add(&event_heap, perf_event_groups_first(groups, -1, NULL)); 3745 } 3746 evt = event_heap.data; 3747 3748 __heap_add(&event_heap, perf_event_groups_first(groups, cpu, NULL)); 3749 3750 #ifdef CONFIG_CGROUP_PERF 3751 for (; css; css = css->parent) 3752 __heap_add(&event_heap, perf_event_groups_first(groups, cpu, css->cgroup)); 3753 #endif 3754 3755 min_heapify_all(&event_heap, &perf_min_heap); 3756 3757 while (event_heap.nr) { 3758 ret = func(*evt, data); 3759 if (ret) 3760 return ret; 3761 3762 *evt = perf_event_groups_next(*evt); 3763 if (*evt) 3764 min_heapify(&event_heap, 0, &perf_min_heap); 3765 else 3766 min_heap_pop(&event_heap, &perf_min_heap); 3767 } 3768 3769 return 0; 3770 } 3771 3772 /* 3773 * Because the userpage is strictly per-event (there is no concept of context, 3774 * so there cannot be a context indirection), every userpage must be updated 3775 * when context time starts :-( 3776 * 3777 * IOW, we must not miss EVENT_TIME edges. 3778 */ 3779 static inline bool event_update_userpage(struct perf_event *event) 3780 { 3781 if (likely(!atomic_read(&event->mmap_count))) 3782 return false; 3783 3784 perf_event_update_time(event); 3785 perf_event_update_userpage(event); 3786 3787 return true; 3788 } 3789 3790 static inline void group_update_userpage(struct perf_event *group_event) 3791 { 3792 struct perf_event *event; 3793 3794 if (!event_update_userpage(group_event)) 3795 return; 3796 3797 for_each_sibling_event(event, group_event) 3798 event_update_userpage(event); 3799 } 3800 3801 static int merge_sched_in(struct perf_event *event, void *data) 3802 { 3803 struct perf_event_context *ctx = event->ctx; 3804 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 3805 int *can_add_hw = data; 3806 3807 if (event->state <= PERF_EVENT_STATE_OFF) 3808 return 0; 3809 3810 if (!event_filter_match(event)) 3811 return 0; 3812 3813 if (group_can_go_on(event, cpuctx, *can_add_hw)) { 3814 if (!group_sched_in(event, cpuctx, ctx)) 3815 list_add_tail(&event->active_list, get_event_list(event)); 3816 } 3817 3818 if (event->state == PERF_EVENT_STATE_INACTIVE) { 3819 *can_add_hw = 0; 3820 if (event->attr.pinned) { 3821 perf_cgroup_event_disable(event, ctx); 3822 perf_event_set_state(event, PERF_EVENT_STATE_ERROR); 3823 } else { 3824 ctx->rotate_necessary = 1; 3825 perf_mux_hrtimer_restart(cpuctx); 3826 group_update_userpage(event); 3827 } 3828 } 3829 3830 return 0; 3831 } 3832 3833 static void 3834 ctx_pinned_sched_in(struct perf_event_context *ctx, 3835 struct perf_cpu_context *cpuctx) 3836 { 3837 int can_add_hw = 1; 3838 3839 if (ctx != &cpuctx->ctx) 3840 cpuctx = NULL; 3841 3842 visit_groups_merge(cpuctx, &ctx->pinned_groups, 3843 smp_processor_id(), 3844 merge_sched_in, &can_add_hw); 3845 } 3846 3847 static void 3848 ctx_flexible_sched_in(struct perf_event_context *ctx, 3849 struct perf_cpu_context *cpuctx) 3850 { 3851 int can_add_hw = 1; 3852 3853 if (ctx != &cpuctx->ctx) 3854 cpuctx = NULL; 3855 3856 visit_groups_merge(cpuctx, &ctx->flexible_groups, 3857 smp_processor_id(), 3858 merge_sched_in, &can_add_hw); 3859 } 3860 3861 static void 3862 ctx_sched_in(struct perf_event_context *ctx, 3863 struct perf_cpu_context *cpuctx, 3864 enum event_type_t event_type, 3865 struct task_struct *task) 3866 { 3867 int is_active = ctx->is_active; 3868 3869 lockdep_assert_held(&ctx->lock); 3870 3871 if (likely(!ctx->nr_events)) 3872 return; 3873 3874 if (is_active ^ EVENT_TIME) { 3875 /* start ctx time */ 3876 __update_context_time(ctx, false); 3877 perf_cgroup_set_timestamp(task, ctx); 3878 /* 3879 * CPU-release for the below ->is_active store, 3880 * see __load_acquire() in perf_event_time_now() 3881 */ 3882 barrier(); 3883 } 3884 3885 ctx->is_active |= (event_type | EVENT_TIME); 3886 if (ctx->task) { 3887 if (!is_active) 3888 cpuctx->task_ctx = ctx; 3889 else 3890 WARN_ON_ONCE(cpuctx->task_ctx != ctx); 3891 } 3892 3893 is_active ^= ctx->is_active; /* changed bits */ 3894 3895 /* 3896 * First go through the list and put on any pinned groups 3897 * in order to give them the best chance of going on. 3898 */ 3899 if (is_active & EVENT_PINNED) 3900 ctx_pinned_sched_in(ctx, cpuctx); 3901 3902 /* Then walk through the lower prio flexible groups */ 3903 if (is_active & EVENT_FLEXIBLE) 3904 ctx_flexible_sched_in(ctx, cpuctx); 3905 } 3906 3907 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, 3908 enum event_type_t event_type, 3909 struct task_struct *task) 3910 { 3911 struct perf_event_context *ctx = &cpuctx->ctx; 3912 3913 ctx_sched_in(ctx, cpuctx, event_type, task); 3914 } 3915 3916 static void perf_event_context_sched_in(struct perf_event_context *ctx, 3917 struct task_struct *task) 3918 { 3919 struct perf_cpu_context *cpuctx; 3920 struct pmu *pmu; 3921 3922 cpuctx = __get_cpu_context(ctx); 3923 3924 /* 3925 * HACK: for HETEROGENEOUS the task context might have switched to a 3926 * different PMU, force (re)set the context, 3927 */ 3928 pmu = ctx->pmu = cpuctx->ctx.pmu; 3929 3930 if (cpuctx->task_ctx == ctx) { 3931 if (cpuctx->sched_cb_usage) 3932 __perf_pmu_sched_task(cpuctx, true); 3933 return; 3934 } 3935 3936 perf_ctx_lock(cpuctx, ctx); 3937 /* 3938 * We must check ctx->nr_events while holding ctx->lock, such 3939 * that we serialize against perf_install_in_context(). 3940 */ 3941 if (!ctx->nr_events) 3942 goto unlock; 3943 3944 perf_pmu_disable(pmu); 3945 /* 3946 * We want to keep the following priority order: 3947 * cpu pinned (that don't need to move), task pinned, 3948 * cpu flexible, task flexible. 3949 * 3950 * However, if task's ctx is not carrying any pinned 3951 * events, no need to flip the cpuctx's events around. 3952 */ 3953 if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) 3954 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE); 3955 perf_event_sched_in(cpuctx, ctx, task); 3956 3957 if (cpuctx->sched_cb_usage && pmu->sched_task) 3958 pmu->sched_task(cpuctx->task_ctx, true); 3959 3960 perf_pmu_enable(pmu); 3961 3962 unlock: 3963 perf_ctx_unlock(cpuctx, ctx); 3964 } 3965 3966 /* 3967 * Called from scheduler to add the events of the current task 3968 * with interrupts disabled. 3969 * 3970 * We restore the event value and then enable it. 3971 * 3972 * This does not protect us against NMI, but enable() 3973 * sets the enabled bit in the control field of event _before_ 3974 * accessing the event control register. If a NMI hits, then it will 3975 * keep the event running. 3976 */ 3977 void __perf_event_task_sched_in(struct task_struct *prev, 3978 struct task_struct *task) 3979 { 3980 struct perf_event_context *ctx; 3981 int ctxn; 3982 3983 /* 3984 * If cgroup events exist on this CPU, then we need to check if we have 3985 * to switch in PMU state; cgroup event are system-wide mode only. 3986 * 3987 * Since cgroup events are CPU events, we must schedule these in before 3988 * we schedule in the task events. 3989 */ 3990 if (atomic_read(this_cpu_ptr(&perf_cgroup_events))) 3991 perf_cgroup_sched_in(prev, task); 3992 3993 for_each_task_context_nr(ctxn) { 3994 ctx = task->perf_event_ctxp[ctxn]; 3995 if (likely(!ctx)) 3996 continue; 3997 3998 perf_event_context_sched_in(ctx, task); 3999 } 4000 4001 if (atomic_read(&nr_switch_events)) 4002 perf_event_switch(task, prev, true); 4003 4004 if (__this_cpu_read(perf_sched_cb_usages)) 4005 perf_pmu_sched_task(prev, task, true); 4006 } 4007 4008 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count) 4009 { 4010 u64 frequency = event->attr.sample_freq; 4011 u64 sec = NSEC_PER_SEC; 4012 u64 divisor, dividend; 4013 4014 int count_fls, nsec_fls, frequency_fls, sec_fls; 4015 4016 count_fls = fls64(count); 4017 nsec_fls = fls64(nsec); 4018 frequency_fls = fls64(frequency); 4019 sec_fls = 30; 4020 4021 /* 4022 * We got @count in @nsec, with a target of sample_freq HZ 4023 * the target period becomes: 4024 * 4025 * @count * 10^9 4026 * period = ------------------- 4027 * @nsec * sample_freq 4028 * 4029 */ 4030 4031 /* 4032 * Reduce accuracy by one bit such that @a and @b converge 4033 * to a similar magnitude. 4034 */ 4035 #define REDUCE_FLS(a, b) \ 4036 do { \ 4037 if (a##_fls > b##_fls) { \ 4038 a >>= 1; \ 4039 a##_fls--; \ 4040 } else { \ 4041 b >>= 1; \ 4042 b##_fls--; \ 4043 } \ 4044 } while (0) 4045 4046 /* 4047 * Reduce accuracy until either term fits in a u64, then proceed with 4048 * the other, so that finally we can do a u64/u64 division. 4049 */ 4050 while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) { 4051 REDUCE_FLS(nsec, frequency); 4052 REDUCE_FLS(sec, count); 4053 } 4054 4055 if (count_fls + sec_fls > 64) { 4056 divisor = nsec * frequency; 4057 4058 while (count_fls + sec_fls > 64) { 4059 REDUCE_FLS(count, sec); 4060 divisor >>= 1; 4061 } 4062 4063 dividend = count * sec; 4064 } else { 4065 dividend = count * sec; 4066 4067 while (nsec_fls + frequency_fls > 64) { 4068 REDUCE_FLS(nsec, frequency); 4069 dividend >>= 1; 4070 } 4071 4072 divisor = nsec * frequency; 4073 } 4074 4075 if (!divisor) 4076 return dividend; 4077 4078 return div64_u64(dividend, divisor); 4079 } 4080 4081 static DEFINE_PER_CPU(int, perf_throttled_count); 4082 static DEFINE_PER_CPU(u64, perf_throttled_seq); 4083 4084 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable) 4085 { 4086 struct hw_perf_event *hwc = &event->hw; 4087 s64 period, sample_period; 4088 s64 delta; 4089 4090 period = perf_calculate_period(event, nsec, count); 4091 4092 delta = (s64)(period - hwc->sample_period); 4093 delta = (delta + 7) / 8; /* low pass filter */ 4094 4095 sample_period = hwc->sample_period + delta; 4096 4097 if (!sample_period) 4098 sample_period = 1; 4099 4100 hwc->sample_period = sample_period; 4101 4102 if (local64_read(&hwc->period_left) > 8*sample_period) { 4103 if (disable) 4104 event->pmu->stop(event, PERF_EF_UPDATE); 4105 4106 local64_set(&hwc->period_left, 0); 4107 4108 if (disable) 4109 event->pmu->start(event, PERF_EF_RELOAD); 4110 } 4111 } 4112 4113 /* 4114 * combine freq adjustment with unthrottling to avoid two passes over the 4115 * events. At the same time, make sure, having freq events does not change 4116 * the rate of unthrottling as that would introduce bias. 4117 */ 4118 static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx, 4119 int needs_unthr) 4120 { 4121 struct perf_event *event; 4122 struct hw_perf_event *hwc; 4123 u64 now, period = TICK_NSEC; 4124 s64 delta; 4125 4126 /* 4127 * only need to iterate over all events iff: 4128 * - context have events in frequency mode (needs freq adjust) 4129 * - there are events to unthrottle on this cpu 4130 */ 4131 if (!(ctx->nr_freq || needs_unthr)) 4132 return; 4133 4134 raw_spin_lock(&ctx->lock); 4135 perf_pmu_disable(ctx->pmu); 4136 4137 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 4138 if (event->state != PERF_EVENT_STATE_ACTIVE) 4139 continue; 4140 4141 if (!event_filter_match(event)) 4142 continue; 4143 4144 perf_pmu_disable(event->pmu); 4145 4146 hwc = &event->hw; 4147 4148 if (hwc->interrupts == MAX_INTERRUPTS) { 4149 hwc->interrupts = 0; 4150 perf_log_throttle(event, 1); 4151 event->pmu->start(event, 0); 4152 } 4153 4154 if (!event->attr.freq || !event->attr.sample_freq) 4155 goto next; 4156 4157 /* 4158 * stop the event and update event->count 4159 */ 4160 event->pmu->stop(event, PERF_EF_UPDATE); 4161 4162 now = local64_read(&event->count); 4163 delta = now - hwc->freq_count_stamp; 4164 hwc->freq_count_stamp = now; 4165 4166 /* 4167 * restart the event 4168 * reload only if value has changed 4169 * we have stopped the event so tell that 4170 * to perf_adjust_period() to avoid stopping it 4171 * twice. 4172 */ 4173 if (delta > 0) 4174 perf_adjust_period(event, period, delta, false); 4175 4176 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0); 4177 next: 4178 perf_pmu_enable(event->pmu); 4179 } 4180 4181 perf_pmu_enable(ctx->pmu); 4182 raw_spin_unlock(&ctx->lock); 4183 } 4184 4185 /* 4186 * Move @event to the tail of the @ctx's elegible events. 4187 */ 4188 static void rotate_ctx(struct perf_event_context *ctx, struct perf_event *event) 4189 { 4190 /* 4191 * Rotate the first entry last of non-pinned groups. Rotation might be 4192 * disabled by the inheritance code. 4193 */ 4194 if (ctx->rotate_disable) 4195 return; 4196 4197 perf_event_groups_delete(&ctx->flexible_groups, event); 4198 perf_event_groups_insert(&ctx->flexible_groups, event); 4199 } 4200 4201 /* pick an event from the flexible_groups to rotate */ 4202 static inline struct perf_event * 4203 ctx_event_to_rotate(struct perf_event_context *ctx) 4204 { 4205 struct perf_event *event; 4206 4207 /* pick the first active flexible event */ 4208 event = list_first_entry_or_null(&ctx->flexible_active, 4209 struct perf_event, active_list); 4210 4211 /* if no active flexible event, pick the first event */ 4212 if (!event) { 4213 event = rb_entry_safe(rb_first(&ctx->flexible_groups.tree), 4214 typeof(*event), group_node); 4215 } 4216 4217 /* 4218 * Unconditionally clear rotate_necessary; if ctx_flexible_sched_in() 4219 * finds there are unschedulable events, it will set it again. 4220 */ 4221 ctx->rotate_necessary = 0; 4222 4223 return event; 4224 } 4225 4226 static bool perf_rotate_context(struct perf_cpu_context *cpuctx) 4227 { 4228 struct perf_event *cpu_event = NULL, *task_event = NULL; 4229 struct perf_event_context *task_ctx = NULL; 4230 int cpu_rotate, task_rotate; 4231 4232 /* 4233 * Since we run this from IRQ context, nobody can install new 4234 * events, thus the event count values are stable. 4235 */ 4236 4237 cpu_rotate = cpuctx->ctx.rotate_necessary; 4238 task_ctx = cpuctx->task_ctx; 4239 task_rotate = task_ctx ? task_ctx->rotate_necessary : 0; 4240 4241 if (!(cpu_rotate || task_rotate)) 4242 return false; 4243 4244 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 4245 perf_pmu_disable(cpuctx->ctx.pmu); 4246 4247 if (task_rotate) 4248 task_event = ctx_event_to_rotate(task_ctx); 4249 if (cpu_rotate) 4250 cpu_event = ctx_event_to_rotate(&cpuctx->ctx); 4251 4252 /* 4253 * As per the order given at ctx_resched() first 'pop' task flexible 4254 * and then, if needed CPU flexible. 4255 */ 4256 if (task_event || (task_ctx && cpu_event)) 4257 ctx_sched_out(task_ctx, cpuctx, EVENT_FLEXIBLE); 4258 if (cpu_event) 4259 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE); 4260 4261 if (task_event) 4262 rotate_ctx(task_ctx, task_event); 4263 if (cpu_event) 4264 rotate_ctx(&cpuctx->ctx, cpu_event); 4265 4266 perf_event_sched_in(cpuctx, task_ctx, current); 4267 4268 perf_pmu_enable(cpuctx->ctx.pmu); 4269 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 4270 4271 return true; 4272 } 4273 4274 void perf_event_task_tick(void) 4275 { 4276 struct list_head *head = this_cpu_ptr(&active_ctx_list); 4277 struct perf_event_context *ctx, *tmp; 4278 int throttled; 4279 4280 lockdep_assert_irqs_disabled(); 4281 4282 __this_cpu_inc(perf_throttled_seq); 4283 throttled = __this_cpu_xchg(perf_throttled_count, 0); 4284 tick_dep_clear_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS); 4285 4286 list_for_each_entry_safe(ctx, tmp, head, active_ctx_list) 4287 perf_adjust_freq_unthr_context(ctx, throttled); 4288 } 4289 4290 static int event_enable_on_exec(struct perf_event *event, 4291 struct perf_event_context *ctx) 4292 { 4293 if (!event->attr.enable_on_exec) 4294 return 0; 4295 4296 event->attr.enable_on_exec = 0; 4297 if (event->state >= PERF_EVENT_STATE_INACTIVE) 4298 return 0; 4299 4300 perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE); 4301 4302 return 1; 4303 } 4304 4305 /* 4306 * Enable all of a task's events that have been marked enable-on-exec. 4307 * This expects task == current. 4308 */ 4309 static void perf_event_enable_on_exec(int ctxn) 4310 { 4311 struct perf_event_context *ctx, *clone_ctx = NULL; 4312 enum event_type_t event_type = 0; 4313 struct perf_cpu_context *cpuctx; 4314 struct perf_event *event; 4315 unsigned long flags; 4316 int enabled = 0; 4317 4318 local_irq_save(flags); 4319 ctx = current->perf_event_ctxp[ctxn]; 4320 if (!ctx || !ctx->nr_events) 4321 goto out; 4322 4323 cpuctx = __get_cpu_context(ctx); 4324 perf_ctx_lock(cpuctx, ctx); 4325 ctx_sched_out(ctx, cpuctx, EVENT_TIME); 4326 list_for_each_entry(event, &ctx->event_list, event_entry) { 4327 enabled |= event_enable_on_exec(event, ctx); 4328 event_type |= get_event_type(event); 4329 } 4330 4331 /* 4332 * Unclone and reschedule this context if we enabled any event. 4333 */ 4334 if (enabled) { 4335 clone_ctx = unclone_ctx(ctx); 4336 ctx_resched(cpuctx, ctx, event_type); 4337 } else { 4338 ctx_sched_in(ctx, cpuctx, EVENT_TIME, current); 4339 } 4340 perf_ctx_unlock(cpuctx, ctx); 4341 4342 out: 4343 local_irq_restore(flags); 4344 4345 if (clone_ctx) 4346 put_ctx(clone_ctx); 4347 } 4348 4349 static void perf_remove_from_owner(struct perf_event *event); 4350 static void perf_event_exit_event(struct perf_event *event, 4351 struct perf_event_context *ctx); 4352 4353 /* 4354 * Removes all events from the current task that have been marked 4355 * remove-on-exec, and feeds their values back to parent events. 4356 */ 4357 static void perf_event_remove_on_exec(int ctxn) 4358 { 4359 struct perf_event_context *ctx, *clone_ctx = NULL; 4360 struct perf_event *event, *next; 4361 LIST_HEAD(free_list); 4362 unsigned long flags; 4363 bool modified = false; 4364 4365 ctx = perf_pin_task_context(current, ctxn); 4366 if (!ctx) 4367 return; 4368 4369 mutex_lock(&ctx->mutex); 4370 4371 if (WARN_ON_ONCE(ctx->task != current)) 4372 goto unlock; 4373 4374 list_for_each_entry_safe(event, next, &ctx->event_list, event_entry) { 4375 if (!event->attr.remove_on_exec) 4376 continue; 4377 4378 if (!is_kernel_event(event)) 4379 perf_remove_from_owner(event); 4380 4381 modified = true; 4382 4383 perf_event_exit_event(event, ctx); 4384 } 4385 4386 raw_spin_lock_irqsave(&ctx->lock, flags); 4387 if (modified) 4388 clone_ctx = unclone_ctx(ctx); 4389 --ctx->pin_count; 4390 raw_spin_unlock_irqrestore(&ctx->lock, flags); 4391 4392 unlock: 4393 mutex_unlock(&ctx->mutex); 4394 4395 put_ctx(ctx); 4396 if (clone_ctx) 4397 put_ctx(clone_ctx); 4398 } 4399 4400 struct perf_read_data { 4401 struct perf_event *event; 4402 bool group; 4403 int ret; 4404 }; 4405 4406 static int __perf_event_read_cpu(struct perf_event *event, int event_cpu) 4407 { 4408 u16 local_pkg, event_pkg; 4409 4410 if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) { 4411 int local_cpu = smp_processor_id(); 4412 4413 event_pkg = topology_physical_package_id(event_cpu); 4414 local_pkg = topology_physical_package_id(local_cpu); 4415 4416 if (event_pkg == local_pkg) 4417 return local_cpu; 4418 } 4419 4420 return event_cpu; 4421 } 4422 4423 /* 4424 * Cross CPU call to read the hardware event 4425 */ 4426 static void __perf_event_read(void *info) 4427 { 4428 struct perf_read_data *data = info; 4429 struct perf_event *sub, *event = data->event; 4430 struct perf_event_context *ctx = event->ctx; 4431 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 4432 struct pmu *pmu = event->pmu; 4433 4434 /* 4435 * If this is a task context, we need to check whether it is 4436 * the current task context of this cpu. If not it has been 4437 * scheduled out before the smp call arrived. In that case 4438 * event->count would have been updated to a recent sample 4439 * when the event was scheduled out. 4440 */ 4441 if (ctx->task && cpuctx->task_ctx != ctx) 4442 return; 4443 4444 raw_spin_lock(&ctx->lock); 4445 if (ctx->is_active & EVENT_TIME) { 4446 update_context_time(ctx); 4447 update_cgrp_time_from_event(event); 4448 } 4449 4450 perf_event_update_time(event); 4451 if (data->group) 4452 perf_event_update_sibling_time(event); 4453 4454 if (event->state != PERF_EVENT_STATE_ACTIVE) 4455 goto unlock; 4456 4457 if (!data->group) { 4458 pmu->read(event); 4459 data->ret = 0; 4460 goto unlock; 4461 } 4462 4463 pmu->start_txn(pmu, PERF_PMU_TXN_READ); 4464 4465 pmu->read(event); 4466 4467 for_each_sibling_event(sub, event) { 4468 if (sub->state == PERF_EVENT_STATE_ACTIVE) { 4469 /* 4470 * Use sibling's PMU rather than @event's since 4471 * sibling could be on different (eg: software) PMU. 4472 */ 4473 sub->pmu->read(sub); 4474 } 4475 } 4476 4477 data->ret = pmu->commit_txn(pmu); 4478 4479 unlock: 4480 raw_spin_unlock(&ctx->lock); 4481 } 4482 4483 static inline u64 perf_event_count(struct perf_event *event) 4484 { 4485 return local64_read(&event->count) + atomic64_read(&event->child_count); 4486 } 4487 4488 static void calc_timer_values(struct perf_event *event, 4489 u64 *now, 4490 u64 *enabled, 4491 u64 *running) 4492 { 4493 u64 ctx_time; 4494 4495 *now = perf_clock(); 4496 ctx_time = perf_event_time_now(event, *now); 4497 __perf_update_times(event, ctx_time, enabled, running); 4498 } 4499 4500 /* 4501 * NMI-safe method to read a local event, that is an event that 4502 * is: 4503 * - either for the current task, or for this CPU 4504 * - does not have inherit set, for inherited task events 4505 * will not be local and we cannot read them atomically 4506 * - must not have a pmu::count method 4507 */ 4508 int perf_event_read_local(struct perf_event *event, u64 *value, 4509 u64 *enabled, u64 *running) 4510 { 4511 unsigned long flags; 4512 int ret = 0; 4513 4514 /* 4515 * Disabling interrupts avoids all counter scheduling (context 4516 * switches, timer based rotation and IPIs). 4517 */ 4518 local_irq_save(flags); 4519 4520 /* 4521 * It must not be an event with inherit set, we cannot read 4522 * all child counters from atomic context. 4523 */ 4524 if (event->attr.inherit) { 4525 ret = -EOPNOTSUPP; 4526 goto out; 4527 } 4528 4529 /* If this is a per-task event, it must be for current */ 4530 if ((event->attach_state & PERF_ATTACH_TASK) && 4531 event->hw.target != current) { 4532 ret = -EINVAL; 4533 goto out; 4534 } 4535 4536 /* If this is a per-CPU event, it must be for this CPU */ 4537 if (!(event->attach_state & PERF_ATTACH_TASK) && 4538 event->cpu != smp_processor_id()) { 4539 ret = -EINVAL; 4540 goto out; 4541 } 4542 4543 /* If this is a pinned event it must be running on this CPU */ 4544 if (event->attr.pinned && event->oncpu != smp_processor_id()) { 4545 ret = -EBUSY; 4546 goto out; 4547 } 4548 4549 /* 4550 * If the event is currently on this CPU, its either a per-task event, 4551 * or local to this CPU. Furthermore it means its ACTIVE (otherwise 4552 * oncpu == -1). 4553 */ 4554 if (event->oncpu == smp_processor_id()) 4555 event->pmu->read(event); 4556 4557 *value = local64_read(&event->count); 4558 if (enabled || running) { 4559 u64 __enabled, __running, __now;; 4560 4561 calc_timer_values(event, &__now, &__enabled, &__running); 4562 if (enabled) 4563 *enabled = __enabled; 4564 if (running) 4565 *running = __running; 4566 } 4567 out: 4568 local_irq_restore(flags); 4569 4570 return ret; 4571 } 4572 4573 static int perf_event_read(struct perf_event *event, bool group) 4574 { 4575 enum perf_event_state state = READ_ONCE(event->state); 4576 int event_cpu, ret = 0; 4577 4578 /* 4579 * If event is enabled and currently active on a CPU, update the 4580 * value in the event structure: 4581 */ 4582 again: 4583 if (state == PERF_EVENT_STATE_ACTIVE) { 4584 struct perf_read_data data; 4585 4586 /* 4587 * Orders the ->state and ->oncpu loads such that if we see 4588 * ACTIVE we must also see the right ->oncpu. 4589 * 4590 * Matches the smp_wmb() from event_sched_in(). 4591 */ 4592 smp_rmb(); 4593 4594 event_cpu = READ_ONCE(event->oncpu); 4595 if ((unsigned)event_cpu >= nr_cpu_ids) 4596 return 0; 4597 4598 data = (struct perf_read_data){ 4599 .event = event, 4600 .group = group, 4601 .ret = 0, 4602 }; 4603 4604 preempt_disable(); 4605 event_cpu = __perf_event_read_cpu(event, event_cpu); 4606 4607 /* 4608 * Purposely ignore the smp_call_function_single() return 4609 * value. 4610 * 4611 * If event_cpu isn't a valid CPU it means the event got 4612 * scheduled out and that will have updated the event count. 4613 * 4614 * Therefore, either way, we'll have an up-to-date event count 4615 * after this. 4616 */ 4617 (void)smp_call_function_single(event_cpu, __perf_event_read, &data, 1); 4618 preempt_enable(); 4619 ret = data.ret; 4620 4621 } else if (state == PERF_EVENT_STATE_INACTIVE) { 4622 struct perf_event_context *ctx = event->ctx; 4623 unsigned long flags; 4624 4625 raw_spin_lock_irqsave(&ctx->lock, flags); 4626 state = event->state; 4627 if (state != PERF_EVENT_STATE_INACTIVE) { 4628 raw_spin_unlock_irqrestore(&ctx->lock, flags); 4629 goto again; 4630 } 4631 4632 /* 4633 * May read while context is not active (e.g., thread is 4634 * blocked), in that case we cannot update context time 4635 */ 4636 if (ctx->is_active & EVENT_TIME) { 4637 update_context_time(ctx); 4638 update_cgrp_time_from_event(event); 4639 } 4640 4641 perf_event_update_time(event); 4642 if (group) 4643 perf_event_update_sibling_time(event); 4644 raw_spin_unlock_irqrestore(&ctx->lock, flags); 4645 } 4646 4647 return ret; 4648 } 4649 4650 /* 4651 * Initialize the perf_event context in a task_struct: 4652 */ 4653 static void __perf_event_init_context(struct perf_event_context *ctx) 4654 { 4655 raw_spin_lock_init(&ctx->lock); 4656 mutex_init(&ctx->mutex); 4657 INIT_LIST_HEAD(&ctx->active_ctx_list); 4658 perf_event_groups_init(&ctx->pinned_groups); 4659 perf_event_groups_init(&ctx->flexible_groups); 4660 INIT_LIST_HEAD(&ctx->event_list); 4661 INIT_LIST_HEAD(&ctx->pinned_active); 4662 INIT_LIST_HEAD(&ctx->flexible_active); 4663 refcount_set(&ctx->refcount, 1); 4664 } 4665 4666 static struct perf_event_context * 4667 alloc_perf_context(struct pmu *pmu, struct task_struct *task) 4668 { 4669 struct perf_event_context *ctx; 4670 4671 ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL); 4672 if (!ctx) 4673 return NULL; 4674 4675 __perf_event_init_context(ctx); 4676 if (task) 4677 ctx->task = get_task_struct(task); 4678 ctx->pmu = pmu; 4679 4680 return ctx; 4681 } 4682 4683 static struct task_struct * 4684 find_lively_task_by_vpid(pid_t vpid) 4685 { 4686 struct task_struct *task; 4687 4688 rcu_read_lock(); 4689 if (!vpid) 4690 task = current; 4691 else 4692 task = find_task_by_vpid(vpid); 4693 if (task) 4694 get_task_struct(task); 4695 rcu_read_unlock(); 4696 4697 if (!task) 4698 return ERR_PTR(-ESRCH); 4699 4700 return task; 4701 } 4702 4703 /* 4704 * Returns a matching context with refcount and pincount. 4705 */ 4706 static struct perf_event_context * 4707 find_get_context(struct pmu *pmu, struct task_struct *task, 4708 struct perf_event *event) 4709 { 4710 struct perf_event_context *ctx, *clone_ctx = NULL; 4711 struct perf_cpu_context *cpuctx; 4712 void *task_ctx_data = NULL; 4713 unsigned long flags; 4714 int ctxn, err; 4715 int cpu = event->cpu; 4716 4717 if (!task) { 4718 /* Must be root to operate on a CPU event: */ 4719 err = perf_allow_cpu(&event->attr); 4720 if (err) 4721 return ERR_PTR(err); 4722 4723 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 4724 ctx = &cpuctx->ctx; 4725 get_ctx(ctx); 4726 raw_spin_lock_irqsave(&ctx->lock, flags); 4727 ++ctx->pin_count; 4728 raw_spin_unlock_irqrestore(&ctx->lock, flags); 4729 4730 return ctx; 4731 } 4732 4733 err = -EINVAL; 4734 ctxn = pmu->task_ctx_nr; 4735 if (ctxn < 0) 4736 goto errout; 4737 4738 if (event->attach_state & PERF_ATTACH_TASK_DATA) { 4739 task_ctx_data = alloc_task_ctx_data(pmu); 4740 if (!task_ctx_data) { 4741 err = -ENOMEM; 4742 goto errout; 4743 } 4744 } 4745 4746 retry: 4747 ctx = perf_lock_task_context(task, ctxn, &flags); 4748 if (ctx) { 4749 clone_ctx = unclone_ctx(ctx); 4750 ++ctx->pin_count; 4751 4752 if (task_ctx_data && !ctx->task_ctx_data) { 4753 ctx->task_ctx_data = task_ctx_data; 4754 task_ctx_data = NULL; 4755 } 4756 raw_spin_unlock_irqrestore(&ctx->lock, flags); 4757 4758 if (clone_ctx) 4759 put_ctx(clone_ctx); 4760 } else { 4761 ctx = alloc_perf_context(pmu, task); 4762 err = -ENOMEM; 4763 if (!ctx) 4764 goto errout; 4765 4766 if (task_ctx_data) { 4767 ctx->task_ctx_data = task_ctx_data; 4768 task_ctx_data = NULL; 4769 } 4770 4771 err = 0; 4772 mutex_lock(&task->perf_event_mutex); 4773 /* 4774 * If it has already passed perf_event_exit_task(). 4775 * we must see PF_EXITING, it takes this mutex too. 4776 */ 4777 if (task->flags & PF_EXITING) 4778 err = -ESRCH; 4779 else if (task->perf_event_ctxp[ctxn]) 4780 err = -EAGAIN; 4781 else { 4782 get_ctx(ctx); 4783 ++ctx->pin_count; 4784 rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx); 4785 } 4786 mutex_unlock(&task->perf_event_mutex); 4787 4788 if (unlikely(err)) { 4789 put_ctx(ctx); 4790 4791 if (err == -EAGAIN) 4792 goto retry; 4793 goto errout; 4794 } 4795 } 4796 4797 free_task_ctx_data(pmu, task_ctx_data); 4798 return ctx; 4799 4800 errout: 4801 free_task_ctx_data(pmu, task_ctx_data); 4802 return ERR_PTR(err); 4803 } 4804 4805 static void perf_event_free_filter(struct perf_event *event); 4806 4807 static void free_event_rcu(struct rcu_head *head) 4808 { 4809 struct perf_event *event; 4810 4811 event = container_of(head, struct perf_event, rcu_head); 4812 if (event->ns) 4813 put_pid_ns(event->ns); 4814 perf_event_free_filter(event); 4815 kmem_cache_free(perf_event_cache, event); 4816 } 4817 4818 static void ring_buffer_attach(struct perf_event *event, 4819 struct perf_buffer *rb); 4820 4821 static void detach_sb_event(struct perf_event *event) 4822 { 4823 struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu); 4824 4825 raw_spin_lock(&pel->lock); 4826 list_del_rcu(&event->sb_list); 4827 raw_spin_unlock(&pel->lock); 4828 } 4829 4830 static bool is_sb_event(struct perf_event *event) 4831 { 4832 struct perf_event_attr *attr = &event->attr; 4833 4834 if (event->parent) 4835 return false; 4836 4837 if (event->attach_state & PERF_ATTACH_TASK) 4838 return false; 4839 4840 if (attr->mmap || attr->mmap_data || attr->mmap2 || 4841 attr->comm || attr->comm_exec || 4842 attr->task || attr->ksymbol || 4843 attr->context_switch || attr->text_poke || 4844 attr->bpf_event) 4845 return true; 4846 return false; 4847 } 4848 4849 static void unaccount_pmu_sb_event(struct perf_event *event) 4850 { 4851 if (is_sb_event(event)) 4852 detach_sb_event(event); 4853 } 4854 4855 static void unaccount_event_cpu(struct perf_event *event, int cpu) 4856 { 4857 if (event->parent) 4858 return; 4859 4860 if (is_cgroup_event(event)) 4861 atomic_dec(&per_cpu(perf_cgroup_events, cpu)); 4862 } 4863 4864 #ifdef CONFIG_NO_HZ_FULL 4865 static DEFINE_SPINLOCK(nr_freq_lock); 4866 #endif 4867 4868 static void unaccount_freq_event_nohz(void) 4869 { 4870 #ifdef CONFIG_NO_HZ_FULL 4871 spin_lock(&nr_freq_lock); 4872 if (atomic_dec_and_test(&nr_freq_events)) 4873 tick_nohz_dep_clear(TICK_DEP_BIT_PERF_EVENTS); 4874 spin_unlock(&nr_freq_lock); 4875 #endif 4876 } 4877 4878 static void unaccount_freq_event(void) 4879 { 4880 if (tick_nohz_full_enabled()) 4881 unaccount_freq_event_nohz(); 4882 else 4883 atomic_dec(&nr_freq_events); 4884 } 4885 4886 static void unaccount_event(struct perf_event *event) 4887 { 4888 bool dec = false; 4889 4890 if (event->parent) 4891 return; 4892 4893 if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB)) 4894 dec = true; 4895 if (event->attr.mmap || event->attr.mmap_data) 4896 atomic_dec(&nr_mmap_events); 4897 if (event->attr.build_id) 4898 atomic_dec(&nr_build_id_events); 4899 if (event->attr.comm) 4900 atomic_dec(&nr_comm_events); 4901 if (event->attr.namespaces) 4902 atomic_dec(&nr_namespaces_events); 4903 if (event->attr.cgroup) 4904 atomic_dec(&nr_cgroup_events); 4905 if (event->attr.task) 4906 atomic_dec(&nr_task_events); 4907 if (event->attr.freq) 4908 unaccount_freq_event(); 4909 if (event->attr.context_switch) { 4910 dec = true; 4911 atomic_dec(&nr_switch_events); 4912 } 4913 if (is_cgroup_event(event)) 4914 dec = true; 4915 if (has_branch_stack(event)) 4916 dec = true; 4917 if (event->attr.ksymbol) 4918 atomic_dec(&nr_ksymbol_events); 4919 if (event->attr.bpf_event) 4920 atomic_dec(&nr_bpf_events); 4921 if (event->attr.text_poke) 4922 atomic_dec(&nr_text_poke_events); 4923 4924 if (dec) { 4925 if (!atomic_add_unless(&perf_sched_count, -1, 1)) 4926 schedule_delayed_work(&perf_sched_work, HZ); 4927 } 4928 4929 unaccount_event_cpu(event, event->cpu); 4930 4931 unaccount_pmu_sb_event(event); 4932 } 4933 4934 static void perf_sched_delayed(struct work_struct *work) 4935 { 4936 mutex_lock(&perf_sched_mutex); 4937 if (atomic_dec_and_test(&perf_sched_count)) 4938 static_branch_disable(&perf_sched_events); 4939 mutex_unlock(&perf_sched_mutex); 4940 } 4941 4942 /* 4943 * The following implement mutual exclusion of events on "exclusive" pmus 4944 * (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled 4945 * at a time, so we disallow creating events that might conflict, namely: 4946 * 4947 * 1) cpu-wide events in the presence of per-task events, 4948 * 2) per-task events in the presence of cpu-wide events, 4949 * 3) two matching events on the same context. 4950 * 4951 * The former two cases are handled in the allocation path (perf_event_alloc(), 4952 * _free_event()), the latter -- before the first perf_install_in_context(). 4953 */ 4954 static int exclusive_event_init(struct perf_event *event) 4955 { 4956 struct pmu *pmu = event->pmu; 4957 4958 if (!is_exclusive_pmu(pmu)) 4959 return 0; 4960 4961 /* 4962 * Prevent co-existence of per-task and cpu-wide events on the 4963 * same exclusive pmu. 4964 * 4965 * Negative pmu::exclusive_cnt means there are cpu-wide 4966 * events on this "exclusive" pmu, positive means there are 4967 * per-task events. 4968 * 4969 * Since this is called in perf_event_alloc() path, event::ctx 4970 * doesn't exist yet; it is, however, safe to use PERF_ATTACH_TASK 4971 * to mean "per-task event", because unlike other attach states it 4972 * never gets cleared. 4973 */ 4974 if (event->attach_state & PERF_ATTACH_TASK) { 4975 if (!atomic_inc_unless_negative(&pmu->exclusive_cnt)) 4976 return -EBUSY; 4977 } else { 4978 if (!atomic_dec_unless_positive(&pmu->exclusive_cnt)) 4979 return -EBUSY; 4980 } 4981 4982 return 0; 4983 } 4984 4985 static void exclusive_event_destroy(struct perf_event *event) 4986 { 4987 struct pmu *pmu = event->pmu; 4988 4989 if (!is_exclusive_pmu(pmu)) 4990 return; 4991 4992 /* see comment in exclusive_event_init() */ 4993 if (event->attach_state & PERF_ATTACH_TASK) 4994 atomic_dec(&pmu->exclusive_cnt); 4995 else 4996 atomic_inc(&pmu->exclusive_cnt); 4997 } 4998 4999 static bool exclusive_event_match(struct perf_event *e1, struct perf_event *e2) 5000 { 5001 if ((e1->pmu == e2->pmu) && 5002 (e1->cpu == e2->cpu || 5003 e1->cpu == -1 || 5004 e2->cpu == -1)) 5005 return true; 5006 return false; 5007 } 5008 5009 static bool exclusive_event_installable(struct perf_event *event, 5010 struct perf_event_context *ctx) 5011 { 5012 struct perf_event *iter_event; 5013 struct pmu *pmu = event->pmu; 5014 5015 lockdep_assert_held(&ctx->mutex); 5016 5017 if (!is_exclusive_pmu(pmu)) 5018 return true; 5019 5020 list_for_each_entry(iter_event, &ctx->event_list, event_entry) { 5021 if (exclusive_event_match(iter_event, event)) 5022 return false; 5023 } 5024 5025 return true; 5026 } 5027 5028 static void perf_addr_filters_splice(struct perf_event *event, 5029 struct list_head *head); 5030 5031 static void _free_event(struct perf_event *event) 5032 { 5033 irq_work_sync(&event->pending); 5034 5035 unaccount_event(event); 5036 5037 security_perf_event_free(event); 5038 5039 if (event->rb) { 5040 /* 5041 * Can happen when we close an event with re-directed output. 5042 * 5043 * Since we have a 0 refcount, perf_mmap_close() will skip 5044 * over us; possibly making our ring_buffer_put() the last. 5045 */ 5046 mutex_lock(&event->mmap_mutex); 5047 ring_buffer_attach(event, NULL); 5048 mutex_unlock(&event->mmap_mutex); 5049 } 5050 5051 if (is_cgroup_event(event)) 5052 perf_detach_cgroup(event); 5053 5054 if (!event->parent) { 5055 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) 5056 put_callchain_buffers(); 5057 } 5058 5059 perf_event_free_bpf_prog(event); 5060 perf_addr_filters_splice(event, NULL); 5061 kfree(event->addr_filter_ranges); 5062 5063 if (event->destroy) 5064 event->destroy(event); 5065 5066 /* 5067 * Must be after ->destroy(), due to uprobe_perf_close() using 5068 * hw.target. 5069 */ 5070 if (event->hw.target) 5071 put_task_struct(event->hw.target); 5072 5073 /* 5074 * perf_event_free_task() relies on put_ctx() being 'last', in particular 5075 * all task references must be cleaned up. 5076 */ 5077 if (event->ctx) 5078 put_ctx(event->ctx); 5079 5080 exclusive_event_destroy(event); 5081 module_put(event->pmu->module); 5082 5083 call_rcu(&event->rcu_head, free_event_rcu); 5084 } 5085 5086 /* 5087 * Used to free events which have a known refcount of 1, such as in error paths 5088 * where the event isn't exposed yet and inherited events. 5089 */ 5090 static void free_event(struct perf_event *event) 5091 { 5092 if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1, 5093 "unexpected event refcount: %ld; ptr=%p\n", 5094 atomic_long_read(&event->refcount), event)) { 5095 /* leak to avoid use-after-free */ 5096 return; 5097 } 5098 5099 _free_event(event); 5100 } 5101 5102 /* 5103 * Remove user event from the owner task. 5104 */ 5105 static void perf_remove_from_owner(struct perf_event *event) 5106 { 5107 struct task_struct *owner; 5108 5109 rcu_read_lock(); 5110 /* 5111 * Matches the smp_store_release() in perf_event_exit_task(). If we 5112 * observe !owner it means the list deletion is complete and we can 5113 * indeed free this event, otherwise we need to serialize on 5114 * owner->perf_event_mutex. 5115 */ 5116 owner = READ_ONCE(event->owner); 5117 if (owner) { 5118 /* 5119 * Since delayed_put_task_struct() also drops the last 5120 * task reference we can safely take a new reference 5121 * while holding the rcu_read_lock(). 5122 */ 5123 get_task_struct(owner); 5124 } 5125 rcu_read_unlock(); 5126 5127 if (owner) { 5128 /* 5129 * If we're here through perf_event_exit_task() we're already 5130 * holding ctx->mutex which would be an inversion wrt. the 5131 * normal lock order. 5132 * 5133 * However we can safely take this lock because its the child 5134 * ctx->mutex. 5135 */ 5136 mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING); 5137 5138 /* 5139 * We have to re-check the event->owner field, if it is cleared 5140 * we raced with perf_event_exit_task(), acquiring the mutex 5141 * ensured they're done, and we can proceed with freeing the 5142 * event. 5143 */ 5144 if (event->owner) { 5145 list_del_init(&event->owner_entry); 5146 smp_store_release(&event->owner, NULL); 5147 } 5148 mutex_unlock(&owner->perf_event_mutex); 5149 put_task_struct(owner); 5150 } 5151 } 5152 5153 static void put_event(struct perf_event *event) 5154 { 5155 if (!atomic_long_dec_and_test(&event->refcount)) 5156 return; 5157 5158 _free_event(event); 5159 } 5160 5161 /* 5162 * Kill an event dead; while event:refcount will preserve the event 5163 * object, it will not preserve its functionality. Once the last 'user' 5164 * gives up the object, we'll destroy the thing. 5165 */ 5166 int perf_event_release_kernel(struct perf_event *event) 5167 { 5168 struct perf_event_context *ctx = event->ctx; 5169 struct perf_event *child, *tmp; 5170 LIST_HEAD(free_list); 5171 5172 /* 5173 * If we got here through err_file: fput(event_file); we will not have 5174 * attached to a context yet. 5175 */ 5176 if (!ctx) { 5177 WARN_ON_ONCE(event->attach_state & 5178 (PERF_ATTACH_CONTEXT|PERF_ATTACH_GROUP)); 5179 goto no_ctx; 5180 } 5181 5182 if (!is_kernel_event(event)) 5183 perf_remove_from_owner(event); 5184 5185 ctx = perf_event_ctx_lock(event); 5186 WARN_ON_ONCE(ctx->parent_ctx); 5187 perf_remove_from_context(event, DETACH_GROUP); 5188 5189 raw_spin_lock_irq(&ctx->lock); 5190 /* 5191 * Mark this event as STATE_DEAD, there is no external reference to it 5192 * anymore. 5193 * 5194 * Anybody acquiring event->child_mutex after the below loop _must_ 5195 * also see this, most importantly inherit_event() which will avoid 5196 * placing more children on the list. 5197 * 5198 * Thus this guarantees that we will in fact observe and kill _ALL_ 5199 * child events. 5200 */ 5201 event->state = PERF_EVENT_STATE_DEAD; 5202 raw_spin_unlock_irq(&ctx->lock); 5203 5204 perf_event_ctx_unlock(event, ctx); 5205 5206 again: 5207 mutex_lock(&event->child_mutex); 5208 list_for_each_entry(child, &event->child_list, child_list) { 5209 5210 /* 5211 * Cannot change, child events are not migrated, see the 5212 * comment with perf_event_ctx_lock_nested(). 5213 */ 5214 ctx = READ_ONCE(child->ctx); 5215 /* 5216 * Since child_mutex nests inside ctx::mutex, we must jump 5217 * through hoops. We start by grabbing a reference on the ctx. 5218 * 5219 * Since the event cannot get freed while we hold the 5220 * child_mutex, the context must also exist and have a !0 5221 * reference count. 5222 */ 5223 get_ctx(ctx); 5224 5225 /* 5226 * Now that we have a ctx ref, we can drop child_mutex, and 5227 * acquire ctx::mutex without fear of it going away. Then we 5228 * can re-acquire child_mutex. 5229 */ 5230 mutex_unlock(&event->child_mutex); 5231 mutex_lock(&ctx->mutex); 5232 mutex_lock(&event->child_mutex); 5233 5234 /* 5235 * Now that we hold ctx::mutex and child_mutex, revalidate our 5236 * state, if child is still the first entry, it didn't get freed 5237 * and we can continue doing so. 5238 */ 5239 tmp = list_first_entry_or_null(&event->child_list, 5240 struct perf_event, child_list); 5241 if (tmp == child) { 5242 perf_remove_from_context(child, DETACH_GROUP); 5243 list_move(&child->child_list, &free_list); 5244 /* 5245 * This matches the refcount bump in inherit_event(); 5246 * this can't be the last reference. 5247 */ 5248 put_event(event); 5249 } 5250 5251 mutex_unlock(&event->child_mutex); 5252 mutex_unlock(&ctx->mutex); 5253 put_ctx(ctx); 5254 goto again; 5255 } 5256 mutex_unlock(&event->child_mutex); 5257 5258 list_for_each_entry_safe(child, tmp, &free_list, child_list) { 5259 void *var = &child->ctx->refcount; 5260 5261 list_del(&child->child_list); 5262 free_event(child); 5263 5264 /* 5265 * Wake any perf_event_free_task() waiting for this event to be 5266 * freed. 5267 */ 5268 smp_mb(); /* pairs with wait_var_event() */ 5269 wake_up_var(var); 5270 } 5271 5272 no_ctx: 5273 put_event(event); /* Must be the 'last' reference */ 5274 return 0; 5275 } 5276 EXPORT_SYMBOL_GPL(perf_event_release_kernel); 5277 5278 /* 5279 * Called when the last reference to the file is gone. 5280 */ 5281 static int perf_release(struct inode *inode, struct file *file) 5282 { 5283 perf_event_release_kernel(file->private_data); 5284 return 0; 5285 } 5286 5287 static u64 __perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running) 5288 { 5289 struct perf_event *child; 5290 u64 total = 0; 5291 5292 *enabled = 0; 5293 *running = 0; 5294 5295 mutex_lock(&event->child_mutex); 5296 5297 (void)perf_event_read(event, false); 5298 total += perf_event_count(event); 5299 5300 *enabled += event->total_time_enabled + 5301 atomic64_read(&event->child_total_time_enabled); 5302 *running += event->total_time_running + 5303 atomic64_read(&event->child_total_time_running); 5304 5305 list_for_each_entry(child, &event->child_list, child_list) { 5306 (void)perf_event_read(child, false); 5307 total += perf_event_count(child); 5308 *enabled += child->total_time_enabled; 5309 *running += child->total_time_running; 5310 } 5311 mutex_unlock(&event->child_mutex); 5312 5313 return total; 5314 } 5315 5316 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running) 5317 { 5318 struct perf_event_context *ctx; 5319 u64 count; 5320 5321 ctx = perf_event_ctx_lock(event); 5322 count = __perf_event_read_value(event, enabled, running); 5323 perf_event_ctx_unlock(event, ctx); 5324 5325 return count; 5326 } 5327 EXPORT_SYMBOL_GPL(perf_event_read_value); 5328 5329 static int __perf_read_group_add(struct perf_event *leader, 5330 u64 read_format, u64 *values) 5331 { 5332 struct perf_event_context *ctx = leader->ctx; 5333 struct perf_event *sub; 5334 unsigned long flags; 5335 int n = 1; /* skip @nr */ 5336 int ret; 5337 5338 ret = perf_event_read(leader, true); 5339 if (ret) 5340 return ret; 5341 5342 raw_spin_lock_irqsave(&ctx->lock, flags); 5343 5344 /* 5345 * Since we co-schedule groups, {enabled,running} times of siblings 5346 * will be identical to those of the leader, so we only publish one 5347 * set. 5348 */ 5349 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { 5350 values[n++] += leader->total_time_enabled + 5351 atomic64_read(&leader->child_total_time_enabled); 5352 } 5353 5354 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { 5355 values[n++] += leader->total_time_running + 5356 atomic64_read(&leader->child_total_time_running); 5357 } 5358 5359 /* 5360 * Write {count,id} tuples for every sibling. 5361 */ 5362 values[n++] += perf_event_count(leader); 5363 if (read_format & PERF_FORMAT_ID) 5364 values[n++] = primary_event_id(leader); 5365 5366 for_each_sibling_event(sub, leader) { 5367 values[n++] += perf_event_count(sub); 5368 if (read_format & PERF_FORMAT_ID) 5369 values[n++] = primary_event_id(sub); 5370 } 5371 5372 raw_spin_unlock_irqrestore(&ctx->lock, flags); 5373 return 0; 5374 } 5375 5376 static int perf_read_group(struct perf_event *event, 5377 u64 read_format, char __user *buf) 5378 { 5379 struct perf_event *leader = event->group_leader, *child; 5380 struct perf_event_context *ctx = leader->ctx; 5381 int ret; 5382 u64 *values; 5383 5384 lockdep_assert_held(&ctx->mutex); 5385 5386 values = kzalloc(event->read_size, GFP_KERNEL); 5387 if (!values) 5388 return -ENOMEM; 5389 5390 values[0] = 1 + leader->nr_siblings; 5391 5392 /* 5393 * By locking the child_mutex of the leader we effectively 5394 * lock the child list of all siblings.. XXX explain how. 5395 */ 5396 mutex_lock(&leader->child_mutex); 5397 5398 ret = __perf_read_group_add(leader, read_format, values); 5399 if (ret) 5400 goto unlock; 5401 5402 list_for_each_entry(child, &leader->child_list, child_list) { 5403 ret = __perf_read_group_add(child, read_format, values); 5404 if (ret) 5405 goto unlock; 5406 } 5407 5408 mutex_unlock(&leader->child_mutex); 5409 5410 ret = event->read_size; 5411 if (copy_to_user(buf, values, event->read_size)) 5412 ret = -EFAULT; 5413 goto out; 5414 5415 unlock: 5416 mutex_unlock(&leader->child_mutex); 5417 out: 5418 kfree(values); 5419 return ret; 5420 } 5421 5422 static int perf_read_one(struct perf_event *event, 5423 u64 read_format, char __user *buf) 5424 { 5425 u64 enabled, running; 5426 u64 values[4]; 5427 int n = 0; 5428 5429 values[n++] = __perf_event_read_value(event, &enabled, &running); 5430 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 5431 values[n++] = enabled; 5432 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 5433 values[n++] = running; 5434 if (read_format & PERF_FORMAT_ID) 5435 values[n++] = primary_event_id(event); 5436 5437 if (copy_to_user(buf, values, n * sizeof(u64))) 5438 return -EFAULT; 5439 5440 return n * sizeof(u64); 5441 } 5442 5443 static bool is_event_hup(struct perf_event *event) 5444 { 5445 bool no_children; 5446 5447 if (event->state > PERF_EVENT_STATE_EXIT) 5448 return false; 5449 5450 mutex_lock(&event->child_mutex); 5451 no_children = list_empty(&event->child_list); 5452 mutex_unlock(&event->child_mutex); 5453 return no_children; 5454 } 5455 5456 /* 5457 * Read the performance event - simple non blocking version for now 5458 */ 5459 static ssize_t 5460 __perf_read(struct perf_event *event, char __user *buf, size_t count) 5461 { 5462 u64 read_format = event->attr.read_format; 5463 int ret; 5464 5465 /* 5466 * Return end-of-file for a read on an event that is in 5467 * error state (i.e. because it was pinned but it couldn't be 5468 * scheduled on to the CPU at some point). 5469 */ 5470 if (event->state == PERF_EVENT_STATE_ERROR) 5471 return 0; 5472 5473 if (count < event->read_size) 5474 return -ENOSPC; 5475 5476 WARN_ON_ONCE(event->ctx->parent_ctx); 5477 if (read_format & PERF_FORMAT_GROUP) 5478 ret = perf_read_group(event, read_format, buf); 5479 else 5480 ret = perf_read_one(event, read_format, buf); 5481 5482 return ret; 5483 } 5484 5485 static ssize_t 5486 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) 5487 { 5488 struct perf_event *event = file->private_data; 5489 struct perf_event_context *ctx; 5490 int ret; 5491 5492 ret = security_perf_event_read(event); 5493 if (ret) 5494 return ret; 5495 5496 ctx = perf_event_ctx_lock(event); 5497 ret = __perf_read(event, buf, count); 5498 perf_event_ctx_unlock(event, ctx); 5499 5500 return ret; 5501 } 5502 5503 static __poll_t perf_poll(struct file *file, poll_table *wait) 5504 { 5505 struct perf_event *event = file->private_data; 5506 struct perf_buffer *rb; 5507 __poll_t events = EPOLLHUP; 5508 5509 poll_wait(file, &event->waitq, wait); 5510 5511 if (is_event_hup(event)) 5512 return events; 5513 5514 /* 5515 * Pin the event->rb by taking event->mmap_mutex; otherwise 5516 * perf_event_set_output() can swizzle our rb and make us miss wakeups. 5517 */ 5518 mutex_lock(&event->mmap_mutex); 5519 rb = event->rb; 5520 if (rb) 5521 events = atomic_xchg(&rb->poll, 0); 5522 mutex_unlock(&event->mmap_mutex); 5523 return events; 5524 } 5525 5526 static void _perf_event_reset(struct perf_event *event) 5527 { 5528 (void)perf_event_read(event, false); 5529 local64_set(&event->count, 0); 5530 perf_event_update_userpage(event); 5531 } 5532 5533 /* Assume it's not an event with inherit set. */ 5534 u64 perf_event_pause(struct perf_event *event, bool reset) 5535 { 5536 struct perf_event_context *ctx; 5537 u64 count; 5538 5539 ctx = perf_event_ctx_lock(event); 5540 WARN_ON_ONCE(event->attr.inherit); 5541 _perf_event_disable(event); 5542 count = local64_read(&event->count); 5543 if (reset) 5544 local64_set(&event->count, 0); 5545 perf_event_ctx_unlock(event, ctx); 5546 5547 return count; 5548 } 5549 EXPORT_SYMBOL_GPL(perf_event_pause); 5550 5551 /* 5552 * Holding the top-level event's child_mutex means that any 5553 * descendant process that has inherited this event will block 5554 * in perf_event_exit_event() if it goes to exit, thus satisfying the 5555 * task existence requirements of perf_event_enable/disable. 5556 */ 5557 static void perf_event_for_each_child(struct perf_event *event, 5558 void (*func)(struct perf_event *)) 5559 { 5560 struct perf_event *child; 5561 5562 WARN_ON_ONCE(event->ctx->parent_ctx); 5563 5564 mutex_lock(&event->child_mutex); 5565 func(event); 5566 list_for_each_entry(child, &event->child_list, child_list) 5567 func(child); 5568 mutex_unlock(&event->child_mutex); 5569 } 5570 5571 static void perf_event_for_each(struct perf_event *event, 5572 void (*func)(struct perf_event *)) 5573 { 5574 struct perf_event_context *ctx = event->ctx; 5575 struct perf_event *sibling; 5576 5577 lockdep_assert_held(&ctx->mutex); 5578 5579 event = event->group_leader; 5580 5581 perf_event_for_each_child(event, func); 5582 for_each_sibling_event(sibling, event) 5583 perf_event_for_each_child(sibling, func); 5584 } 5585 5586 static void __perf_event_period(struct perf_event *event, 5587 struct perf_cpu_context *cpuctx, 5588 struct perf_event_context *ctx, 5589 void *info) 5590 { 5591 u64 value = *((u64 *)info); 5592 bool active; 5593 5594 if (event->attr.freq) { 5595 event->attr.sample_freq = value; 5596 } else { 5597 event->attr.sample_period = value; 5598 event->hw.sample_period = value; 5599 } 5600 5601 active = (event->state == PERF_EVENT_STATE_ACTIVE); 5602 if (active) { 5603 perf_pmu_disable(ctx->pmu); 5604 /* 5605 * We could be throttled; unthrottle now to avoid the tick 5606 * trying to unthrottle while we already re-started the event. 5607 */ 5608 if (event->hw.interrupts == MAX_INTERRUPTS) { 5609 event->hw.interrupts = 0; 5610 perf_log_throttle(event, 1); 5611 } 5612 event->pmu->stop(event, PERF_EF_UPDATE); 5613 } 5614 5615 local64_set(&event->hw.period_left, 0); 5616 5617 if (active) { 5618 event->pmu->start(event, PERF_EF_RELOAD); 5619 perf_pmu_enable(ctx->pmu); 5620 } 5621 } 5622 5623 static int perf_event_check_period(struct perf_event *event, u64 value) 5624 { 5625 return event->pmu->check_period(event, value); 5626 } 5627 5628 static int _perf_event_period(struct perf_event *event, u64 value) 5629 { 5630 if (!is_sampling_event(event)) 5631 return -EINVAL; 5632 5633 if (!value) 5634 return -EINVAL; 5635 5636 if (event->attr.freq && value > sysctl_perf_event_sample_rate) 5637 return -EINVAL; 5638 5639 if (perf_event_check_period(event, value)) 5640 return -EINVAL; 5641 5642 if (!event->attr.freq && (value & (1ULL << 63))) 5643 return -EINVAL; 5644 5645 event_function_call(event, __perf_event_period, &value); 5646 5647 return 0; 5648 } 5649 5650 int perf_event_period(struct perf_event *event, u64 value) 5651 { 5652 struct perf_event_context *ctx; 5653 int ret; 5654 5655 ctx = perf_event_ctx_lock(event); 5656 ret = _perf_event_period(event, value); 5657 perf_event_ctx_unlock(event, ctx); 5658 5659 return ret; 5660 } 5661 EXPORT_SYMBOL_GPL(perf_event_period); 5662 5663 static const struct file_operations perf_fops; 5664 5665 static inline int perf_fget_light(int fd, struct fd *p) 5666 { 5667 struct fd f = fdget(fd); 5668 if (!f.file) 5669 return -EBADF; 5670 5671 if (f.file->f_op != &perf_fops) { 5672 fdput(f); 5673 return -EBADF; 5674 } 5675 *p = f; 5676 return 0; 5677 } 5678 5679 static int perf_event_set_output(struct perf_event *event, 5680 struct perf_event *output_event); 5681 static int perf_event_set_filter(struct perf_event *event, void __user *arg); 5682 static int perf_copy_attr(struct perf_event_attr __user *uattr, 5683 struct perf_event_attr *attr); 5684 5685 static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg) 5686 { 5687 void (*func)(struct perf_event *); 5688 u32 flags = arg; 5689 5690 switch (cmd) { 5691 case PERF_EVENT_IOC_ENABLE: 5692 func = _perf_event_enable; 5693 break; 5694 case PERF_EVENT_IOC_DISABLE: 5695 func = _perf_event_disable; 5696 break; 5697 case PERF_EVENT_IOC_RESET: 5698 func = _perf_event_reset; 5699 break; 5700 5701 case PERF_EVENT_IOC_REFRESH: 5702 return _perf_event_refresh(event, arg); 5703 5704 case PERF_EVENT_IOC_PERIOD: 5705 { 5706 u64 value; 5707 5708 if (copy_from_user(&value, (u64 __user *)arg, sizeof(value))) 5709 return -EFAULT; 5710 5711 return _perf_event_period(event, value); 5712 } 5713 case PERF_EVENT_IOC_ID: 5714 { 5715 u64 id = primary_event_id(event); 5716 5717 if (copy_to_user((void __user *)arg, &id, sizeof(id))) 5718 return -EFAULT; 5719 return 0; 5720 } 5721 5722 case PERF_EVENT_IOC_SET_OUTPUT: 5723 { 5724 int ret; 5725 if (arg != -1) { 5726 struct perf_event *output_event; 5727 struct fd output; 5728 ret = perf_fget_light(arg, &output); 5729 if (ret) 5730 return ret; 5731 output_event = output.file->private_data; 5732 ret = perf_event_set_output(event, output_event); 5733 fdput(output); 5734 } else { 5735 ret = perf_event_set_output(event, NULL); 5736 } 5737 return ret; 5738 } 5739 5740 case PERF_EVENT_IOC_SET_FILTER: 5741 return perf_event_set_filter(event, (void __user *)arg); 5742 5743 case PERF_EVENT_IOC_SET_BPF: 5744 { 5745 struct bpf_prog *prog; 5746 int err; 5747 5748 prog = bpf_prog_get(arg); 5749 if (IS_ERR(prog)) 5750 return PTR_ERR(prog); 5751 5752 err = perf_event_set_bpf_prog(event, prog, 0); 5753 if (err) { 5754 bpf_prog_put(prog); 5755 return err; 5756 } 5757 5758 return 0; 5759 } 5760 5761 case PERF_EVENT_IOC_PAUSE_OUTPUT: { 5762 struct perf_buffer *rb; 5763 5764 rcu_read_lock(); 5765 rb = rcu_dereference(event->rb); 5766 if (!rb || !rb->nr_pages) { 5767 rcu_read_unlock(); 5768 return -EINVAL; 5769 } 5770 rb_toggle_paused(rb, !!arg); 5771 rcu_read_unlock(); 5772 return 0; 5773 } 5774 5775 case PERF_EVENT_IOC_QUERY_BPF: 5776 return perf_event_query_prog_array(event, (void __user *)arg); 5777 5778 case PERF_EVENT_IOC_MODIFY_ATTRIBUTES: { 5779 struct perf_event_attr new_attr; 5780 int err = perf_copy_attr((struct perf_event_attr __user *)arg, 5781 &new_attr); 5782 5783 if (err) 5784 return err; 5785 5786 return perf_event_modify_attr(event, &new_attr); 5787 } 5788 default: 5789 return -ENOTTY; 5790 } 5791 5792 if (flags & PERF_IOC_FLAG_GROUP) 5793 perf_event_for_each(event, func); 5794 else 5795 perf_event_for_each_child(event, func); 5796 5797 return 0; 5798 } 5799 5800 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 5801 { 5802 struct perf_event *event = file->private_data; 5803 struct perf_event_context *ctx; 5804 long ret; 5805 5806 /* Treat ioctl like writes as it is likely a mutating operation. */ 5807 ret = security_perf_event_write(event); 5808 if (ret) 5809 return ret; 5810 5811 ctx = perf_event_ctx_lock(event); 5812 ret = _perf_ioctl(event, cmd, arg); 5813 perf_event_ctx_unlock(event, ctx); 5814 5815 return ret; 5816 } 5817 5818 #ifdef CONFIG_COMPAT 5819 static long perf_compat_ioctl(struct file *file, unsigned int cmd, 5820 unsigned long arg) 5821 { 5822 switch (_IOC_NR(cmd)) { 5823 case _IOC_NR(PERF_EVENT_IOC_SET_FILTER): 5824 case _IOC_NR(PERF_EVENT_IOC_ID): 5825 case _IOC_NR(PERF_EVENT_IOC_QUERY_BPF): 5826 case _IOC_NR(PERF_EVENT_IOC_MODIFY_ATTRIBUTES): 5827 /* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */ 5828 if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) { 5829 cmd &= ~IOCSIZE_MASK; 5830 cmd |= sizeof(void *) << IOCSIZE_SHIFT; 5831 } 5832 break; 5833 } 5834 return perf_ioctl(file, cmd, arg); 5835 } 5836 #else 5837 # define perf_compat_ioctl NULL 5838 #endif 5839 5840 int perf_event_task_enable(void) 5841 { 5842 struct perf_event_context *ctx; 5843 struct perf_event *event; 5844 5845 mutex_lock(¤t->perf_event_mutex); 5846 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { 5847 ctx = perf_event_ctx_lock(event); 5848 perf_event_for_each_child(event, _perf_event_enable); 5849 perf_event_ctx_unlock(event, ctx); 5850 } 5851 mutex_unlock(¤t->perf_event_mutex); 5852 5853 return 0; 5854 } 5855 5856 int perf_event_task_disable(void) 5857 { 5858 struct perf_event_context *ctx; 5859 struct perf_event *event; 5860 5861 mutex_lock(¤t->perf_event_mutex); 5862 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { 5863 ctx = perf_event_ctx_lock(event); 5864 perf_event_for_each_child(event, _perf_event_disable); 5865 perf_event_ctx_unlock(event, ctx); 5866 } 5867 mutex_unlock(¤t->perf_event_mutex); 5868 5869 return 0; 5870 } 5871 5872 static int perf_event_index(struct perf_event *event) 5873 { 5874 if (event->hw.state & PERF_HES_STOPPED) 5875 return 0;