~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/trace/trace_uprobe.c

Version: ~ [ linux-5.16 ] ~ [ linux-5.15.13 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.90 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.170 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.224 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.261 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.296 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.298 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * uprobes-based tracing events
  4  *
  5  * Copyright (C) IBM Corporation, 2010-2012
  6  * Author:      Srikar Dronamraju <srikar@linux.vnet.ibm.com>
  7  */
  8 #define pr_fmt(fmt)     "trace_uprobe: " fmt
  9 
 10 #include <linux/security.h>
 11 #include <linux/ctype.h>
 12 #include <linux/module.h>
 13 #include <linux/uaccess.h>
 14 #include <linux/uprobes.h>
 15 #include <linux/namei.h>
 16 #include <linux/string.h>
 17 #include <linux/rculist.h>
 18 
 19 #include "trace_dynevent.h"
 20 #include "trace_probe.h"
 21 #include "trace_probe_tmpl.h"
 22 
 23 #define UPROBE_EVENT_SYSTEM     "uprobes"
 24 
 25 struct uprobe_trace_entry_head {
 26         struct trace_entry      ent;
 27         unsigned long           vaddr[];
 28 };
 29 
 30 #define SIZEOF_TRACE_ENTRY(is_return)                   \
 31         (sizeof(struct uprobe_trace_entry_head) +       \
 32          sizeof(unsigned long) * (is_return ? 2 : 1))
 33 
 34 #define DATAOF_TRACE_ENTRY(entry, is_return)            \
 35         ((void*)(entry) + SIZEOF_TRACE_ENTRY(is_return))
 36 
 37 static int trace_uprobe_create(int argc, const char **argv);
 38 static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev);
 39 static int trace_uprobe_release(struct dyn_event *ev);
 40 static bool trace_uprobe_is_busy(struct dyn_event *ev);
 41 static bool trace_uprobe_match(const char *system, const char *event,
 42                         int argc, const char **argv, struct dyn_event *ev);
 43 
 44 static struct dyn_event_operations trace_uprobe_ops = {
 45         .create = trace_uprobe_create,
 46         .show = trace_uprobe_show,
 47         .is_busy = trace_uprobe_is_busy,
 48         .free = trace_uprobe_release,
 49         .match = trace_uprobe_match,
 50 };
 51 
 52 /*
 53  * uprobe event core functions
 54  */
 55 struct trace_uprobe {
 56         struct dyn_event                devent;
 57         struct uprobe_consumer          consumer;
 58         struct path                     path;
 59         struct inode                    *inode;
 60         char                            *filename;
 61         unsigned long                   offset;
 62         unsigned long                   ref_ctr_offset;
 63         unsigned long                   nhit;
 64         struct trace_probe              tp;
 65 };
 66 
 67 static bool is_trace_uprobe(struct dyn_event *ev)
 68 {
 69         return ev->ops == &trace_uprobe_ops;
 70 }
 71 
 72 static struct trace_uprobe *to_trace_uprobe(struct dyn_event *ev)
 73 {
 74         return container_of(ev, struct trace_uprobe, devent);
 75 }
 76 
 77 /**
 78  * for_each_trace_uprobe - iterate over the trace_uprobe list
 79  * @pos:        the struct trace_uprobe * for each entry
 80  * @dpos:       the struct dyn_event * to use as a loop cursor
 81  */
 82 #define for_each_trace_uprobe(pos, dpos)        \
 83         for_each_dyn_event(dpos)                \
 84                 if (is_trace_uprobe(dpos) && (pos = to_trace_uprobe(dpos)))
 85 
 86 #define SIZEOF_TRACE_UPROBE(n)                          \
 87         (offsetof(struct trace_uprobe, tp.args) +       \
 88         (sizeof(struct probe_arg) * (n)))
 89 
 90 static int register_uprobe_event(struct trace_uprobe *tu);
 91 static int unregister_uprobe_event(struct trace_uprobe *tu);
 92 
 93 struct uprobe_dispatch_data {
 94         struct trace_uprobe     *tu;
 95         unsigned long           bp_addr;
 96 };
 97 
 98 static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs);
 99 static int uretprobe_dispatcher(struct uprobe_consumer *con,
100                                 unsigned long func, struct pt_regs *regs);
101 
102 #ifdef CONFIG_STACK_GROWSUP
103 static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n)
104 {
105         return addr - (n * sizeof(long));
106 }
107 #else
108 static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n)
109 {
110         return addr + (n * sizeof(long));
111 }
112 #endif
113 
114 static unsigned long get_user_stack_nth(struct pt_regs *regs, unsigned int n)
115 {
116         unsigned long ret;
117         unsigned long addr = user_stack_pointer(regs);
118 
119         addr = adjust_stack_addr(addr, n);
120 
121         if (copy_from_user(&ret, (void __force __user *) addr, sizeof(ret)))
122                 return 0;
123 
124         return ret;
125 }
126 
127 /*
128  * Uprobes-specific fetch functions
129  */
130 static nokprobe_inline int
131 probe_mem_read(void *dest, void *src, size_t size)
132 {
133         void __user *vaddr = (void __force __user *)src;
134 
135         return copy_from_user(dest, vaddr, size) ? -EFAULT : 0;
136 }
137 
138 static nokprobe_inline int
139 probe_mem_read_user(void *dest, void *src, size_t size)
140 {
141         return probe_mem_read(dest, src, size);
142 }
143 
144 /*
145  * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
146  * length and relative data location.
147  */
148 static nokprobe_inline int
149 fetch_store_string(unsigned long addr, void *dest, void *base)
150 {
151         long ret;
152         u32 loc = *(u32 *)dest;
153         int maxlen  = get_loc_len(loc);
154         u8 *dst = get_loc_data(dest, base);
155         void __user *src = (void __force __user *) addr;
156 
157         if (unlikely(!maxlen))
158                 return -ENOMEM;
159 
160         if (addr == FETCH_TOKEN_COMM)
161                 ret = strlcpy(dst, current->comm, maxlen);
162         else
163                 ret = strncpy_from_user(dst, src, maxlen);
164         if (ret >= 0) {
165                 if (ret == maxlen)
166                         dst[ret - 1] = '\0';
167                 else
168                         /*
169                          * Include the terminating null byte. In this case it
170                          * was copied by strncpy_from_user but not accounted
171                          * for in ret.
172                          */
173                         ret++;
174                 *(u32 *)dest = make_data_loc(ret, (void *)dst - base);
175         }
176 
177         return ret;
178 }
179 
180 static nokprobe_inline int
181 fetch_store_string_user(unsigned long addr, void *dest, void *base)
182 {
183         return fetch_store_string(addr, dest, base);
184 }
185 
186 /* Return the length of string -- including null terminal byte */
187 static nokprobe_inline int
188 fetch_store_strlen(unsigned long addr)
189 {
190         int len;
191         void __user *vaddr = (void __force __user *) addr;
192 
193         if (addr == FETCH_TOKEN_COMM)
194                 len = strlen(current->comm) + 1;
195         else
196                 len = strnlen_user(vaddr, MAX_STRING_SIZE);
197 
198         return (len > MAX_STRING_SIZE) ? 0 : len;
199 }
200 
201 static nokprobe_inline int
202 fetch_store_strlen_user(unsigned long addr)
203 {
204         return fetch_store_strlen(addr);
205 }
206 
207 static unsigned long translate_user_vaddr(unsigned long file_offset)
208 {
209         unsigned long base_addr;
210         struct uprobe_dispatch_data *udd;
211 
212         udd = (void *) current->utask->vaddr;
213 
214         base_addr = udd->bp_addr - udd->tu->offset;
215         return base_addr + file_offset;
216 }
217 
218 /* Note that we don't verify it, since the code does not come from user space */
219 static int
220 process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs, void *dest,
221                    void *base)
222 {
223         unsigned long val;
224 
225         /* 1st stage: get value from context */
226         switch (code->op) {
227         case FETCH_OP_REG:
228                 val = regs_get_register(regs, code->param);
229                 break;
230         case FETCH_OP_STACK:
231                 val = get_user_stack_nth(regs, code->param);
232                 break;
233         case FETCH_OP_STACKP:
234                 val = user_stack_pointer(regs);
235                 break;
236         case FETCH_OP_RETVAL:
237                 val = regs_return_value(regs);
238                 break;
239         case FETCH_OP_IMM:
240                 val = code->immediate;
241                 break;
242         case FETCH_OP_COMM:
243                 val = FETCH_TOKEN_COMM;
244                 break;
245         case FETCH_OP_DATA:
246                 val = (unsigned long)code->data;
247                 break;
248         case FETCH_OP_FOFFS:
249                 val = translate_user_vaddr(code->immediate);
250                 break;
251         default:
252                 return -EILSEQ;
253         }
254         code++;
255 
256         return process_fetch_insn_bottom(code, val, dest, base);
257 }
258 NOKPROBE_SYMBOL(process_fetch_insn)
259 
260 static inline void init_trace_uprobe_filter(struct trace_uprobe_filter *filter)
261 {
262         rwlock_init(&filter->rwlock);
263         filter->nr_systemwide = 0;
264         INIT_LIST_HEAD(&filter->perf_events);
265 }
266 
267 static inline bool uprobe_filter_is_empty(struct trace_uprobe_filter *filter)
268 {
269         return !filter->nr_systemwide && list_empty(&filter->perf_events);
270 }
271 
272 static inline bool is_ret_probe(struct trace_uprobe *tu)
273 {
274         return tu->consumer.ret_handler != NULL;
275 }
276 
277 static bool trace_uprobe_is_busy(struct dyn_event *ev)
278 {
279         struct trace_uprobe *tu = to_trace_uprobe(ev);
280 
281         return trace_probe_is_enabled(&tu->tp);
282 }
283 
284 static bool trace_uprobe_match_command_head(struct trace_uprobe *tu,
285                                             int argc, const char **argv)
286 {
287         char buf[MAX_ARGSTR_LEN + 1];
288         int len;
289 
290         if (!argc)
291                 return true;
292 
293         len = strlen(tu->filename);
294         if (strncmp(tu->filename, argv[0], len) || argv[0][len] != ':')
295                 return false;
296 
297         if (tu->ref_ctr_offset == 0)
298                 snprintf(buf, sizeof(buf), "0x%0*lx",
299                                 (int)(sizeof(void *) * 2), tu->offset);
300         else
301                 snprintf(buf, sizeof(buf), "0x%0*lx(0x%lx)",
302                                 (int)(sizeof(void *) * 2), tu->offset,
303                                 tu->ref_ctr_offset);
304         if (strcmp(buf, &argv[0][len + 1]))
305                 return false;
306 
307         argc--; argv++;
308 
309         return trace_probe_match_command_args(&tu->tp, argc, argv);
310 }
311 
312 static bool trace_uprobe_match(const char *system, const char *event,
313                         int argc, const char **argv, struct dyn_event *ev)
314 {
315         struct trace_uprobe *tu = to_trace_uprobe(ev);
316 
317         return strcmp(trace_probe_name(&tu->tp), event) == 0 &&
318            (!system || strcmp(trace_probe_group_name(&tu->tp), system) == 0) &&
319            trace_uprobe_match_command_head(tu, argc, argv);
320 }
321 
322 static nokprobe_inline struct trace_uprobe *
323 trace_uprobe_primary_from_call(struct trace_event_call *call)
324 {
325         struct trace_probe *tp;
326 
327         tp = trace_probe_primary_from_call(call);
328         if (WARN_ON_ONCE(!tp))
329                 return NULL;
330 
331         return container_of(tp, struct trace_uprobe, tp);
332 }
333 
334 /*
335  * Allocate new trace_uprobe and initialize it (including uprobes).
336  */
337 static struct trace_uprobe *
338 alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret)
339 {
340         struct trace_uprobe *tu;
341         int ret;
342 
343         tu = kzalloc(SIZEOF_TRACE_UPROBE(nargs), GFP_KERNEL);
344         if (!tu)
345                 return ERR_PTR(-ENOMEM);
346 
347         ret = trace_probe_init(&tu->tp, event, group, true);
348         if (ret < 0)
349                 goto error;
350 
351         dyn_event_init(&tu->devent, &trace_uprobe_ops);
352         tu->consumer.handler = uprobe_dispatcher;
353         if (is_ret)
354                 tu->consumer.ret_handler = uretprobe_dispatcher;
355         init_trace_uprobe_filter(tu->tp.event->filter);
356         return tu;
357 
358 error:
359         kfree(tu);
360 
361         return ERR_PTR(ret);
362 }
363 
364 static void free_trace_uprobe(struct trace_uprobe *tu)
365 {
366         if (!tu)
367                 return;
368 
369         path_put(&tu->path);
370         trace_probe_cleanup(&tu->tp);
371         kfree(tu->filename);
372         kfree(tu);
373 }
374 
375 static struct trace_uprobe *find_probe_event(const char *event, const char *group)
376 {
377         struct dyn_event *pos;
378         struct trace_uprobe *tu;
379 
380         for_each_trace_uprobe(tu, pos)
381                 if (strcmp(trace_probe_name(&tu->tp), event) == 0 &&
382                     strcmp(trace_probe_group_name(&tu->tp), group) == 0)
383                         return tu;
384 
385         return NULL;
386 }
387 
388 /* Unregister a trace_uprobe and probe_event */
389 static int unregister_trace_uprobe(struct trace_uprobe *tu)
390 {
391         int ret;
392 
393         if (trace_probe_has_sibling(&tu->tp))
394                 goto unreg;
395 
396         ret = unregister_uprobe_event(tu);
397         if (ret)
398                 return ret;
399 
400 unreg:
401         dyn_event_remove(&tu->devent);
402         trace_probe_unlink(&tu->tp);
403         free_trace_uprobe(tu);
404         return 0;
405 }
406 
407 static bool trace_uprobe_has_same_uprobe(struct trace_uprobe *orig,
408                                          struct trace_uprobe *comp)
409 {
410         struct trace_probe_event *tpe = orig->tp.event;
411         struct trace_probe *pos;
412         struct inode *comp_inode = d_real_inode(comp->path.dentry);
413         int i;
414 
415         list_for_each_entry(pos, &tpe->probes, list) {
416                 orig = container_of(pos, struct trace_uprobe, tp);
417                 if (comp_inode != d_real_inode(orig->path.dentry) ||
418                     comp->offset != orig->offset)
419                         continue;
420 
421                 /*
422                  * trace_probe_compare_arg_type() ensured that nr_args and
423                  * each argument name and type are same. Let's compare comm.
424                  */
425                 for (i = 0; i < orig->tp.nr_args; i++) {
426                         if (strcmp(orig->tp.args[i].comm,
427                                    comp->tp.args[i].comm))
428                                 break;
429                 }
430 
431                 if (i == orig->tp.nr_args)
432                         return true;
433         }
434 
435         return false;
436 }
437 
438 static int append_trace_uprobe(struct trace_uprobe *tu, struct trace_uprobe *to)
439 {
440         int ret;
441 
442         ret = trace_probe_compare_arg_type(&tu->tp, &to->tp);
443         if (ret) {
444                 /* Note that argument starts index = 2 */
445                 trace_probe_log_set_index(ret + 1);
446                 trace_probe_log_err(0, DIFF_ARG_TYPE);
447                 return -EEXIST;
448         }
449         if (trace_uprobe_has_same_uprobe(to, tu)) {
450                 trace_probe_log_set_index(0);
451                 trace_probe_log_err(0, SAME_PROBE);
452                 return -EEXIST;
453         }
454 
455         /* Append to existing event */
456         ret = trace_probe_append(&tu->tp, &to->tp);
457         if (!ret)
458                 dyn_event_add(&tu->devent);
459 
460         return ret;
461 }
462 
463 /*
464  * Uprobe with multiple reference counter is not allowed. i.e.
465  * If inode and offset matches, reference counter offset *must*
466  * match as well. Though, there is one exception: If user is
467  * replacing old trace_uprobe with new one(same group/event),
468  * then we allow same uprobe with new reference counter as far
469  * as the new one does not conflict with any other existing
470  * ones.
471  */
472 static int validate_ref_ctr_offset(struct trace_uprobe *new)
473 {
474         struct dyn_event *pos;
475         struct trace_uprobe *tmp;
476         struct inode *new_inode = d_real_inode(new->path.dentry);
477 
478         for_each_trace_uprobe(tmp, pos) {
479                 if (new_inode == d_real_inode(tmp->path.dentry) &&
480                     new->offset == tmp->offset &&
481                     new->ref_ctr_offset != tmp->ref_ctr_offset) {
482                         pr_warn("Reference counter offset mismatch.");
483                         return -EINVAL;
484                 }
485         }
486         return 0;
487 }
488 
489 /* Register a trace_uprobe and probe_event */
490 static int register_trace_uprobe(struct trace_uprobe *tu)
491 {
492         struct trace_uprobe *old_tu;
493         int ret;
494 
495         mutex_lock(&event_mutex);
496 
497         ret = validate_ref_ctr_offset(tu);
498         if (ret)
499                 goto end;
500 
501         /* register as an event */
502         old_tu = find_probe_event(trace_probe_name(&tu->tp),
503                                   trace_probe_group_name(&tu->tp));
504         if (old_tu) {
505                 if (is_ret_probe(tu) != is_ret_probe(old_tu)) {
506                         trace_probe_log_set_index(0);
507                         trace_probe_log_err(0, DIFF_PROBE_TYPE);
508                         ret = -EEXIST;
509                 } else {
510                         ret = append_trace_uprobe(tu, old_tu);
511                 }
512                 goto end;
513         }
514 
515         ret = register_uprobe_event(tu);
516         if (ret) {
517                 pr_warn("Failed to register probe event(%d)\n", ret);
518                 goto end;
519         }
520 
521         dyn_event_add(&tu->devent);
522 
523 end:
524         mutex_unlock(&event_mutex);
525 
526         return ret;
527 }
528 
529 /*
530  * Argument syntax:
531  *  - Add uprobe: p|r[:[GRP/]EVENT] PATH:OFFSET [FETCHARGS]
532  */
533 static int trace_uprobe_create(int argc, const char **argv)
534 {
535         struct trace_uprobe *tu;
536         const char *event = NULL, *group = UPROBE_EVENT_SYSTEM;
537         char *arg, *filename, *rctr, *rctr_end, *tmp;
538         char buf[MAX_EVENT_NAME_LEN];
539         struct path path;
540         unsigned long offset, ref_ctr_offset;
541         bool is_return = false;
542         int i, ret;
543 
544         ret = 0;
545         ref_ctr_offset = 0;
546 
547         switch (argv[0][0]) {
548         case 'r':
549                 is_return = true;
550                 break;
551         case 'p':
552                 break;
553         default:
554                 return -ECANCELED;
555         }
556 
557         if (argc < 2)
558                 return -ECANCELED;
559 
560         if (argv[0][1] == ':')
561                 event = &argv[0][2];
562 
563         if (!strchr(argv[1], '/'))
564                 return -ECANCELED;
565 
566         filename = kstrdup(argv[1], GFP_KERNEL);
567         if (!filename)
568                 return -ENOMEM;
569 
570         /* Find the last occurrence, in case the path contains ':' too. */
571         arg = strrchr(filename, ':');
572         if (!arg || !isdigit(arg[1])) {
573                 kfree(filename);
574                 return -ECANCELED;
575         }
576 
577         trace_probe_log_init("trace_uprobe", argc, argv);
578         trace_probe_log_set_index(1);   /* filename is the 2nd argument */
579 
580         *arg++ = '\0';
581         ret = kern_path(filename, LOOKUP_FOLLOW, &path);
582         if (ret) {
583                 trace_probe_log_err(0, FILE_NOT_FOUND);
584                 kfree(filename);
585                 trace_probe_log_clear();
586                 return ret;
587         }
588         if (!d_is_reg(path.dentry)) {
589                 trace_probe_log_err(0, NO_REGULAR_FILE);
590                 ret = -EINVAL;
591                 goto fail_address_parse;
592         }
593 
594         /* Parse reference counter offset if specified. */
595         rctr = strchr(arg, '(');
596         if (rctr) {
597                 rctr_end = strchr(rctr, ')');
598                 if (!rctr_end) {
599                         ret = -EINVAL;
600                         rctr_end = rctr + strlen(rctr);
601                         trace_probe_log_err(rctr_end - filename,
602                                             REFCNT_OPEN_BRACE);
603                         goto fail_address_parse;
604                 } else if (rctr_end[1] != '\0') {
605                         ret = -EINVAL;
606                         trace_probe_log_err(rctr_end + 1 - filename,
607                                             BAD_REFCNT_SUFFIX);
608                         goto fail_address_parse;
609                 }
610 
611                 *rctr++ = '\0';
612                 *rctr_end = '\0';
613                 ret = kstrtoul(rctr, 0, &ref_ctr_offset);
614                 if (ret) {
615                         trace_probe_log_err(rctr - filename, BAD_REFCNT);
616                         goto fail_address_parse;
617                 }
618         }
619 
620         /* Parse uprobe offset. */
621         ret = kstrtoul(arg, 0, &offset);
622         if (ret) {
623                 trace_probe_log_err(arg - filename, BAD_UPROBE_OFFS);
624                 goto fail_address_parse;
625         }
626 
627         /* setup a probe */
628         trace_probe_log_set_index(0);
629         if (event) {
630                 ret = traceprobe_parse_event_name(&event, &group, buf,
631                                                   event - argv[0]);
632                 if (ret)
633                         goto fail_address_parse;
634         } else {
635                 char *tail;
636                 char *ptr;
637 
638                 tail = kstrdup(kbasename(filename), GFP_KERNEL);
639                 if (!tail) {
640                         ret = -ENOMEM;
641                         goto fail_address_parse;
642                 }
643 
644                 ptr = strpbrk(tail, ".-_");
645                 if (ptr)
646                         *ptr = '\0';
647 
648                 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_0x%lx", 'p', tail, offset);
649                 event = buf;
650                 kfree(tail);
651         }
652 
653         argc -= 2;
654         argv += 2;
655 
656         tu = alloc_trace_uprobe(group, event, argc, is_return);
657         if (IS_ERR(tu)) {
658                 ret = PTR_ERR(tu);
659                 /* This must return -ENOMEM otherwise there is a bug */
660                 WARN_ON_ONCE(ret != -ENOMEM);
661                 goto fail_address_parse;
662         }
663         tu->offset = offset;
664         tu->ref_ctr_offset = ref_ctr_offset;
665         tu->path = path;
666         tu->filename = filename;
667 
668         /* parse arguments */
669         for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
670                 tmp = kstrdup(argv[i], GFP_KERNEL);
671                 if (!tmp) {
672                         ret = -ENOMEM;
673                         goto error;
674                 }
675 
676                 trace_probe_log_set_index(i + 2);
677                 ret = traceprobe_parse_probe_arg(&tu->tp, i, tmp,
678                                         is_return ? TPARG_FL_RETURN : 0);
679                 kfree(tmp);
680                 if (ret)
681                         goto error;
682         }
683 
684         ret = traceprobe_set_print_fmt(&tu->tp, is_ret_probe(tu));
685         if (ret < 0)
686                 goto error;
687 
688         ret = register_trace_uprobe(tu);
689         if (!ret)
690                 goto out;
691 
692 error:
693         free_trace_uprobe(tu);
694 out:
695         trace_probe_log_clear();
696         return ret;
697 
698 fail_address_parse:
699         trace_probe_log_clear();
700         path_put(&path);
701         kfree(filename);
702 
703         return ret;
704 }
705 
706 static int create_or_delete_trace_uprobe(int argc, char **argv)
707 {
708         int ret;
709 
710         if (argv[0][0] == '-')
711                 return dyn_event_release(argc, argv, &trace_uprobe_ops);
712 
713         ret = trace_uprobe_create(argc, (const char **)argv);
714         return ret == -ECANCELED ? -EINVAL : ret;
715 }
716 
717 static int trace_uprobe_release(struct dyn_event *ev)
718 {
719         struct trace_uprobe *tu = to_trace_uprobe(ev);
720 
721         return unregister_trace_uprobe(tu);
722 }
723 
724 /* Probes listing interfaces */
725 static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev)
726 {
727         struct trace_uprobe *tu = to_trace_uprobe(ev);
728         char c = is_ret_probe(tu) ? 'r' : 'p';
729         int i;
730 
731         seq_printf(m, "%c:%s/%s %s:0x%0*lx", c, trace_probe_group_name(&tu->tp),
732                         trace_probe_name(&tu->tp), tu->filename,
733                         (int)(sizeof(void *) * 2), tu->offset);
734 
735         if (tu->ref_ctr_offset)
736                 seq_printf(m, "(0x%lx)", tu->ref_ctr_offset);
737 
738         for (i = 0; i < tu->tp.nr_args; i++)
739                 seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm);
740 
741         seq_putc(m, '\n');
742         return 0;
743 }
744 
745 static int probes_seq_show(struct seq_file *m, void *v)
746 {
747         struct dyn_event *ev = v;
748 
749         if (!is_trace_uprobe(ev))
750                 return 0;
751 
752         return trace_uprobe_show(m, ev);
753 }
754 
755 static const struct seq_operations probes_seq_op = {
756         .start  = dyn_event_seq_start,
757         .next   = dyn_event_seq_next,
758         .stop   = dyn_event_seq_stop,
759         .show   = probes_seq_show
760 };
761 
762 static int probes_open(struct inode *inode, struct file *file)
763 {
764         int ret;
765 
766         ret = security_locked_down(LOCKDOWN_TRACEFS);
767         if (ret)
768                 return ret;
769 
770         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
771                 ret = dyn_events_release_all(&trace_uprobe_ops);
772                 if (ret)
773                         return ret;
774         }
775 
776         return seq_open(file, &probes_seq_op);
777 }
778 
779 static ssize_t probes_write(struct file *file, const char __user *buffer,
780                             size_t count, loff_t *ppos)
781 {
782         return trace_parse_run_command(file, buffer, count, ppos,
783                                         create_or_delete_trace_uprobe);
784 }
785 
786 static const struct file_operations uprobe_events_ops = {
787         .owner          = THIS_MODULE,
788         .open           = probes_open,
789         .read           = seq_read,
790         .llseek         = seq_lseek,
791         .release        = seq_release,
792         .write          = probes_write,
793 };
794 
795 /* Probes profiling interfaces */
796 static int probes_profile_seq_show(struct seq_file *m, void *v)
797 {
798         struct dyn_event *ev = v;
799         struct trace_uprobe *tu;
800 
801         if (!is_trace_uprobe(ev))
802                 return 0;
803 
804         tu = to_trace_uprobe(ev);
805         seq_printf(m, "  %s %-44s %15lu\n", tu->filename,
806                         trace_probe_name(&tu->tp), tu->nhit);
807         return 0;
808 }
809 
810 static const struct seq_operations profile_seq_op = {
811         .start  = dyn_event_seq_start,
812         .next   = dyn_event_seq_next,
813         .stop   = dyn_event_seq_stop,
814         .show   = probes_profile_seq_show
815 };
816 
817 static int profile_open(struct inode *inode, struct file *file)
818 {
819         int ret;
820 
821         ret = security_locked_down(LOCKDOWN_TRACEFS);
822         if (ret)
823                 return ret;
824 
825         return seq_open(file, &profile_seq_op);
826 }
827 
828 static const struct file_operations uprobe_profile_ops = {
829         .owner          = THIS_MODULE,
830         .open           = profile_open,
831         .read           = seq_read,
832         .llseek         = seq_lseek,
833         .release        = seq_release,
834 };
835 
836 struct uprobe_cpu_buffer {
837         struct mutex mutex;
838         void *buf;
839 };
840 static struct uprobe_cpu_buffer __percpu *uprobe_cpu_buffer;
841 static int uprobe_buffer_refcnt;
842 
843 static int uprobe_buffer_init(void)
844 {
845         int cpu, err_cpu;
846 
847         uprobe_cpu_buffer = alloc_percpu(struct uprobe_cpu_buffer);
848         if (uprobe_cpu_buffer == NULL)
849                 return -ENOMEM;
850 
851         for_each_possible_cpu(cpu) {
852                 struct page *p = alloc_pages_node(cpu_to_node(cpu),
853                                                   GFP_KERNEL, 0);
854                 if (p == NULL) {
855                         err_cpu = cpu;
856                         goto err;
857                 }
858                 per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf = page_address(p);
859                 mutex_init(&per_cpu_ptr(uprobe_cpu_buffer, cpu)->mutex);
860         }
861 
862         return 0;
863 
864 err:
865         for_each_possible_cpu(cpu) {
866                 if (cpu == err_cpu)
867                         break;
868                 free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf);
869         }
870 
871         free_percpu(uprobe_cpu_buffer);
872         return -ENOMEM;
873 }
874 
875 static int uprobe_buffer_enable(void)
876 {
877         int ret = 0;
878 
879         BUG_ON(!mutex_is_locked(&event_mutex));
880 
881         if (uprobe_buffer_refcnt++ == 0) {
882                 ret = uprobe_buffer_init();
883                 if (ret < 0)
884                         uprobe_buffer_refcnt--;
885         }
886 
887         return ret;
888 }
889 
890 static void uprobe_buffer_disable(void)
891 {
892         int cpu;
893 
894         BUG_ON(!mutex_is_locked(&event_mutex));
895 
896         if (--uprobe_buffer_refcnt == 0) {
897                 for_each_possible_cpu(cpu)
898                         free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer,
899                                                              cpu)->buf);
900 
901                 free_percpu(uprobe_cpu_buffer);
902                 uprobe_cpu_buffer = NULL;
903         }
904 }
905 
906 static struct uprobe_cpu_buffer *uprobe_buffer_get(void)
907 {
908         struct uprobe_cpu_buffer *ucb;
909         int cpu;
910 
911         cpu = raw_smp_processor_id();
912         ucb = per_cpu_ptr(uprobe_cpu_buffer, cpu);
913 
914         /*
915          * Use per-cpu buffers for fastest access, but we might migrate
916          * so the mutex makes sure we have sole access to it.
917          */
918         mutex_lock(&ucb->mutex);
919 
920         return ucb;
921 }
922 
923 static void uprobe_buffer_put(struct uprobe_cpu_buffer *ucb)
924 {
925         mutex_unlock(&ucb->mutex);
926 }
927 
928 static void __uprobe_trace_func(struct trace_uprobe *tu,
929                                 unsigned long func, struct pt_regs *regs,
930                                 struct uprobe_cpu_buffer *ucb, int dsize,
931                                 struct trace_event_file *trace_file)
932 {
933         struct uprobe_trace_entry_head *entry;
934         struct trace_buffer *buffer;
935         struct ring_buffer_event *event;
936         void *data;
937         int size, esize;
938         struct trace_event_call *call = trace_probe_event_call(&tu->tp);
939 
940         WARN_ON(call != trace_file->event_call);
941 
942         if (WARN_ON_ONCE(tu->tp.size + dsize > PAGE_SIZE))
943                 return;
944 
945         if (trace_trigger_soft_disabled(trace_file))
946                 return;
947 
948         esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
949         size = esize + tu->tp.size + dsize;
950         event = trace_event_buffer_lock_reserve(&buffer, trace_file,
951                                                 call->event.type, size, 0, 0);
952         if (!event)
953                 return;
954 
955         entry = ring_buffer_event_data(event);
956         if (is_ret_probe(tu)) {
957                 entry->vaddr[0] = func;
958                 entry->vaddr[1] = instruction_pointer(regs);
959                 data = DATAOF_TRACE_ENTRY(entry, true);
960         } else {
961                 entry->vaddr[0] = instruction_pointer(regs);
962                 data = DATAOF_TRACE_ENTRY(entry, false);
963         }
964 
965         memcpy(data, ucb->buf, tu->tp.size + dsize);
966 
967         event_trigger_unlock_commit(trace_file, buffer, event, entry, 0, 0);
968 }
969 
970 /* uprobe handler */
971 static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs,
972                              struct uprobe_cpu_buffer *ucb, int dsize)
973 {
974         struct event_file_link *link;
975 
976         if (is_ret_probe(tu))
977                 return 0;
978 
979         rcu_read_lock();
980         trace_probe_for_each_link_rcu(link, &tu->tp)
981                 __uprobe_trace_func(tu, 0, regs, ucb, dsize, link->file);
982         rcu_read_unlock();
983 
984         return 0;
985 }
986 
987 static void uretprobe_trace_func(struct trace_uprobe *tu, unsigned long func,
988                                  struct pt_regs *regs,
989                                  struct uprobe_cpu_buffer *ucb, int dsize)
990 {
991         struct event_file_link *link;
992 
993         rcu_read_lock();
994         trace_probe_for_each_link_rcu(link, &tu->tp)
995                 __uprobe_trace_func(tu, func, regs, ucb, dsize, link->file);
996         rcu_read_unlock();
997 }
998 
999 /* Event entry printers */
1000 static enum print_line_t
1001 print_uprobe_event(struct trace_iterator *iter, int flags, struct trace_event *event)
1002 {
1003         struct uprobe_trace_entry_head *entry;
1004         struct trace_seq *s = &iter->seq;
1005         struct trace_uprobe *tu;
1006         u8 *data;
1007 
1008         entry = (struct uprobe_trace_entry_head *)iter->ent;
1009         tu = trace_uprobe_primary_from_call(
1010                 container_of(event, struct trace_event_call, event));
1011         if (unlikely(!tu))
1012                 goto out;
1013 
1014         if (is_ret_probe(tu)) {
1015                 trace_seq_printf(s, "%s: (0x%lx <- 0x%lx)",
1016                                  trace_probe_name(&tu->tp),
1017                                  entry->vaddr[1], entry->vaddr[0]);
1018                 data = DATAOF_TRACE_ENTRY(entry, true);
1019         } else {
1020                 trace_seq_printf(s, "%s: (0x%lx)",
1021                                  trace_probe_name(&tu->tp),
1022                                  entry->vaddr[0]);
1023                 data = DATAOF_TRACE_ENTRY(entry, false);
1024         }
1025 
1026         if (print_probe_args(s, tu->tp.args, tu->tp.nr_args, data, entry) < 0)
1027                 goto out;
1028 
1029         trace_seq_putc(s, '\n');
1030 
1031  out:
1032         return trace_handle_return(s);
1033 }
1034 
1035 typedef bool (*filter_func_t)(struct uprobe_consumer *self,
1036                                 enum uprobe_filter_ctx ctx,
1037                                 struct mm_struct *mm);
1038 
1039 static int trace_uprobe_enable(struct trace_uprobe *tu, filter_func_t filter)
1040 {
1041         int ret;
1042 
1043         tu->consumer.filter = filter;
1044         tu->inode = d_real_inode(tu->path.dentry);
1045 
1046         if (tu->ref_ctr_offset)
1047                 ret = uprobe_register_refctr(tu->inode, tu->offset,
1048                                 tu->ref_ctr_offset, &tu->consumer);
1049         else
1050                 ret = uprobe_register(tu->inode, tu->offset, &tu->consumer);
1051 
1052         if (ret)
1053                 tu->inode = NULL;
1054 
1055         return ret;
1056 }
1057 
1058 static void __probe_event_disable(struct trace_probe *tp)
1059 {
1060         struct trace_probe *pos;
1061         struct trace_uprobe *tu;
1062 
1063         tu = container_of(tp, struct trace_uprobe, tp);
1064         WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter));
1065 
1066         list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
1067                 tu = container_of(pos, struct trace_uprobe, tp);
1068                 if (!tu->inode)
1069                         continue;
1070 
1071                 uprobe_unregister(tu->inode, tu->offset, &tu->consumer);
1072                 tu->inode = NULL;
1073         }
1074 }
1075 
1076 static int probe_event_enable(struct trace_event_call *call,
1077                         struct trace_event_file *file, filter_func_t filter)
1078 {
1079         struct trace_probe *pos, *tp;
1080         struct trace_uprobe *tu;
1081         bool enabled;
1082         int ret;
1083 
1084         tp = trace_probe_primary_from_call(call);
1085         if (WARN_ON_ONCE(!tp))
1086                 return -ENODEV;
1087         enabled = trace_probe_is_enabled(tp);
1088 
1089         /* This may also change "enabled" state */
1090         if (file) {
1091                 if (trace_probe_test_flag(tp, TP_FLAG_PROFILE))
1092                         return -EINTR;
1093 
1094                 ret = trace_probe_add_file(tp, file);
1095                 if (ret < 0)
1096                         return ret;
1097         } else {
1098                 if (trace_probe_test_flag(tp, TP_FLAG_TRACE))
1099                         return -EINTR;
1100 
1101                 trace_probe_set_flag(tp, TP_FLAG_PROFILE);
1102         }
1103 
1104         tu = container_of(tp, struct trace_uprobe, tp);
1105         WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter));
1106 
1107         if (enabled)
1108                 return 0;
1109 
1110         ret = uprobe_buffer_enable();
1111         if (ret)
1112                 goto err_flags;
1113 
1114         list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
1115                 tu = container_of(pos, struct trace_uprobe, tp);
1116                 ret = trace_uprobe_enable(tu, filter);
1117                 if (ret) {
1118                         __probe_event_disable(tp);
1119                         goto err_buffer;
1120                 }
1121         }
1122 
1123         return 0;
1124 
1125  err_buffer:
1126         uprobe_buffer_disable();
1127 
1128  err_flags:
1129         if (file)
1130                 trace_probe_remove_file(tp, file);
1131         else
1132                 trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
1133 
1134         return ret;
1135 }
1136 
1137 static void probe_event_disable(struct trace_event_call *call,
1138                                 struct trace_event_file *file)
1139 {
1140         struct trace_probe *tp;
1141 
1142         tp = trace_probe_primary_from_call(call);
1143         if (WARN_ON_ONCE(!tp))
1144                 return;
1145 
1146         if (!trace_probe_is_enabled(tp))
1147                 return;
1148 
1149         if (file) {
1150                 if (trace_probe_remove_file(tp, file) < 0)
1151                         return;
1152 
1153                 if (trace_probe_is_enabled(tp))
1154                         return;
1155         } else
1156                 trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
1157 
1158         __probe_event_disable(tp);
1159         uprobe_buffer_disable();
1160 }
1161 
1162 static int uprobe_event_define_fields(struct trace_event_call *event_call)
1163 {
1164         int ret, size;
1165         struct uprobe_trace_entry_head field;
1166         struct trace_uprobe *tu;
1167 
1168         tu = trace_uprobe_primary_from_call(event_call);
1169         if (unlikely(!tu))
1170                 return -ENODEV;
1171 
1172         if (is_ret_probe(tu)) {
1173                 DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_FUNC, 0);
1174                 DEFINE_FIELD(unsigned long, vaddr[1], FIELD_STRING_RETIP, 0);
1175                 size = SIZEOF_TRACE_ENTRY(true);
1176         } else {
1177                 DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_IP, 0);
1178                 size = SIZEOF_TRACE_ENTRY(false);
1179         }
1180 
1181         return traceprobe_define_arg_fields(event_call, size, &tu->tp);
1182 }
1183 
1184 #ifdef CONFIG_PERF_EVENTS
1185 static bool
1186 __uprobe_perf_filter(struct trace_uprobe_filter *filter, struct mm_struct *mm)
1187 {
1188         struct perf_event *event;
1189 
1190         if (filter->nr_systemwide)
1191                 return true;
1192 
1193         list_for_each_entry(event, &filter->perf_events, hw.tp_list) {
1194                 if (event->hw.target->mm == mm)
1195                         return true;
1196         }
1197 
1198         return false;
1199 }
1200 
1201 static inline bool
1202 trace_uprobe_filter_event(struct trace_uprobe_filter *filter,
1203                           struct perf_event *event)
1204 {
1205         return __uprobe_perf_filter(filter, event->hw.target->mm);
1206 }
1207 
1208 static bool trace_uprobe_filter_remove(struct trace_uprobe_filter *filter,
1209                                        struct perf_event *event)
1210 {
1211         bool done;
1212 
1213         write_lock(&filter->rwlock);
1214         if (event->hw.target) {
1215                 list_del(&event->hw.tp_list);
1216                 done = filter->nr_systemwide ||
1217                         (event->hw.target->flags & PF_EXITING) ||
1218                         trace_uprobe_filter_event(filter, event);
1219         } else {
1220                 filter->nr_systemwide--;
1221                 done = filter->nr_systemwide;
1222         }
1223         write_unlock(&filter->rwlock);
1224 
1225         return done;
1226 }
1227 
1228 /* This returns true if the filter always covers target mm */
1229 static bool trace_uprobe_filter_add(struct trace_uprobe_filter *filter,
1230                                     struct perf_event *event)
1231 {
1232         bool done;
1233 
1234         write_lock(&filter->rwlock);
1235         if (event->hw.target) {
1236                 /*
1237                  * event->parent != NULL means copy_process(), we can avoid
1238                  * uprobe_apply(). current->mm must be probed and we can rely
1239                  * on dup_mmap() which preserves the already installed bp's.
1240                  *
1241                  * attr.enable_on_exec means that exec/mmap will install the
1242                  * breakpoints we need.
1243                  */
1244                 done = filter->nr_systemwide ||
1245                         event->parent || event->attr.enable_on_exec ||
1246                         trace_uprobe_filter_event(filter, event);
1247                 list_add(&event->hw.tp_list, &filter->perf_events);
1248         } else {
1249                 done = filter->nr_systemwide;
1250                 filter->nr_systemwide++;
1251         }
1252         write_unlock(&filter->rwlock);
1253 
1254         return done;
1255 }
1256 
1257 static int uprobe_perf_close(struct trace_event_call *call,
1258                              struct perf_event *event)
1259 {
1260         struct trace_probe *pos, *tp;
1261         struct trace_uprobe *tu;
1262         int ret = 0;
1263 
1264         tp = trace_probe_primary_from_call(call);
1265         if (WARN_ON_ONCE(!tp))
1266                 return -ENODEV;
1267 
1268         tu = container_of(tp, struct trace_uprobe, tp);
1269         if (trace_uprobe_filter_remove(tu->tp.event->filter, event))
1270                 return 0;
1271 
1272         list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
1273                 tu = container_of(pos, struct trace_uprobe, tp);
1274                 ret = uprobe_apply(tu->inode, tu->offset, &tu->consumer, false);
1275                 if (ret)
1276                         break;
1277         }
1278 
1279         return ret;
1280 }
1281 
1282 static int uprobe_perf_open(struct trace_event_call *call,
1283                             struct perf_event *event)
1284 {
1285         struct trace_probe *pos, *tp;
1286         struct trace_uprobe *tu;
1287         int err = 0;
1288 
1289         tp = trace_probe_primary_from_call(call);
1290         if (WARN_ON_ONCE(!tp))
1291                 return -ENODEV;
1292 
1293         tu = container_of(tp, struct trace_uprobe, tp);
1294         if (trace_uprobe_filter_add(tu->tp.event->filter, event))
1295                 return 0;
1296 
1297         list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
1298                 err = uprobe_apply(tu->inode, tu->offset, &tu->consumer, true);
1299                 if (err) {
1300                         uprobe_perf_close(call, event);
1301                         break;
1302                 }
1303         }
1304 
1305         return err;
1306 }
1307 
1308 static bool uprobe_perf_filter(struct uprobe_consumer *uc,
1309                                 enum uprobe_filter_ctx ctx, struct mm_struct *mm)
1310 {
1311         struct trace_uprobe_filter *filter;
1312         struct trace_uprobe *tu;
1313         int ret;
1314 
1315         tu = container_of(uc, struct trace_uprobe, consumer);
1316         filter = tu->tp.event->filter;
1317 
1318         read_lock(&filter->rwlock);
1319         ret = __uprobe_perf_filter(filter, mm);
1320         read_unlock(&filter->rwlock);
1321 
1322         return ret;
1323 }
1324 
1325 static void __uprobe_perf_func(struct trace_uprobe *tu,
1326                                unsigned long func, struct pt_regs *regs,
1327                                struct uprobe_cpu_buffer *ucb, int dsize)
1328 {
1329         struct trace_event_call *call = trace_probe_event_call(&tu->tp);
1330         struct uprobe_trace_entry_head *entry;
1331         struct hlist_head *head;
1332         void *data;
1333         int size, esize;
1334         int rctx;
1335 
1336         if (bpf_prog_array_valid(call)) {
1337                 u32 ret;
1338 
1339                 preempt_disable();
1340                 ret = trace_call_bpf(call, regs);
1341                 preempt_enable();
1342                 if (!ret)
1343                         return;
1344         }
1345 
1346         esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
1347 
1348         size = esize + tu->tp.size + dsize;
1349         size = ALIGN(size + sizeof(u32), sizeof(u64)) - sizeof(u32);
1350         if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, "profile buffer not large enough"))
1351                 return;
1352 
1353         preempt_disable();
1354         head = this_cpu_ptr(call->perf_events);
1355         if (hlist_empty(head))
1356                 goto out;
1357 
1358         entry = perf_trace_buf_alloc(size, NULL, &rctx);
1359         if (!entry)
1360                 goto out;
1361 
1362         if (is_ret_probe(tu)) {
1363                 entry->vaddr[0] = func;
1364                 entry->vaddr[1] = instruction_pointer(regs);
1365                 data = DATAOF_TRACE_ENTRY(entry, true);
1366         } else {
1367                 entry->vaddr[0] = instruction_pointer(regs);
1368                 data = DATAOF_TRACE_ENTRY(entry, false);
1369         }
1370 
1371         memcpy(data, ucb->buf, tu->tp.size + dsize);
1372 
1373         if (size - esize > tu->tp.size + dsize) {
1374                 int len = tu->tp.size + dsize;
1375 
1376                 memset(data + len, 0, size - esize - len);
1377         }
1378 
1379         perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1380                               head, NULL);
1381  out:
1382         preempt_enable();
1383 }
1384 
1385 /* uprobe profile handler */
1386 static int uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs,
1387                             struct uprobe_cpu_buffer *ucb, int dsize)
1388 {
1389         if (!uprobe_perf_filter(&tu->consumer, 0, current->mm))
1390                 return UPROBE_HANDLER_REMOVE;
1391 
1392         if (!is_ret_probe(tu))
1393                 __uprobe_perf_func(tu, 0, regs, ucb, dsize);
1394         return 0;
1395 }
1396 
1397 static void uretprobe_perf_func(struct trace_uprobe *tu, unsigned long func,
1398                                 struct pt_regs *regs,
1399                                 struct uprobe_cpu_buffer *ucb, int dsize)
1400 {
1401         __uprobe_perf_func(tu, func, regs, ucb, dsize);
1402 }
1403 
1404 int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type,
1405                         const char **filename, u64 *probe_offset,
1406                         bool perf_type_tracepoint)
1407 {
1408         const char *pevent = trace_event_name(event->tp_event);
1409         const char *group = event->tp_event->class->system;
1410         struct trace_uprobe *tu;
1411 
1412         if (perf_type_tracepoint)
1413                 tu = find_probe_event(pevent, group);
1414         else
1415                 tu = trace_uprobe_primary_from_call(event->tp_event);
1416         if (!tu)
1417                 return -EINVAL;
1418 
1419         *fd_type = is_ret_probe(tu) ? BPF_FD_TYPE_URETPROBE
1420                                     : BPF_FD_TYPE_UPROBE;
1421         *filename = tu->filename;
1422         *probe_offset = tu->offset;
1423         return 0;
1424 }
1425 #endif  /* CONFIG_PERF_EVENTS */
1426 
1427 static int
1428 trace_uprobe_register(struct trace_event_call *event, enum trace_reg type,
1429                       void *data)
1430 {
1431         struct trace_event_file *file = data;
1432 
1433         switch (type) {
1434         case TRACE_REG_REGISTER:
1435                 return probe_event_enable(event, file, NULL);
1436 
1437         case TRACE_REG_UNREGISTER:
1438                 probe_event_disable(event, file);
1439                 return 0;
1440 
1441 #ifdef CONFIG_PERF_EVENTS
1442         case TRACE_REG_PERF_REGISTER:
1443                 return probe_event_enable(event, NULL, uprobe_perf_filter);
1444 
1445         case TRACE_REG_PERF_UNREGISTER:
1446                 probe_event_disable(event, NULL);
1447                 return 0;
1448 
1449         case TRACE_REG_PERF_OPEN:
1450                 return uprobe_perf_open(event, data);
1451 
1452         case TRACE_REG_PERF_CLOSE:
1453                 return uprobe_perf_close(event, data);
1454 
1455 #endif
1456         default:
1457                 return 0;
1458         }
1459         return 0;
1460 }
1461 
1462 static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs)
1463 {
1464         struct trace_uprobe *tu;
1465         struct uprobe_dispatch_data udd;
1466         struct uprobe_cpu_buffer *ucb;
1467         int dsize, esize;
1468         int ret = 0;
1469 
1470 
1471         tu = container_of(con, struct trace_uprobe, consumer);
1472         tu->nhit++;
1473 
1474         udd.tu = tu;
1475         udd.bp_addr = instruction_pointer(regs);
1476 
1477         current->utask->vaddr = (unsigned long) &udd;
1478 
1479         if (WARN_ON_ONCE(!uprobe_cpu_buffer))
1480                 return 0;
1481 
1482         dsize = __get_data_size(&tu->tp, regs);
1483         esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
1484 
1485         ucb = uprobe_buffer_get();
1486         store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize);
1487 
1488         if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE))
1489                 ret |= uprobe_trace_func(tu, regs, ucb, dsize);
1490 
1491 #ifdef CONFIG_PERF_EVENTS
1492         if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE))
1493                 ret |= uprobe_perf_func(tu, regs, ucb, dsize);
1494 #endif
1495         uprobe_buffer_put(ucb);
1496         return ret;
1497 }
1498 
1499 static int uretprobe_dispatcher(struct uprobe_consumer *con,
1500                                 unsigned long func, struct pt_regs *regs)
1501 {
1502         struct trace_uprobe *tu;
1503         struct uprobe_dispatch_data udd;
1504         struct uprobe_cpu_buffer *ucb;
1505         int dsize, esize;
1506 
1507         tu = container_of(con, struct trace_uprobe, consumer);
1508 
1509         udd.tu = tu;
1510         udd.bp_addr = func;
1511 
1512         current->utask->vaddr = (unsigned long) &udd;
1513 
1514         if (WARN_ON_ONCE(!uprobe_cpu_buffer))
1515                 return 0;
1516 
1517         dsize = __get_data_size(&tu->tp, regs);
1518         esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
1519 
1520         ucb = uprobe_buffer_get();
1521         store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize);
1522 
1523         if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE))
1524                 uretprobe_trace_func(tu, func, regs, ucb, dsize);
1525 
1526 #ifdef CONFIG_PERF_EVENTS
1527         if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE))
1528                 uretprobe_perf_func(tu, func, regs, ucb, dsize);
1529 #endif
1530         uprobe_buffer_put(ucb);
1531         return 0;
1532 }
1533 
1534 static struct trace_event_functions uprobe_funcs = {
1535         .trace          = print_uprobe_event
1536 };
1537 
1538 static struct trace_event_fields uprobe_fields_array[] = {
1539         { .type = TRACE_FUNCTION_TYPE,
1540           .define_fields = uprobe_event_define_fields },
1541         {}
1542 };
1543 
1544 static inline void init_trace_event_call(struct trace_uprobe *tu)
1545 {
1546         struct trace_event_call *call = trace_probe_event_call(&tu->tp);
1547         call->event.funcs = &uprobe_funcs;
1548         call->class->fields_array = uprobe_fields_array;
1549 
1550         call->flags = TRACE_EVENT_FL_UPROBE | TRACE_EVENT_FL_CAP_ANY;
1551         call->class->reg = trace_uprobe_register;
1552 }
1553 
1554 static int register_uprobe_event(struct trace_uprobe *tu)
1555 {
1556         init_trace_event_call(tu);
1557 
1558         return trace_probe_register_event_call(&tu->tp);
1559 }
1560 
1561 static int unregister_uprobe_event(struct trace_uprobe *tu)
1562 {
1563         return trace_probe_unregister_event_call(&tu->tp);
1564 }
1565 
1566 #ifdef CONFIG_PERF_EVENTS
1567 struct trace_event_call *
1568 create_local_trace_uprobe(char *name, unsigned long offs,
1569                           unsigned long ref_ctr_offset, bool is_return)
1570 {
1571         struct trace_uprobe *tu;
1572         struct path path;
1573         int ret;
1574 
1575         ret = kern_path(name, LOOKUP_FOLLOW, &path);
1576         if (ret)
1577                 return ERR_PTR(ret);
1578 
1579         if (!d_is_reg(path.dentry)) {
1580                 path_put(&path);
1581                 return ERR_PTR(-EINVAL);
1582         }
1583 
1584         /*
1585          * local trace_kprobes are not added to dyn_event, so they are never
1586          * searched in find_trace_kprobe(). Therefore, there is no concern of
1587          * duplicated name "DUMMY_EVENT" here.
1588          */
1589         tu = alloc_trace_uprobe(UPROBE_EVENT_SYSTEM, "DUMMY_EVENT", 0,
1590                                 is_return);
1591 
1592         if (IS_ERR(tu)) {
1593                 pr_info("Failed to allocate trace_uprobe.(%d)\n",
1594                         (int)PTR_ERR(tu));
1595                 path_put(&path);
1596                 return ERR_CAST(tu);
1597         }
1598 
1599         tu->offset = offs;
1600         tu->path = path;
1601         tu->ref_ctr_offset = ref_ctr_offset;
1602         tu->filename = kstrdup(name, GFP_KERNEL);
1603         init_trace_event_call(tu);
1604 
1605         if (traceprobe_set_print_fmt(&tu->tp, is_ret_probe(tu)) < 0) {
1606                 ret = -ENOMEM;
1607                 goto error;
1608         }
1609 
1610         return trace_probe_event_call(&tu->tp);
1611 error:
1612         free_trace_uprobe(tu);
1613         return ERR_PTR(ret);
1614 }
1615 
1616 void destroy_local_trace_uprobe(struct trace_event_call *event_call)
1617 {
1618         struct trace_uprobe *tu;
1619 
1620         tu = trace_uprobe_primary_from_call(event_call);
1621 
1622         free_trace_uprobe(tu);
1623 }
1624 #endif /* CONFIG_PERF_EVENTS */
1625 
1626 /* Make a trace interface for controling probe points */
1627 static __init int init_uprobe_trace(void)
1628 {
1629         struct dentry *d_tracer;
1630         int ret;
1631 
1632         ret = dyn_event_register(&trace_uprobe_ops);
1633         if (ret)
1634                 return ret;
1635 
1636         d_tracer = tracing_init_dentry();
1637         if (IS_ERR(d_tracer))
1638                 return 0;
1639 
1640         trace_create_file("uprobe_events", 0644, d_tracer,
1641                                     NULL, &uprobe_events_ops);
1642         /* Profile interface */
1643         trace_create_file("uprobe_profile", 0444, d_tracer,
1644                                     NULL, &uprobe_profile_ops);
1645         return 0;
1646 }
1647 
1648 fs_initcall(init_uprobe_trace);
1649 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp