~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/trace/trace_kprobe.c

Version: ~ [ linux-4.17-rc6 ] ~ [ linux-4.16.10 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.42 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.101 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.132 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.51 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.109 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.56 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.101 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Kprobes-based tracing events
  3  *
  4  * Created by Masami Hiramatsu <mhiramat@redhat.com>
  5  *
  6  * This program is free software; you can redistribute it and/or modify
  7  * it under the terms of the GNU General Public License version 2 as
  8  * published by the Free Software Foundation.
  9  *
 10  * This program is distributed in the hope that it will be useful,
 11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13  * GNU General Public License for more details.
 14  *
 15  * You should have received a copy of the GNU General Public License
 16  * along with this program; if not, write to the Free Software
 17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 18  */
 19 #define pr_fmt(fmt)     "trace_kprobe: " fmt
 20 
 21 #include <linux/module.h>
 22 #include <linux/uaccess.h>
 23 #include <linux/rculist.h>
 24 #include <linux/error-injection.h>
 25 
 26 #include "trace_probe.h"
 27 
 28 #define KPROBE_EVENT_SYSTEM "kprobes"
 29 #define KRETPROBE_MAXACTIVE_MAX 4096
 30 
 31 /**
 32  * Kprobe event core functions
 33  */
 34 struct trace_kprobe {
 35         struct list_head        list;
 36         struct kretprobe        rp;     /* Use rp.kp for kprobe use */
 37         unsigned long __percpu *nhit;
 38         const char              *symbol;        /* symbol name */
 39         struct trace_probe      tp;
 40 };
 41 
 42 #define SIZEOF_TRACE_KPROBE(n)                          \
 43         (offsetof(struct trace_kprobe, tp.args) +       \
 44         (sizeof(struct probe_arg) * (n)))
 45 
 46 static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk)
 47 {
 48         return tk->rp.handler != NULL;
 49 }
 50 
 51 static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk)
 52 {
 53         return tk->symbol ? tk->symbol : "unknown";
 54 }
 55 
 56 static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk)
 57 {
 58         return tk->rp.kp.offset;
 59 }
 60 
 61 static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk)
 62 {
 63         return !!(kprobe_gone(&tk->rp.kp));
 64 }
 65 
 66 static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,
 67                                                  struct module *mod)
 68 {
 69         int len = strlen(mod->name);
 70         const char *name = trace_kprobe_symbol(tk);
 71         return strncmp(mod->name, name, len) == 0 && name[len] == ':';
 72 }
 73 
 74 static nokprobe_inline bool trace_kprobe_is_on_module(struct trace_kprobe *tk)
 75 {
 76         return !!strchr(trace_kprobe_symbol(tk), ':');
 77 }
 78 
 79 static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk)
 80 {
 81         unsigned long nhit = 0;
 82         int cpu;
 83 
 84         for_each_possible_cpu(cpu)
 85                 nhit += *per_cpu_ptr(tk->nhit, cpu);
 86 
 87         return nhit;
 88 }
 89 
 90 bool trace_kprobe_on_func_entry(struct trace_event_call *call)
 91 {
 92         struct trace_kprobe *tk = (struct trace_kprobe *)call->data;
 93 
 94         return kprobe_on_func_entry(tk->rp.kp.addr,
 95                         tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name,
 96                         tk->rp.kp.addr ? 0 : tk->rp.kp.offset);
 97 }
 98 
 99 bool trace_kprobe_error_injectable(struct trace_event_call *call)
100 {
101         struct trace_kprobe *tk = (struct trace_kprobe *)call->data;
102         unsigned long addr;
103 
104         if (tk->symbol) {
105                 addr = (unsigned long)
106                         kallsyms_lookup_name(trace_kprobe_symbol(tk));
107                 addr += tk->rp.kp.offset;
108         } else {
109                 addr = (unsigned long)tk->rp.kp.addr;
110         }
111         return within_error_injection_list(addr);
112 }
113 
114 static int register_kprobe_event(struct trace_kprobe *tk);
115 static int unregister_kprobe_event(struct trace_kprobe *tk);
116 
117 static DEFINE_MUTEX(probe_lock);
118 static LIST_HEAD(probe_list);
119 
120 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
121 static int kretprobe_dispatcher(struct kretprobe_instance *ri,
122                                 struct pt_regs *regs);
123 
124 /* Memory fetching by symbol */
125 struct symbol_cache {
126         char            *symbol;
127         long            offset;
128         unsigned long   addr;
129 };
130 
131 unsigned long update_symbol_cache(struct symbol_cache *sc)
132 {
133         sc->addr = (unsigned long)kallsyms_lookup_name(sc->symbol);
134 
135         if (sc->addr)
136                 sc->addr += sc->offset;
137 
138         return sc->addr;
139 }
140 
141 void free_symbol_cache(struct symbol_cache *sc)
142 {
143         kfree(sc->symbol);
144         kfree(sc);
145 }
146 
147 struct symbol_cache *alloc_symbol_cache(const char *sym, long offset)
148 {
149         struct symbol_cache *sc;
150 
151         if (!sym || strlen(sym) == 0)
152                 return NULL;
153 
154         sc = kzalloc(sizeof(struct symbol_cache), GFP_KERNEL);
155         if (!sc)
156                 return NULL;
157 
158         sc->symbol = kstrdup(sym, GFP_KERNEL);
159         if (!sc->symbol) {
160                 kfree(sc);
161                 return NULL;
162         }
163         sc->offset = offset;
164         update_symbol_cache(sc);
165 
166         return sc;
167 }
168 
169 /*
170  * Kprobes-specific fetch functions
171  */
172 #define DEFINE_FETCH_stack(type)                                        \
173 static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs,          \
174                                           void *offset, void *dest)     \
175 {                                                                       \
176         *(type *)dest = (type)regs_get_kernel_stack_nth(regs,           \
177                                 (unsigned int)((unsigned long)offset)); \
178 }                                                                       \
179 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(stack, type));
180 
181 DEFINE_BASIC_FETCH_FUNCS(stack)
182 /* No string on the stack entry */
183 #define fetch_stack_string      NULL
184 #define fetch_stack_string_size NULL
185 
186 #define DEFINE_FETCH_memory(type)                                       \
187 static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,         \
188                                           void *addr, void *dest)       \
189 {                                                                       \
190         type retval;                                                    \
191         if (probe_kernel_address(addr, retval))                         \
192                 *(type *)dest = 0;                                      \
193         else                                                            \
194                 *(type *)dest = retval;                                 \
195 }                                                                       \
196 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, type));
197 
198 DEFINE_BASIC_FETCH_FUNCS(memory)
199 /*
200  * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
201  * length and relative data location.
202  */
203 static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
204                                             void *addr, void *dest)
205 {
206         int maxlen = get_rloc_len(*(u32 *)dest);
207         u8 *dst = get_rloc_data(dest);
208         long ret;
209 
210         if (!maxlen)
211                 return;
212 
213         /*
214          * Try to get string again, since the string can be changed while
215          * probing.
216          */
217         ret = strncpy_from_unsafe(dst, addr, maxlen);
218 
219         if (ret < 0) {  /* Failed to fetch string */
220                 dst[0] = '\0';
221                 *(u32 *)dest = make_data_rloc(0, get_rloc_offs(*(u32 *)dest));
222         } else {
223                 *(u32 *)dest = make_data_rloc(ret, get_rloc_offs(*(u32 *)dest));
224         }
225 }
226 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string));
227 
228 /* Return the length of string -- including null terminal byte */
229 static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
230                                                  void *addr, void *dest)
231 {
232         mm_segment_t old_fs;
233         int ret, len = 0;
234         u8 c;
235 
236         old_fs = get_fs();
237         set_fs(KERNEL_DS);
238         pagefault_disable();
239 
240         do {
241                 ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1);
242                 len++;
243         } while (c && ret == 0 && len < MAX_STRING_SIZE);
244 
245         pagefault_enable();
246         set_fs(old_fs);
247 
248         if (ret < 0)    /* Failed to check the length */
249                 *(u32 *)dest = 0;
250         else
251                 *(u32 *)dest = len;
252 }
253 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string_size));
254 
255 #define DEFINE_FETCH_symbol(type)                                       \
256 void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs, void *data, void *dest)\
257 {                                                                       \
258         struct symbol_cache *sc = data;                                 \
259         if (sc->addr)                                                   \
260                 fetch_memory_##type(regs, (void *)sc->addr, dest);      \
261         else                                                            \
262                 *(type *)dest = 0;                                      \
263 }                                                                       \
264 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(symbol, type));
265 
266 DEFINE_BASIC_FETCH_FUNCS(symbol)
267 DEFINE_FETCH_symbol(string)
268 DEFINE_FETCH_symbol(string_size)
269 
270 /* kprobes don't support file_offset fetch methods */
271 #define fetch_file_offset_u8            NULL
272 #define fetch_file_offset_u16           NULL
273 #define fetch_file_offset_u32           NULL
274 #define fetch_file_offset_u64           NULL
275 #define fetch_file_offset_string        NULL
276 #define fetch_file_offset_string_size   NULL
277 
278 /* Fetch type information table */
279 static const struct fetch_type kprobes_fetch_type_table[] = {
280         /* Special types */
281         [FETCH_TYPE_STRING] = __ASSIGN_FETCH_TYPE("string", string, string,
282                                         sizeof(u32), 1, "__data_loc char[]"),
283         [FETCH_TYPE_STRSIZE] = __ASSIGN_FETCH_TYPE("string_size", u32,
284                                         string_size, sizeof(u32), 0, "u32"),
285         /* Basic types */
286         ASSIGN_FETCH_TYPE(u8,  u8,  0),
287         ASSIGN_FETCH_TYPE(u16, u16, 0),
288         ASSIGN_FETCH_TYPE(u32, u32, 0),
289         ASSIGN_FETCH_TYPE(u64, u64, 0),
290         ASSIGN_FETCH_TYPE(s8,  u8,  1),
291         ASSIGN_FETCH_TYPE(s16, u16, 1),
292         ASSIGN_FETCH_TYPE(s32, u32, 1),
293         ASSIGN_FETCH_TYPE(s64, u64, 1),
294         ASSIGN_FETCH_TYPE_ALIAS(x8,  u8,  u8,  0),
295         ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
296         ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
297         ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
298 
299         ASSIGN_FETCH_TYPE_END
300 };
301 
302 /*
303  * Allocate new trace_probe and initialize it (including kprobes).
304  */
305 static struct trace_kprobe *alloc_trace_kprobe(const char *group,
306                                              const char *event,
307                                              void *addr,
308                                              const char *symbol,
309                                              unsigned long offs,
310                                              int maxactive,
311                                              int nargs, bool is_return)
312 {
313         struct trace_kprobe *tk;
314         int ret = -ENOMEM;
315 
316         tk = kzalloc(SIZEOF_TRACE_KPROBE(nargs), GFP_KERNEL);
317         if (!tk)
318                 return ERR_PTR(ret);
319 
320         tk->nhit = alloc_percpu(unsigned long);
321         if (!tk->nhit)
322                 goto error;
323 
324         if (symbol) {
325                 tk->symbol = kstrdup(symbol, GFP_KERNEL);
326                 if (!tk->symbol)
327                         goto error;
328                 tk->rp.kp.symbol_name = tk->symbol;
329                 tk->rp.kp.offset = offs;
330         } else
331                 tk->rp.kp.addr = addr;
332 
333         if (is_return)
334                 tk->rp.handler = kretprobe_dispatcher;
335         else
336                 tk->rp.kp.pre_handler = kprobe_dispatcher;
337 
338         tk->rp.maxactive = maxactive;
339 
340         if (!event || !is_good_name(event)) {
341                 ret = -EINVAL;
342                 goto error;
343         }
344 
345         tk->tp.call.class = &tk->tp.class;
346         tk->tp.call.name = kstrdup(event, GFP_KERNEL);
347         if (!tk->tp.call.name)
348                 goto error;
349 
350         if (!group || !is_good_name(group)) {
351                 ret = -EINVAL;
352                 goto error;
353         }
354 
355         tk->tp.class.system = kstrdup(group, GFP_KERNEL);
356         if (!tk->tp.class.system)
357                 goto error;
358 
359         INIT_LIST_HEAD(&tk->list);
360         INIT_LIST_HEAD(&tk->tp.files);
361         return tk;
362 error:
363         kfree(tk->tp.call.name);
364         kfree(tk->symbol);
365         free_percpu(tk->nhit);
366         kfree(tk);
367         return ERR_PTR(ret);
368 }
369 
370 static void free_trace_kprobe(struct trace_kprobe *tk)
371 {
372         int i;
373 
374         for (i = 0; i < tk->tp.nr_args; i++)
375                 traceprobe_free_probe_arg(&tk->tp.args[i]);
376 
377         kfree(tk->tp.call.class->system);
378         kfree(tk->tp.call.name);
379         kfree(tk->symbol);
380         free_percpu(tk->nhit);
381         kfree(tk);
382 }
383 
384 static struct trace_kprobe *find_trace_kprobe(const char *event,
385                                               const char *group)
386 {
387         struct trace_kprobe *tk;
388 
389         list_for_each_entry(tk, &probe_list, list)
390                 if (strcmp(trace_event_name(&tk->tp.call), event) == 0 &&
391                     strcmp(tk->tp.call.class->system, group) == 0)
392                         return tk;
393         return NULL;
394 }
395 
396 /*
397  * Enable trace_probe
398  * if the file is NULL, enable "perf" handler, or enable "trace" handler.
399  */
400 static int
401 enable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file)
402 {
403         int ret = 0;
404 
405         if (file) {
406                 struct event_file_link *link;
407 
408                 link = kmalloc(sizeof(*link), GFP_KERNEL);
409                 if (!link) {
410                         ret = -ENOMEM;
411                         goto out;
412                 }
413 
414                 link->file = file;
415                 list_add_tail_rcu(&link->list, &tk->tp.files);
416 
417                 tk->tp.flags |= TP_FLAG_TRACE;
418         } else
419                 tk->tp.flags |= TP_FLAG_PROFILE;
420 
421         if (trace_probe_is_registered(&tk->tp) && !trace_kprobe_has_gone(tk)) {
422                 if (trace_kprobe_is_return(tk))
423                         ret = enable_kretprobe(&tk->rp);
424                 else
425                         ret = enable_kprobe(&tk->rp.kp);
426         }
427  out:
428         return ret;
429 }
430 
431 /*
432  * Disable trace_probe
433  * if the file is NULL, disable "perf" handler, or disable "trace" handler.
434  */
435 static int
436 disable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file)
437 {
438         struct event_file_link *link = NULL;
439         int wait = 0;
440         int ret = 0;
441 
442         if (file) {
443                 link = find_event_file_link(&tk->tp, file);
444                 if (!link) {
445                         ret = -EINVAL;
446                         goto out;
447                 }
448 
449                 list_del_rcu(&link->list);
450                 wait = 1;
451                 if (!list_empty(&tk->tp.files))
452                         goto out;
453 
454                 tk->tp.flags &= ~TP_FLAG_TRACE;
455         } else
456                 tk->tp.flags &= ~TP_FLAG_PROFILE;
457 
458         if (!trace_probe_is_enabled(&tk->tp) && trace_probe_is_registered(&tk->tp)) {
459                 if (trace_kprobe_is_return(tk))
460                         disable_kretprobe(&tk->rp);
461                 else
462                         disable_kprobe(&tk->rp.kp);
463                 wait = 1;
464         }
465 
466         /*
467          * if tk is not added to any list, it must be a local trace_kprobe
468          * created with perf_event_open. We don't need to wait for these
469          * trace_kprobes
470          */
471         if (list_empty(&tk->list))
472                 wait = 0;
473  out:
474         if (wait) {
475                 /*
476                  * Synchronize with kprobe_trace_func/kretprobe_trace_func
477                  * to ensure disabled (all running handlers are finished).
478                  * This is not only for kfree(), but also the caller,
479                  * trace_remove_event_call() supposes it for releasing
480                  * event_call related objects, which will be accessed in
481                  * the kprobe_trace_func/kretprobe_trace_func.
482                  */
483                 synchronize_sched();
484                 kfree(link);    /* Ignored if link == NULL */
485         }
486 
487         return ret;
488 }
489 
490 /* Internal register function - just handle k*probes and flags */
491 static int __register_trace_kprobe(struct trace_kprobe *tk)
492 {
493         int i, ret;
494 
495         if (trace_probe_is_registered(&tk->tp))
496                 return -EINVAL;
497 
498         for (i = 0; i < tk->tp.nr_args; i++)
499                 traceprobe_update_arg(&tk->tp.args[i]);
500 
501         /* Set/clear disabled flag according to tp->flag */
502         if (trace_probe_is_enabled(&tk->tp))
503                 tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;
504         else
505                 tk->rp.kp.flags |= KPROBE_FLAG_DISABLED;
506 
507         if (trace_kprobe_is_return(tk))
508                 ret = register_kretprobe(&tk->rp);
509         else
510                 ret = register_kprobe(&tk->rp.kp);
511 
512         if (ret == 0)
513                 tk->tp.flags |= TP_FLAG_REGISTERED;
514         else {
515                 if (ret == -ENOENT && trace_kprobe_is_on_module(tk)) {
516                         pr_warn("This probe might be able to register after target module is loaded. Continue.\n");
517                         ret = 0;
518                 } else if (ret == -EILSEQ) {
519                         pr_warn("Probing address(0x%p) is not an instruction boundary.\n",
520                                 tk->rp.kp.addr);
521                         ret = -EINVAL;
522                 }
523         }
524 
525         return ret;
526 }
527 
528 /* Internal unregister function - just handle k*probes and flags */
529 static void __unregister_trace_kprobe(struct trace_kprobe *tk)
530 {
531         if (trace_probe_is_registered(&tk->tp)) {
532                 if (trace_kprobe_is_return(tk))
533                         unregister_kretprobe(&tk->rp);
534                 else
535                         unregister_kprobe(&tk->rp.kp);
536                 tk->tp.flags &= ~TP_FLAG_REGISTERED;
537                 /* Cleanup kprobe for reuse */
538                 if (tk->rp.kp.symbol_name)
539                         tk->rp.kp.addr = NULL;
540         }
541 }
542 
543 /* Unregister a trace_probe and probe_event: call with locking probe_lock */
544 static int unregister_trace_kprobe(struct trace_kprobe *tk)
545 {
546         /* Enabled event can not be unregistered */
547         if (trace_probe_is_enabled(&tk->tp))
548                 return -EBUSY;
549 
550         /* Will fail if probe is being used by ftrace or perf */
551         if (unregister_kprobe_event(tk))
552                 return -EBUSY;
553 
554         __unregister_trace_kprobe(tk);
555         list_del(&tk->list);
556 
557         return 0;
558 }
559 
560 /* Register a trace_probe and probe_event */
561 static int register_trace_kprobe(struct trace_kprobe *tk)
562 {
563         struct trace_kprobe *old_tk;
564         int ret;
565 
566         mutex_lock(&probe_lock);
567 
568         /* Delete old (same name) event if exist */
569         old_tk = find_trace_kprobe(trace_event_name(&tk->tp.call),
570                         tk->tp.call.class->system);
571         if (old_tk) {
572                 ret = unregister_trace_kprobe(old_tk);
573                 if (ret < 0)
574                         goto end;
575                 free_trace_kprobe(old_tk);
576         }
577 
578         /* Register new event */
579         ret = register_kprobe_event(tk);
580         if (ret) {
581                 pr_warn("Failed to register probe event(%d)\n", ret);
582                 goto end;
583         }
584 
585         /* Register k*probe */
586         ret = __register_trace_kprobe(tk);
587         if (ret < 0)
588                 unregister_kprobe_event(tk);
589         else
590                 list_add_tail(&tk->list, &probe_list);
591 
592 end:
593         mutex_unlock(&probe_lock);
594         return ret;
595 }
596 
597 /* Module notifier call back, checking event on the module */
598 static int trace_kprobe_module_callback(struct notifier_block *nb,
599                                        unsigned long val, void *data)
600 {
601         struct module *mod = data;
602         struct trace_kprobe *tk;
603         int ret;
604 
605         if (val != MODULE_STATE_COMING)
606                 return NOTIFY_DONE;
607 
608         /* Update probes on coming module */
609         mutex_lock(&probe_lock);
610         list_for_each_entry(tk, &probe_list, list) {
611                 if (trace_kprobe_within_module(tk, mod)) {
612                         /* Don't need to check busy - this should have gone. */
613                         __unregister_trace_kprobe(tk);
614                         ret = __register_trace_kprobe(tk);
615                         if (ret)
616                                 pr_warn("Failed to re-register probe %s on %s: %d\n",
617                                         trace_event_name(&tk->tp.call),
618                                         mod->name, ret);
619                 }
620         }
621         mutex_unlock(&probe_lock);
622 
623         return NOTIFY_DONE;
624 }
625 
626 static struct notifier_block trace_kprobe_module_nb = {
627         .notifier_call = trace_kprobe_module_callback,
628         .priority = 1   /* Invoked after kprobe module callback */
629 };
630 
631 /* Convert certain expected symbols into '_' when generating event names */
632 static inline void sanitize_event_name(char *name)
633 {
634         while (*name++ != '\0')
635                 if (*name == ':' || *name == '.')
636                         *name = '_';
637 }
638 
639 static int create_trace_kprobe(int argc, char **argv)
640 {
641         /*
642          * Argument syntax:
643          *  - Add kprobe:
644          *      p[:[GRP/]EVENT] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS]
645          *  - Add kretprobe:
646          *      r[MAXACTIVE][:[GRP/]EVENT] [MOD:]KSYM[+0] [FETCHARGS]
647          * Fetch args:
648          *  $retval     : fetch return value
649          *  $stack      : fetch stack address
650          *  $stackN     : fetch Nth of stack (N:0-)
651          *  $comm       : fetch current task comm
652          *  @ADDR       : fetch memory at ADDR (ADDR should be in kernel)
653          *  @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
654          *  %REG        : fetch register REG
655          * Dereferencing memory fetch:
656          *  +|-offs(ARG) : fetch memory at ARG +|- offs address.
657          * Alias name of args:
658          *  NAME=FETCHARG : set NAME as alias of FETCHARG.
659          * Type of args:
660          *  FETCHARG:TYPE : use TYPE instead of unsigned long.
661          */
662         struct trace_kprobe *tk;
663         int i, ret = 0;
664         bool is_return = false, is_delete = false;
665         char *symbol = NULL, *event = NULL, *group = NULL;
666         int maxactive = 0;
667         char *arg;
668         long offset = 0;
669         void *addr = NULL;
670         char buf[MAX_EVENT_NAME_LEN];
671 
672         /* argc must be >= 1 */
673         if (argv[0][0] == 'p')
674                 is_return = false;
675         else if (argv[0][0] == 'r')
676                 is_return = true;
677         else if (argv[0][0] == '-')
678                 is_delete = true;
679         else {
680                 pr_info("Probe definition must be started with 'p', 'r' or"
681                         " '-'.\n");
682                 return -EINVAL;
683         }
684 
685         event = strchr(&argv[0][1], ':');
686         if (event) {
687                 event[0] = '\0';
688                 event++;
689         }
690         if (is_return && isdigit(argv[0][1])) {
691                 ret = kstrtouint(&argv[0][1], 0, &maxactive);
692                 if (ret) {
693                         pr_info("Failed to parse maxactive.\n");
694                         return ret;
695                 }
696                 /* kretprobes instances are iterated over via a list. The
697                  * maximum should stay reasonable.
698                  */
699                 if (maxactive > KRETPROBE_MAXACTIVE_MAX) {
700                         pr_info("Maxactive is too big (%d > %d).\n",
701                                 maxactive, KRETPROBE_MAXACTIVE_MAX);
702                         return -E2BIG;
703                 }
704         }
705 
706         if (event) {
707                 if (strchr(event, '/')) {
708                         group = event;
709                         event = strchr(group, '/') + 1;
710                         event[-1] = '\0';
711                         if (strlen(group) == 0) {
712                                 pr_info("Group name is not specified\n");
713                                 return -EINVAL;
714                         }
715                 }
716                 if (strlen(event) == 0) {
717                         pr_info("Event name is not specified\n");
718                         return -EINVAL;
719                 }
720         }
721         if (!group)
722                 group = KPROBE_EVENT_SYSTEM;
723 
724         if (is_delete) {
725                 if (!event) {
726                         pr_info("Delete command needs an event name.\n");
727                         return -EINVAL;
728                 }
729                 mutex_lock(&probe_lock);
730                 tk = find_trace_kprobe(event, group);
731                 if (!tk) {
732                         mutex_unlock(&probe_lock);
733                         pr_info("Event %s/%s doesn't exist.\n", group, event);
734                         return -ENOENT;
735                 }
736                 /* delete an event */
737                 ret = unregister_trace_kprobe(tk);
738                 if (ret == 0)
739                         free_trace_kprobe(tk);
740                 mutex_unlock(&probe_lock);
741                 return ret;
742         }
743 
744         if (argc < 2) {
745                 pr_info("Probe point is not specified.\n");
746                 return -EINVAL;
747         }
748 
749         /* try to parse an address. if that fails, try to read the
750          * input as a symbol. */
751         if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) {
752                 /* a symbol specified */
753                 symbol = argv[1];
754                 /* TODO: support .init module functions */
755                 ret = traceprobe_split_symbol_offset(symbol, &offset);
756                 if (ret || offset < 0 || offset > UINT_MAX) {
757                         pr_info("Failed to parse either an address or a symbol.\n");
758                         return ret;
759                 }
760                 if (offset && is_return &&
761                     !kprobe_on_func_entry(NULL, symbol, offset)) {
762                         pr_info("Given offset is not valid for return probe.\n");
763                         return -EINVAL;
764                 }
765         }
766         argc -= 2; argv += 2;
767 
768         /* setup a probe */
769         if (!event) {
770                 /* Make a new event name */
771                 if (symbol)
772                         snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld",
773                                  is_return ? 'r' : 'p', symbol, offset);
774                 else
775                         snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p",
776                                  is_return ? 'r' : 'p', addr);
777                 sanitize_event_name(buf);
778                 event = buf;
779         }
780         tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive,
781                                argc, is_return);
782         if (IS_ERR(tk)) {
783                 pr_info("Failed to allocate trace_probe.(%d)\n",
784                         (int)PTR_ERR(tk));
785                 return PTR_ERR(tk);
786         }
787 
788         /* parse arguments */
789         ret = 0;
790         for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
791                 struct probe_arg *parg = &tk->tp.args[i];
792 
793                 /* Increment count for freeing args in error case */
794                 tk->tp.nr_args++;
795 
796                 /* Parse argument name */
797                 arg = strchr(argv[i], '=');
798                 if (arg) {
799                         *arg++ = '\0';
800                         parg->name = kstrdup(argv[i], GFP_KERNEL);
801                 } else {
802                         arg = argv[i];
803                         /* If argument name is omitted, set "argN" */
804                         snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1);
805                         parg->name = kstrdup(buf, GFP_KERNEL);
806                 }
807 
808                 if (!parg->name) {
809                         pr_info("Failed to allocate argument[%d] name.\n", i);
810                         ret = -ENOMEM;
811                         goto error;
812                 }
813 
814                 if (!is_good_name(parg->name)) {
815                         pr_info("Invalid argument[%d] name: %s\n",
816                                 i, parg->name);
817                         ret = -EINVAL;
818                         goto error;
819                 }
820 
821                 if (traceprobe_conflict_field_name(parg->name,
822                                                         tk->tp.args, i)) {
823                         pr_info("Argument[%d] name '%s' conflicts with "
824                                 "another field.\n", i, argv[i]);
825                         ret = -EINVAL;
826                         goto error;
827                 }
828 
829                 /* Parse fetch argument */
830                 ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg,
831                                                 is_return, true,
832                                                 kprobes_fetch_type_table);
833                 if (ret) {
834                         pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
835                         goto error;
836                 }
837         }
838 
839         ret = register_trace_kprobe(tk);
840         if (ret)
841                 goto error;
842         return 0;
843 
844 error:
845         free_trace_kprobe(tk);
846         return ret;
847 }
848 
849 static int release_all_trace_kprobes(void)
850 {
851         struct trace_kprobe *tk;
852         int ret = 0;
853 
854         mutex_lock(&probe_lock);
855         /* Ensure no probe is in use. */
856         list_for_each_entry(tk, &probe_list, list)
857                 if (trace_probe_is_enabled(&tk->tp)) {
858                         ret = -EBUSY;
859                         goto end;
860                 }
861         /* TODO: Use batch unregistration */
862         while (!list_empty(&probe_list)) {
863                 tk = list_entry(probe_list.next, struct trace_kprobe, list);
864                 ret = unregister_trace_kprobe(tk);
865                 if (ret)
866                         goto end;
867                 free_trace_kprobe(tk);
868         }
869 
870 end:
871         mutex_unlock(&probe_lock);
872 
873         return ret;
874 }
875 
876 /* Probes listing interfaces */
877 static void *probes_seq_start(struct seq_file *m, loff_t *pos)
878 {
879         mutex_lock(&probe_lock);
880         return seq_list_start(&probe_list, *pos);
881 }
882 
883 static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos)
884 {
885         return seq_list_next(v, &probe_list, pos);
886 }
887 
888 static void probes_seq_stop(struct seq_file *m, void *v)
889 {
890         mutex_unlock(&probe_lock);
891 }
892 
893 static int probes_seq_show(struct seq_file *m, void *v)
894 {
895         struct trace_kprobe *tk = v;
896         int i;
897 
898         seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p');
899         seq_printf(m, ":%s/%s", tk->tp.call.class->system,
900                         trace_event_name(&tk->tp.call));
901 
902         if (!tk->symbol)
903                 seq_printf(m, " 0x%p", tk->rp.kp.addr);
904         else if (tk->rp.kp.offset)
905                 seq_printf(m, " %s+%u", trace_kprobe_symbol(tk),
906                            tk->rp.kp.offset);
907         else
908                 seq_printf(m, " %s", trace_kprobe_symbol(tk));
909 
910         for (i = 0; i < tk->tp.nr_args; i++)
911                 seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm);
912         seq_putc(m, '\n');
913 
914         return 0;
915 }
916 
917 static const struct seq_operations probes_seq_op = {
918         .start  = probes_seq_start,
919         .next   = probes_seq_next,
920         .stop   = probes_seq_stop,
921         .show   = probes_seq_show
922 };
923 
924 static int probes_open(struct inode *inode, struct file *file)
925 {
926         int ret;
927 
928         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
929                 ret = release_all_trace_kprobes();
930                 if (ret < 0)
931                         return ret;
932         }
933 
934         return seq_open(file, &probes_seq_op);
935 }
936 
937 static ssize_t probes_write(struct file *file, const char __user *buffer,
938                             size_t count, loff_t *ppos)
939 {
940         return trace_parse_run_command(file, buffer, count, ppos,
941                                        create_trace_kprobe);
942 }
943 
944 static const struct file_operations kprobe_events_ops = {
945         .owner          = THIS_MODULE,
946         .open           = probes_open,
947         .read           = seq_read,
948         .llseek         = seq_lseek,
949         .release        = seq_release,
950         .write          = probes_write,
951 };
952 
953 /* Probes profiling interfaces */
954 static int probes_profile_seq_show(struct seq_file *m, void *v)
955 {
956         struct trace_kprobe *tk = v;
957 
958         seq_printf(m, "  %-44s %15lu %15lu\n",
959                    trace_event_name(&tk->tp.call),
960                    trace_kprobe_nhit(tk),
961                    tk->rp.kp.nmissed);
962 
963         return 0;
964 }
965 
966 static const struct seq_operations profile_seq_op = {
967         .start  = probes_seq_start,
968         .next   = probes_seq_next,
969         .stop   = probes_seq_stop,
970         .show   = probes_profile_seq_show
971 };
972 
973 static int profile_open(struct inode *inode, struct file *file)
974 {
975         return seq_open(file, &profile_seq_op);
976 }
977 
978 static const struct file_operations kprobe_profile_ops = {
979         .owner          = THIS_MODULE,
980         .open           = profile_open,
981         .read           = seq_read,
982         .llseek         = seq_lseek,
983         .release        = seq_release,
984 };
985 
986 /* Kprobe handler */
987 static nokprobe_inline void
988 __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
989                     struct trace_event_file *trace_file)
990 {
991         struct kprobe_trace_entry_head *entry;
992         struct ring_buffer_event *event;
993         struct ring_buffer *buffer;
994         int size, dsize, pc;
995         unsigned long irq_flags;
996         struct trace_event_call *call = &tk->tp.call;
997 
998         WARN_ON(call != trace_file->event_call);
999 
1000         if (trace_trigger_soft_disabled(trace_file))
1001                 return;
1002 
1003         local_save_flags(irq_flags);
1004         pc = preempt_count();
1005 
1006         dsize = __get_data_size(&tk->tp, regs);
1007         size = sizeof(*entry) + tk->tp.size + dsize;
1008 
1009         event = trace_event_buffer_lock_reserve(&buffer, trace_file,
1010                                                 call->event.type,
1011                                                 size, irq_flags, pc);
1012         if (!event)
1013                 return;
1014 
1015         entry = ring_buffer_event_data(event);
1016         entry->ip = (unsigned long)tk->rp.kp.addr;
1017         store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1018 
1019         event_trigger_unlock_commit_regs(trace_file, buffer, event,
1020                                          entry, irq_flags, pc, regs);
1021 }
1022 
1023 static void
1024 kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs)
1025 {
1026         struct event_file_link *link;
1027 
1028         list_for_each_entry_rcu(link, &tk->tp.files, list)
1029                 __kprobe_trace_func(tk, regs, link->file);
1030 }
1031 NOKPROBE_SYMBOL(kprobe_trace_func);
1032 
1033 /* Kretprobe handler */
1034 static nokprobe_inline void
1035 __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1036                        struct pt_regs *regs,
1037                        struct trace_event_file *trace_file)
1038 {
1039         struct kretprobe_trace_entry_head *entry;
1040         struct ring_buffer_event *event;
1041         struct ring_buffer *buffer;
1042         int size, pc, dsize;
1043         unsigned long irq_flags;
1044         struct trace_event_call *call = &tk->tp.call;
1045 
1046         WARN_ON(call != trace_file->event_call);
1047 
1048         if (trace_trigger_soft_disabled(trace_file))
1049                 return;
1050 
1051         local_save_flags(irq_flags);
1052         pc = preempt_count();
1053 
1054         dsize = __get_data_size(&tk->tp, regs);
1055         size = sizeof(*entry) + tk->tp.size + dsize;
1056 
1057         event = trace_event_buffer_lock_reserve(&buffer, trace_file,
1058                                                 call->event.type,
1059                                                 size, irq_flags, pc);
1060         if (!event)
1061                 return;
1062 
1063         entry = ring_buffer_event_data(event);
1064         entry->func = (unsigned long)tk->rp.kp.addr;
1065         entry->ret_ip = (unsigned long)ri->ret_addr;
1066         store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1067 
1068         event_trigger_unlock_commit_regs(trace_file, buffer, event,
1069                                          entry, irq_flags, pc, regs);
1070 }
1071 
1072 static void
1073 kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1074                      struct pt_regs *regs)
1075 {
1076         struct event_file_link *link;
1077 
1078         list_for_each_entry_rcu(link, &tk->tp.files, list)
1079                 __kretprobe_trace_func(tk, ri, regs, link->file);
1080 }
1081 NOKPROBE_SYMBOL(kretprobe_trace_func);
1082 
1083 /* Event entry printers */
1084 static enum print_line_t
1085 print_kprobe_event(struct trace_iterator *iter, int flags,
1086                    struct trace_event *event)
1087 {
1088         struct kprobe_trace_entry_head *field;
1089         struct trace_seq *s = &iter->seq;
1090         struct trace_probe *tp;
1091         u8 *data;
1092         int i;
1093 
1094         field = (struct kprobe_trace_entry_head *)iter->ent;
1095         tp = container_of(event, struct trace_probe, call.event);
1096 
1097         trace_seq_printf(s, "%s: (", trace_event_name(&tp->call));
1098 
1099         if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))
1100                 goto out;
1101 
1102         trace_seq_putc(s, ')');
1103 
1104         data = (u8 *)&field[1];
1105         for (i = 0; i < tp->nr_args; i++)
1106                 if (!tp->args[i].type->print(s, tp->args[i].name,
1107                                              data + tp->args[i].offset, field))
1108                         goto out;
1109 
1110         trace_seq_putc(s, '\n');
1111  out:
1112         return trace_handle_return(s);
1113 }
1114 
1115 static enum print_line_t
1116 print_kretprobe_event(struct trace_iterator *iter, int flags,
1117                       struct trace_event *event)
1118 {
1119         struct kretprobe_trace_entry_head *field;
1120         struct trace_seq *s = &iter->seq;
1121         struct trace_probe *tp;
1122         u8 *data;
1123         int i;
1124 
1125         field = (struct kretprobe_trace_entry_head *)iter->ent;
1126         tp = container_of(event, struct trace_probe, call.event);
1127 
1128         trace_seq_printf(s, "%s: (", trace_event_name(&tp->call));
1129 
1130         if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET))
1131                 goto out;
1132 
1133         trace_seq_puts(s, " <- ");
1134 
1135         if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET))
1136                 goto out;
1137 
1138         trace_seq_putc(s, ')');
1139 
1140         data = (u8 *)&field[1];
1141         for (i = 0; i < tp->nr_args; i++)
1142                 if (!tp->args[i].type->print(s, tp->args[i].name,
1143                                              data + tp->args[i].offset, field))
1144                         goto out;
1145 
1146         trace_seq_putc(s, '\n');
1147 
1148  out:
1149         return trace_handle_return(s);
1150 }
1151 
1152 
1153 static int kprobe_event_define_fields(struct trace_event_call *event_call)
1154 {
1155         int ret, i;
1156         struct kprobe_trace_entry_head field;
1157         struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;
1158 
1159         DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
1160         /* Set argument names as fields */
1161         for (i = 0; i < tk->tp.nr_args; i++) {
1162                 struct probe_arg *parg = &tk->tp.args[i];
1163 
1164                 ret = trace_define_field(event_call, parg->type->fmttype,
1165                                          parg->name,
1166                                          sizeof(field) + parg->offset,
1167                                          parg->type->size,
1168                                          parg->type->is_signed,
1169                                          FILTER_OTHER);
1170                 if (ret)
1171                         return ret;
1172         }
1173         return 0;
1174 }
1175 
1176 static int kretprobe_event_define_fields(struct trace_event_call *event_call)
1177 {
1178         int ret, i;
1179         struct kretprobe_trace_entry_head field;
1180         struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;
1181 
1182         DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
1183         DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
1184         /* Set argument names as fields */
1185         for (i = 0; i < tk->tp.nr_args; i++) {
1186                 struct probe_arg *parg = &tk->tp.args[i];
1187 
1188                 ret = trace_define_field(event_call, parg->type->fmttype,
1189                                          parg->name,
1190                                          sizeof(field) + parg->offset,
1191                                          parg->type->size,
1192                                          parg->type->is_signed,
1193                                          FILTER_OTHER);
1194                 if (ret)
1195                         return ret;
1196         }
1197         return 0;
1198 }
1199 
1200 #ifdef CONFIG_PERF_EVENTS
1201 
1202 /* Kprobe profile handler */
1203 static int
1204 kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
1205 {
1206         struct trace_event_call *call = &tk->tp.call;
1207         struct kprobe_trace_entry_head *entry;
1208         struct hlist_head *head;
1209         int size, __size, dsize;
1210         int rctx;
1211 
1212         if (bpf_prog_array_valid(call)) {
1213                 unsigned long orig_ip = instruction_pointer(regs);
1214                 int ret;
1215 
1216                 ret = trace_call_bpf(call, regs);
1217 
1218                 /*
1219                  * We need to check and see if we modified the pc of the
1220                  * pt_regs, and if so clear the kprobe and return 1 so that we
1221                  * don't do the single stepping.
1222                  * The ftrace kprobe handler leaves it up to us to re-enable
1223                  * preemption here before returning if we've modified the ip.
1224                  */
1225                 if (orig_ip != instruction_pointer(regs)) {
1226                         reset_current_kprobe();
1227                         preempt_enable_no_resched();
1228                         return 1;
1229                 }
1230                 if (!ret)
1231                         return 0;
1232         }
1233 
1234         head = this_cpu_ptr(call->perf_events);
1235         if (hlist_empty(head))
1236                 return 0;
1237 
1238         dsize = __get_data_size(&tk->tp, regs);
1239         __size = sizeof(*entry) + tk->tp.size + dsize;
1240         size = ALIGN(__size + sizeof(u32), sizeof(u64));
1241         size -= sizeof(u32);
1242 
1243         entry = perf_trace_buf_alloc(size, NULL, &rctx);
1244         if (!entry)
1245                 return 0;
1246 
1247         entry->ip = (unsigned long)tk->rp.kp.addr;
1248         memset(&entry[1], 0, dsize);
1249         store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1250         perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1251                               head, NULL);
1252         return 0;
1253 }
1254 NOKPROBE_SYMBOL(kprobe_perf_func);
1255 
1256 /* Kretprobe profile handler */
1257 static void
1258 kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1259                     struct pt_regs *regs)
1260 {
1261         struct trace_event_call *call = &tk->tp.call;
1262         struct kretprobe_trace_entry_head *entry;
1263         struct hlist_head *head;
1264         int size, __size, dsize;
1265         int rctx;
1266 
1267         if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs))
1268                 return;
1269 
1270         head = this_cpu_ptr(call->perf_events);
1271         if (hlist_empty(head))
1272                 return;
1273 
1274         dsize = __get_data_size(&tk->tp, regs);
1275         __size = sizeof(*entry) + tk->tp.size + dsize;
1276         size = ALIGN(__size + sizeof(u32), sizeof(u64));
1277         size -= sizeof(u32);
1278 
1279         entry = perf_trace_buf_alloc(size, NULL, &rctx);
1280         if (!entry)
1281                 return;
1282 
1283         entry->func = (unsigned long)tk->rp.kp.addr;
1284         entry->ret_ip = (unsigned long)ri->ret_addr;
1285         store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1286         perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1287                               head, NULL);
1288 }
1289 NOKPROBE_SYMBOL(kretprobe_perf_func);
1290 #endif  /* CONFIG_PERF_EVENTS */
1291 
1292 /*
1293  * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
1294  *
1295  * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe
1296  * lockless, but we can't race with this __init function.
1297  */
1298 static int kprobe_register(struct trace_event_call *event,
1299                            enum trace_reg type, void *data)
1300 {
1301         struct trace_kprobe *tk = (struct trace_kprobe *)event->data;
1302         struct trace_event_file *file = data;
1303 
1304         switch (type) {
1305         case TRACE_REG_REGISTER:
1306                 return enable_trace_kprobe(tk, file);
1307         case TRACE_REG_UNREGISTER:
1308                 return disable_trace_kprobe(tk, file);
1309 
1310 #ifdef CONFIG_PERF_EVENTS
1311         case TRACE_REG_PERF_REGISTER:
1312                 return enable_trace_kprobe(tk, NULL);
1313         case TRACE_REG_PERF_UNREGISTER:
1314                 return disable_trace_kprobe(tk, NULL);
1315         case TRACE_REG_PERF_OPEN:
1316         case TRACE_REG_PERF_CLOSE:
1317         case TRACE_REG_PERF_ADD:
1318         case TRACE_REG_PERF_DEL:
1319                 return 0;
1320 #endif
1321         }
1322         return 0;
1323 }
1324 
1325 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
1326 {
1327         struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp);
1328         int ret = 0;
1329 
1330         raw_cpu_inc(*tk->nhit);
1331 
1332         if (tk->tp.flags & TP_FLAG_TRACE)
1333                 kprobe_trace_func(tk, regs);
1334 #ifdef CONFIG_PERF_EVENTS
1335         if (tk->tp.flags & TP_FLAG_PROFILE)
1336                 ret = kprobe_perf_func(tk, regs);
1337 #endif
1338         return ret;
1339 }
1340 NOKPROBE_SYMBOL(kprobe_dispatcher);
1341 
1342 static int
1343 kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
1344 {
1345         struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp);
1346 
1347         raw_cpu_inc(*tk->nhit);
1348 
1349         if (tk->tp.flags & TP_FLAG_TRACE)
1350                 kretprobe_trace_func(tk, ri, regs);
1351 #ifdef CONFIG_PERF_EVENTS
1352         if (tk->tp.flags & TP_FLAG_PROFILE)
1353                 kretprobe_perf_func(tk, ri, regs);
1354 #endif
1355         return 0;       /* We don't tweek kernel, so just return 0 */
1356 }
1357 NOKPROBE_SYMBOL(kretprobe_dispatcher);
1358 
1359 static struct trace_event_functions kretprobe_funcs = {
1360         .trace          = print_kretprobe_event
1361 };
1362 
1363 static struct trace_event_functions kprobe_funcs = {
1364         .trace          = print_kprobe_event
1365 };
1366 
1367 static inline void init_trace_event_call(struct trace_kprobe *tk,
1368                                          struct trace_event_call *call)
1369 {
1370         INIT_LIST_HEAD(&call->class->fields);
1371         if (trace_kprobe_is_return(tk)) {
1372                 call->event.funcs = &kretprobe_funcs;
1373                 call->class->define_fields = kretprobe_event_define_fields;
1374         } else {
1375                 call->event.funcs = &kprobe_funcs;
1376                 call->class->define_fields = kprobe_event_define_fields;
1377         }
1378 
1379         call->flags = TRACE_EVENT_FL_KPROBE;
1380         call->class->reg = kprobe_register;
1381         call->data = tk;
1382 }
1383 
1384 static int register_kprobe_event(struct trace_kprobe *tk)
1385 {
1386         struct trace_event_call *call = &tk->tp.call;
1387         int ret = 0;
1388 
1389         init_trace_event_call(tk, call);
1390 
1391         if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0)
1392                 return -ENOMEM;
1393         ret = register_trace_event(&call->event);
1394         if (!ret) {
1395                 kfree(call->print_fmt);
1396                 return -ENODEV;
1397         }
1398         ret = trace_add_event_call(call);
1399         if (ret) {
1400                 pr_info("Failed to register kprobe event: %s\n",
1401                         trace_event_name(call));
1402                 kfree(call->print_fmt);
1403                 unregister_trace_event(&call->event);
1404         }
1405         return ret;
1406 }
1407 
1408 static int unregister_kprobe_event(struct trace_kprobe *tk)
1409 {
1410         int ret;
1411 
1412         /* tp->event is unregistered in trace_remove_event_call() */
1413         ret = trace_remove_event_call(&tk->tp.call);
1414         if (!ret)
1415                 kfree(tk->tp.call.print_fmt);
1416         return ret;
1417 }
1418 
1419 #ifdef CONFIG_PERF_EVENTS
1420 /* create a trace_kprobe, but don't add it to global lists */
1421 struct trace_event_call *
1422 create_local_trace_kprobe(char *func, void *addr, unsigned long offs,
1423                           bool is_return)
1424 {
1425         struct trace_kprobe *tk;
1426         int ret;
1427         char *event;
1428 
1429         /*
1430          * local trace_kprobes are not added to probe_list, so they are never
1431          * searched in find_trace_kprobe(). Therefore, there is no concern of
1432          * duplicated name here.
1433          */
1434         event = func ? func : "DUMMY_EVENT";
1435 
1436         tk = alloc_trace_kprobe(KPROBE_EVENT_SYSTEM, event, (void *)addr, func,
1437                                 offs, 0 /* maxactive */, 0 /* nargs */,
1438                                 is_return);
1439 
1440         if (IS_ERR(tk)) {
1441                 pr_info("Failed to allocate trace_probe.(%d)\n",
1442                         (int)PTR_ERR(tk));
1443                 return ERR_CAST(tk);
1444         }
1445 
1446         init_trace_event_call(tk, &tk->tp.call);
1447 
1448         if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) {
1449                 ret = -ENOMEM;
1450                 goto error;
1451         }
1452 
1453         ret = __register_trace_kprobe(tk);
1454         if (ret < 0)
1455                 goto error;
1456 
1457         return &tk->tp.call;
1458 error:
1459         free_trace_kprobe(tk);
1460         return ERR_PTR(ret);
1461 }
1462 
1463 void destroy_local_trace_kprobe(struct trace_event_call *event_call)
1464 {
1465         struct trace_kprobe *tk;
1466 
1467         tk = container_of(event_call, struct trace_kprobe, tp.call);
1468 
1469         if (trace_probe_is_enabled(&tk->tp)) {
1470                 WARN_ON(1);
1471                 return;
1472         }
1473 
1474         __unregister_trace_kprobe(tk);
1475         free_trace_kprobe(tk);
1476 }
1477 #endif /* CONFIG_PERF_EVENTS */
1478 
1479 /* Make a tracefs interface for controlling probe points */
1480 static __init int init_kprobe_trace(void)
1481 {
1482         struct dentry *d_tracer;
1483         struct dentry *entry;
1484 
1485         if (register_module_notifier(&trace_kprobe_module_nb))
1486                 return -EINVAL;
1487 
1488         d_tracer = tracing_init_dentry();
1489         if (IS_ERR(d_tracer))
1490                 return 0;
1491 
1492         entry = tracefs_create_file("kprobe_events", 0644, d_tracer,
1493                                     NULL, &kprobe_events_ops);
1494 
1495         /* Event list interface */
1496         if (!entry)
1497                 pr_warn("Could not create tracefs 'kprobe_events' entry\n");
1498 
1499         /* Profile interface */
1500         entry = tracefs_create_file("kprobe_profile", 0444, d_tracer,
1501                                     NULL, &kprobe_profile_ops);
1502 
1503         if (!entry)
1504                 pr_warn("Could not create tracefs 'kprobe_profile' entry\n");
1505         return 0;
1506 }
1507 fs_initcall(init_kprobe_trace);
1508 
1509 
1510 #ifdef CONFIG_FTRACE_STARTUP_TEST
1511 /*
1512  * The "__used" keeps gcc from removing the function symbol
1513  * from the kallsyms table. 'noinline' makes sure that there
1514  * isn't an inlined version used by the test method below
1515  */
1516 static __used __init noinline int
1517 kprobe_trace_selftest_target(int a1, int a2, int a3, int a4, int a5, int a6)
1518 {
1519         return a1 + a2 + a3 + a4 + a5 + a6;
1520 }
1521 
1522 static __init struct trace_event_file *
1523 find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr)
1524 {
1525         struct trace_event_file *file;
1526 
1527         list_for_each_entry(file, &tr->events, list)
1528                 if (file->event_call == &tk->tp.call)
1529                         return file;
1530 
1531         return NULL;
1532 }
1533 
1534 /*
1535  * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this
1536  * stage, we can do this lockless.
1537  */
1538 static __init int kprobe_trace_self_tests_init(void)
1539 {
1540         int ret, warn = 0;
1541         int (*target)(int, int, int, int, int, int);
1542         struct trace_kprobe *tk;
1543         struct trace_event_file *file;
1544 
1545         if (tracing_is_disabled())
1546                 return -ENODEV;
1547 
1548         target = kprobe_trace_selftest_target;
1549 
1550         pr_info("Testing kprobe tracing: ");
1551 
1552         ret = trace_run_command("p:testprobe kprobe_trace_selftest_target "
1553                                 "$stack $stack0 +0($stack)",
1554                                 create_trace_kprobe);
1555         if (WARN_ON_ONCE(ret)) {
1556                 pr_warn("error on probing function entry.\n");
1557                 warn++;
1558         } else {
1559                 /* Enable trace point */
1560                 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
1561                 if (WARN_ON_ONCE(tk == NULL)) {
1562                         pr_warn("error on getting new probe.\n");
1563                         warn++;
1564                 } else {
1565                         file = find_trace_probe_file(tk, top_trace_array());
1566                         if (WARN_ON_ONCE(file == NULL)) {
1567                                 pr_warn("error on getting probe file.\n");
1568                                 warn++;
1569                         } else
1570                                 enable_trace_kprobe(tk, file);
1571                 }
1572         }
1573 
1574         ret = trace_run_command("r:testprobe2 kprobe_trace_selftest_target "
1575                                 "$retval", create_trace_kprobe);
1576         if (WARN_ON_ONCE(ret)) {
1577                 pr_warn("error on probing function return.\n");
1578                 warn++;
1579         } else {
1580                 /* Enable trace point */
1581                 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
1582                 if (WARN_ON_ONCE(tk == NULL)) {
1583                         pr_warn("error on getting 2nd new probe.\n");
1584                         warn++;
1585                 } else {
1586                         file = find_trace_probe_file(tk, top_trace_array());
1587                         if (WARN_ON_ONCE(file == NULL)) {
1588                                 pr_warn("error on getting probe file.\n");
1589                                 warn++;
1590                         } else
1591                                 enable_trace_kprobe(tk, file);
1592                 }
1593         }
1594 
1595         if (warn)
1596                 goto end;
1597 
1598         ret = target(1, 2, 3, 4, 5, 6);
1599 
1600         /*
1601          * Not expecting an error here, the check is only to prevent the
1602          * optimizer from removing the call to target() as otherwise there
1603          * are no side-effects and the call is never performed.
1604          */
1605         if (ret != 21)
1606                 warn++;
1607 
1608         /* Disable trace points before removing it */
1609         tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
1610         if (WARN_ON_ONCE(tk == NULL)) {
1611                 pr_warn("error on getting test probe.\n");
1612                 warn++;
1613         } else {
1614                 if (trace_kprobe_nhit(tk) != 1) {
1615                         pr_warn("incorrect number of testprobe hits\n");
1616                         warn++;
1617                 }
1618 
1619                 file = find_trace_probe_file(tk, top_trace_array());
1620                 if (WARN_ON_ONCE(file == NULL)) {
1621                         pr_warn("error on getting probe file.\n");
1622                         warn++;
1623                 } else
1624                         disable_trace_kprobe(tk, file);
1625         }
1626 
1627         tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
1628         if (WARN_ON_ONCE(tk == NULL)) {
1629                 pr_warn("error on getting 2nd test probe.\n");
1630                 warn++;
1631         } else {
1632                 if (trace_kprobe_nhit(tk) != 1) {
1633                         pr_warn("incorrect number of testprobe2 hits\n");
1634                         warn++;
1635                 }
1636 
1637                 file = find_trace_probe_file(tk, top_trace_array());
1638                 if (WARN_ON_ONCE(file == NULL)) {
1639                         pr_warn("error on getting probe file.\n");
1640                         warn++;
1641                 } else
1642                         disable_trace_kprobe(tk, file);
1643         }
1644 
1645         ret = trace_run_command("-:testprobe", create_trace_kprobe);
1646         if (WARN_ON_ONCE(ret)) {
1647                 pr_warn("error on deleting a probe.\n");
1648                 warn++;
1649         }
1650 
1651         ret = trace_run_command("-:testprobe2", create_trace_kprobe);
1652         if (WARN_ON_ONCE(ret)) {
1653                 pr_warn("error on deleting a probe.\n");
1654                 warn++;
1655         }
1656 
1657 end:
1658         release_all_trace_kprobes();
1659         /*
1660          * Wait for the optimizer work to finish. Otherwise it might fiddle
1661          * with probes in already freed __init text.
1662          */
1663         wait_for_kprobe_optimizer();
1664         if (warn)
1665                 pr_cont("NG: Some tests are failed. Please check them.\n");
1666         else
1667                 pr_cont("OK\n");
1668         return 0;
1669 }
1670 
1671 late_initcall(kprobe_trace_self_tests_init);
1672 
1673 #endif
1674 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp