~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/powerpc/kernel/ftrace.c

Version: ~ [ linux-5.0-rc6 ] ~ [ linux-4.20.10 ] ~ [ linux-4.19.23 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.101 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.158 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.174 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.134 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.63 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Code for replacing ftrace calls with jumps.
  3  *
  4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
  5  *
  6  * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box.
  7  *
  8  * Added function graph tracer code, taken from x86 that was written
  9  * by Frederic Weisbecker, and ported to PPC by Steven Rostedt.
 10  *
 11  */
 12 
 13 #include <linux/spinlock.h>
 14 #include <linux/hardirq.h>
 15 #include <linux/uaccess.h>
 16 #include <linux/module.h>
 17 #include <linux/ftrace.h>
 18 #include <linux/percpu.h>
 19 #include <linux/init.h>
 20 #include <linux/list.h>
 21 
 22 #include <asm/cacheflush.h>
 23 #include <asm/code-patching.h>
 24 #include <asm/ftrace.h>
 25 #include <asm/syscall.h>
 26 
 27 
 28 #ifdef CONFIG_DYNAMIC_FTRACE
 29 static unsigned int
 30 ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
 31 {
 32         unsigned int op;
 33 
 34         addr = ppc_function_entry((void *)addr);
 35 
 36         /* if (link) set op to 'bl' else 'b' */
 37         op = create_branch((unsigned int *)ip, addr, link ? 1 : 0);
 38 
 39         return op;
 40 }
 41 
 42 static int
 43 ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
 44 {
 45         unsigned int replaced;
 46 
 47         /*
 48          * Note: Due to modules and __init, code can
 49          *  disappear and change, we need to protect against faulting
 50          *  as well as code changing. We do this by using the
 51          *  probe_kernel_* functions.
 52          *
 53          * No real locking needed, this code is run through
 54          * kstop_machine, or before SMP starts.
 55          */
 56 
 57         /* read the text we want to modify */
 58         if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE))
 59                 return -EFAULT;
 60 
 61         /* Make sure it is what we expect it to be */
 62         if (replaced != old)
 63                 return -EINVAL;
 64 
 65         /* replace the text with the new text */
 66         if (probe_kernel_write((void *)ip, &new, MCOUNT_INSN_SIZE))
 67                 return -EPERM;
 68 
 69         flush_icache_range(ip, ip + 8);
 70 
 71         return 0;
 72 }
 73 
 74 /*
 75  * Helper functions that are the same for both PPC64 and PPC32.
 76  */
 77 static int test_24bit_addr(unsigned long ip, unsigned long addr)
 78 {
 79 
 80         /* use the create_branch to verify that this offset can be branched */
 81         return create_branch((unsigned int *)ip, addr, 0);
 82 }
 83 
 84 #ifdef CONFIG_MODULES
 85 
 86 static int is_bl_op(unsigned int op)
 87 {
 88         return (op & 0xfc000003) == 0x48000001;
 89 }
 90 
 91 static unsigned long find_bl_target(unsigned long ip, unsigned int op)
 92 {
 93         static int offset;
 94 
 95         offset = (op & 0x03fffffc);
 96         /* make it signed */
 97         if (offset & 0x02000000)
 98                 offset |= 0xfe000000;
 99 
100         return ip + (long)offset;
101 }
102 
103 #ifdef CONFIG_PPC64
104 static int
105 __ftrace_make_nop(struct module *mod,
106                   struct dyn_ftrace *rec, unsigned long addr)
107 {
108         unsigned int op;
109         unsigned int jmp[5];
110         unsigned long ptr;
111         unsigned long ip = rec->ip;
112         unsigned long tramp;
113         int offset;
114 
115         /* read where this goes */
116         if (probe_kernel_read(&op, (void *)ip, sizeof(int)))
117                 return -EFAULT;
118 
119         /* Make sure that that this is still a 24bit jump */
120         if (!is_bl_op(op)) {
121                 printk(KERN_ERR "Not expected bl: opcode is %x\n", op);
122                 return -EINVAL;
123         }
124 
125         /* lets find where the pointer goes */
126         tramp = find_bl_target(ip, op);
127 
128         /*
129          * On PPC64 the trampoline looks like:
130          * 0x3d, 0x82, 0x00, 0x00,    addis   r12,r2, <high>
131          * 0x39, 0x8c, 0x00, 0x00,    addi    r12,r12, <low>
132          *   Where the bytes 2,3,6 and 7 make up the 32bit offset
133          *   to the TOC that holds the pointer.
134          *   to jump to.
135          * 0xf8, 0x41, 0x00, 0x28,    std     r2,40(r1)
136          * 0xe9, 0x6c, 0x00, 0x20,    ld      r11,32(r12)
137          *   The actually address is 32 bytes from the offset
138          *   into the TOC.
139          * 0xe8, 0x4c, 0x00, 0x28,    ld      r2,40(r12)
140          */
141 
142         pr_devel("ip:%lx jumps to %lx r2: %lx", ip, tramp, mod->arch.toc);
143 
144         /* Find where the trampoline jumps to */
145         if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
146                 printk(KERN_ERR "Failed to read %lx\n", tramp);
147                 return -EFAULT;
148         }
149 
150         pr_devel(" %08x %08x", jmp[0], jmp[1]);
151 
152         /* verify that this is what we expect it to be */
153         if (((jmp[0] & 0xffff0000) != 0x3d820000) ||
154             ((jmp[1] & 0xffff0000) != 0x398c0000) ||
155             (jmp[2] != 0xf8410028) ||
156             (jmp[3] != 0xe96c0020) ||
157             (jmp[4] != 0xe84c0028)) {
158                 printk(KERN_ERR "Not a trampoline\n");
159                 return -EINVAL;
160         }
161 
162         /* The bottom half is signed extended */
163         offset = ((unsigned)((unsigned short)jmp[0]) << 16) +
164                 (int)((short)jmp[1]);
165 
166         pr_devel(" %x ", offset);
167 
168         /* get the address this jumps too */
169         tramp = mod->arch.toc + offset + 32;
170         pr_devel("toc: %lx", tramp);
171 
172         if (probe_kernel_read(jmp, (void *)tramp, 8)) {
173                 printk(KERN_ERR "Failed to read %lx\n", tramp);
174                 return -EFAULT;
175         }
176 
177         pr_devel(" %08x %08x\n", jmp[0], jmp[1]);
178 
179         ptr = ((unsigned long)jmp[0] << 32) + jmp[1];
180 
181         /* This should match what was called */
182         if (ptr != ppc_function_entry((void *)addr)) {
183                 printk(KERN_ERR "addr does not match %lx\n", ptr);
184                 return -EINVAL;
185         }
186 
187         /*
188          * We want to nop the line, but the next line is
189          *  0xe8, 0x41, 0x00, 0x28   ld r2,40(r1)
190          * This needs to be turned to a nop too.
191          */
192         if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE))
193                 return -EFAULT;
194 
195         if (op != 0xe8410028) {
196                 printk(KERN_ERR "Next line is not ld! (%08x)\n", op);
197                 return -EINVAL;
198         }
199 
200         /*
201          * Milton Miller pointed out that we can not blindly do nops.
202          * If a task was preempted when calling a trace function,
203          * the nops will remove the way to restore the TOC in r2
204          * and the r2 TOC will get corrupted.
205          */
206 
207         /*
208          * Replace:
209          *   bl <tramp>  <==== will be replaced with "b 1f"
210          *   ld r2,40(r1)
211          *  1:
212          */
213         op = 0x48000008;        /* b +8 */
214 
215         if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
216                 return -EPERM;
217 
218 
219         flush_icache_range(ip, ip + 8);
220 
221         return 0;
222 }
223 
224 #else /* !PPC64 */
225 static int
226 __ftrace_make_nop(struct module *mod,
227                   struct dyn_ftrace *rec, unsigned long addr)
228 {
229         unsigned int op;
230         unsigned int jmp[4];
231         unsigned long ip = rec->ip;
232         unsigned long tramp;
233 
234         if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
235                 return -EFAULT;
236 
237         /* Make sure that that this is still a 24bit jump */
238         if (!is_bl_op(op)) {
239                 printk(KERN_ERR "Not expected bl: opcode is %x\n", op);
240                 return -EINVAL;
241         }
242 
243         /* lets find where the pointer goes */
244         tramp = find_bl_target(ip, op);
245 
246         /*
247          * On PPC32 the trampoline looks like:
248          *  0x3d, 0x80, 0x00, 0x00  lis r12,sym@ha
249          *  0x39, 0x8c, 0x00, 0x00  addi r12,r12,sym@l
250          *  0x7d, 0x89, 0x03, 0xa6  mtctr r12
251          *  0x4e, 0x80, 0x04, 0x20  bctr
252          */
253 
254         pr_devel("ip:%lx jumps to %lx", ip, tramp);
255 
256         /* Find where the trampoline jumps to */
257         if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
258                 printk(KERN_ERR "Failed to read %lx\n", tramp);
259                 return -EFAULT;
260         }
261 
262         pr_devel(" %08x %08x ", jmp[0], jmp[1]);
263 
264         /* verify that this is what we expect it to be */
265         if (((jmp[0] & 0xffff0000) != 0x3d800000) ||
266             ((jmp[1] & 0xffff0000) != 0x398c0000) ||
267             (jmp[2] != 0x7d8903a6) ||
268             (jmp[3] != 0x4e800420)) {
269                 printk(KERN_ERR "Not a trampoline\n");
270                 return -EINVAL;
271         }
272 
273         tramp = (jmp[1] & 0xffff) |
274                 ((jmp[0] & 0xffff) << 16);
275         if (tramp & 0x8000)
276                 tramp -= 0x10000;
277 
278         pr_devel(" %lx ", tramp);
279 
280         if (tramp != addr) {
281                 printk(KERN_ERR
282                        "Trampoline location %08lx does not match addr\n",
283                        tramp);
284                 return -EINVAL;
285         }
286 
287         op = PPC_INST_NOP;
288 
289         if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
290                 return -EPERM;
291 
292         flush_icache_range(ip, ip + 8);
293 
294         return 0;
295 }
296 #endif /* PPC64 */
297 #endif /* CONFIG_MODULES */
298 
299 int ftrace_make_nop(struct module *mod,
300                     struct dyn_ftrace *rec, unsigned long addr)
301 {
302         unsigned long ip = rec->ip;
303         unsigned int old, new;
304 
305         /*
306          * If the calling address is more that 24 bits away,
307          * then we had to use a trampoline to make the call.
308          * Otherwise just update the call site.
309          */
310         if (test_24bit_addr(ip, addr)) {
311                 /* within range */
312                 old = ftrace_call_replace(ip, addr, 1);
313                 new = PPC_INST_NOP;
314                 return ftrace_modify_code(ip, old, new);
315         }
316 
317 #ifdef CONFIG_MODULES
318         /*
319          * Out of range jumps are called from modules.
320          * We should either already have a pointer to the module
321          * or it has been passed in.
322          */
323         if (!rec->arch.mod) {
324                 if (!mod) {
325                         printk(KERN_ERR "No module loaded addr=%lx\n",
326                                addr);
327                         return -EFAULT;
328                 }
329                 rec->arch.mod = mod;
330         } else if (mod) {
331                 if (mod != rec->arch.mod) {
332                         printk(KERN_ERR
333                                "Record mod %p not equal to passed in mod %p\n",
334                                rec->arch.mod, mod);
335                         return -EINVAL;
336                 }
337                 /* nothing to do if mod == rec->arch.mod */
338         } else
339                 mod = rec->arch.mod;
340 
341         return __ftrace_make_nop(mod, rec, addr);
342 #else
343         /* We should not get here without modules */
344         return -EINVAL;
345 #endif /* CONFIG_MODULES */
346 }
347 
348 #ifdef CONFIG_MODULES
349 #ifdef CONFIG_PPC64
350 static int
351 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
352 {
353         unsigned int op[2];
354         unsigned long ip = rec->ip;
355 
356         /* read where this goes */
357         if (probe_kernel_read(op, (void *)ip, MCOUNT_INSN_SIZE * 2))
358                 return -EFAULT;
359 
360         /*
361          * It should be pointing to two nops or
362          *  b +8; ld r2,40(r1)
363          */
364         if (((op[0] != 0x48000008) || (op[1] != 0xe8410028)) &&
365             ((op[0] != PPC_INST_NOP) || (op[1] != PPC_INST_NOP))) {
366                 printk(KERN_ERR "Expected NOPs but have %x %x\n", op[0], op[1]);
367                 return -EINVAL;
368         }
369 
370         /* If we never set up a trampoline to ftrace_caller, then bail */
371         if (!rec->arch.mod->arch.tramp) {
372                 printk(KERN_ERR "No ftrace trampoline\n");
373                 return -EINVAL;
374         }
375 
376         /* create the branch to the trampoline */
377         op[0] = create_branch((unsigned int *)ip,
378                               rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
379         if (!op[0]) {
380                 printk(KERN_ERR "REL24 out of range!\n");
381                 return -EINVAL;
382         }
383 
384         /* ld r2,40(r1) */
385         op[1] = 0xe8410028;
386 
387         pr_devel("write to %lx\n", rec->ip);
388 
389         if (probe_kernel_write((void *)ip, op, MCOUNT_INSN_SIZE * 2))
390                 return -EPERM;
391 
392         flush_icache_range(ip, ip + 8);
393 
394         return 0;
395 }
396 #else
397 static int
398 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
399 {
400         unsigned int op;
401         unsigned long ip = rec->ip;
402 
403         /* read where this goes */
404         if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
405                 return -EFAULT;
406 
407         /* It should be pointing to a nop */
408         if (op != PPC_INST_NOP) {
409                 printk(KERN_ERR "Expected NOP but have %x\n", op);
410                 return -EINVAL;
411         }
412 
413         /* If we never set up a trampoline to ftrace_caller, then bail */
414         if (!rec->arch.mod->arch.tramp) {
415                 printk(KERN_ERR "No ftrace trampoline\n");
416                 return -EINVAL;
417         }
418 
419         /* create the branch to the trampoline */
420         op = create_branch((unsigned int *)ip,
421                            rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
422         if (!op) {
423                 printk(KERN_ERR "REL24 out of range!\n");
424                 return -EINVAL;
425         }
426 
427         pr_devel("write to %lx\n", rec->ip);
428 
429         if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
430                 return -EPERM;
431 
432         flush_icache_range(ip, ip + 8);
433 
434         return 0;
435 }
436 #endif /* CONFIG_PPC64 */
437 #endif /* CONFIG_MODULES */
438 
439 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
440 {
441         unsigned long ip = rec->ip;
442         unsigned int old, new;
443 
444         /*
445          * If the calling address is more that 24 bits away,
446          * then we had to use a trampoline to make the call.
447          * Otherwise just update the call site.
448          */
449         if (test_24bit_addr(ip, addr)) {
450                 /* within range */
451                 old = PPC_INST_NOP;
452                 new = ftrace_call_replace(ip, addr, 1);
453                 return ftrace_modify_code(ip, old, new);
454         }
455 
456 #ifdef CONFIG_MODULES
457         /*
458          * Out of range jumps are called from modules.
459          * Being that we are converting from nop, it had better
460          * already have a module defined.
461          */
462         if (!rec->arch.mod) {
463                 printk(KERN_ERR "No module loaded\n");
464                 return -EINVAL;
465         }
466 
467         return __ftrace_make_call(rec, addr);
468 #else
469         /* We should not get here without modules */
470         return -EINVAL;
471 #endif /* CONFIG_MODULES */
472 }
473 
474 int ftrace_update_ftrace_func(ftrace_func_t func)
475 {
476         unsigned long ip = (unsigned long)(&ftrace_call);
477         unsigned int old, new;
478         int ret;
479 
480         old = *(unsigned int *)&ftrace_call;
481         new = ftrace_call_replace(ip, (unsigned long)func, 1);
482         ret = ftrace_modify_code(ip, old, new);
483 
484         return ret;
485 }
486 
487 int __init ftrace_dyn_arch_init(void *data)
488 {
489         /* caller expects data to be zero */
490         unsigned long *p = data;
491 
492         *p = 0;
493 
494         return 0;
495 }
496 #endif /* CONFIG_DYNAMIC_FTRACE */
497 
498 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
499 
500 #ifdef CONFIG_DYNAMIC_FTRACE
501 extern void ftrace_graph_call(void);
502 extern void ftrace_graph_stub(void);
503 
504 int ftrace_enable_ftrace_graph_caller(void)
505 {
506         unsigned long ip = (unsigned long)(&ftrace_graph_call);
507         unsigned long addr = (unsigned long)(&ftrace_graph_caller);
508         unsigned long stub = (unsigned long)(&ftrace_graph_stub);
509         unsigned int old, new;
510 
511         old = ftrace_call_replace(ip, stub, 0);
512         new = ftrace_call_replace(ip, addr, 0);
513 
514         return ftrace_modify_code(ip, old, new);
515 }
516 
517 int ftrace_disable_ftrace_graph_caller(void)
518 {
519         unsigned long ip = (unsigned long)(&ftrace_graph_call);
520         unsigned long addr = (unsigned long)(&ftrace_graph_caller);
521         unsigned long stub = (unsigned long)(&ftrace_graph_stub);
522         unsigned int old, new;
523 
524         old = ftrace_call_replace(ip, addr, 0);
525         new = ftrace_call_replace(ip, stub, 0);
526 
527         return ftrace_modify_code(ip, old, new);
528 }
529 #endif /* CONFIG_DYNAMIC_FTRACE */
530 
531 #ifdef CONFIG_PPC64
532 extern void mod_return_to_handler(void);
533 #endif
534 
535 /*
536  * Hook the return address and push it in the stack of return addrs
537  * in current thread info.
538  */
539 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
540 {
541         unsigned long old;
542         int faulted;
543         struct ftrace_graph_ent trace;
544         unsigned long return_hooker = (unsigned long)&return_to_handler;
545 
546         if (unlikely(atomic_read(&current->tracing_graph_pause)))
547                 return;
548 
549 #ifdef CONFIG_PPC64
550         /* non core kernel code needs to save and restore the TOC */
551         if (REGION_ID(self_addr) != KERNEL_REGION_ID)
552                 return_hooker = (unsigned long)&mod_return_to_handler;
553 #endif
554 
555         return_hooker = ppc_function_entry((void *)return_hooker);
556 
557         /*
558          * Protect against fault, even if it shouldn't
559          * happen. This tool is too much intrusive to
560          * ignore such a protection.
561          */
562         asm volatile(
563                 "1: " PPC_LL "%[old], 0(%[parent])\n"
564                 "2: " PPC_STL "%[return_hooker], 0(%[parent])\n"
565                 "   li %[faulted], 0\n"
566                 "3:\n"
567 
568                 ".section .fixup, \"ax\"\n"
569                 "4: li %[faulted], 1\n"
570                 "   b 3b\n"
571                 ".previous\n"
572 
573                 ".section __ex_table,\"a\"\n"
574                         PPC_LONG_ALIGN "\n"
575                         PPC_LONG "1b,4b\n"
576                         PPC_LONG "2b,4b\n"
577                 ".previous"
578 
579                 : [old] "=&r" (old), [faulted] "=r" (faulted)
580                 : [parent] "r" (parent), [return_hooker] "r" (return_hooker)
581                 : "memory"
582         );
583 
584         if (unlikely(faulted)) {
585                 ftrace_graph_stop();
586                 WARN_ON(1);
587                 return;
588         }
589 
590         if (ftrace_push_return_trace(old, self_addr, &trace.depth, 0) == -EBUSY) {
591                 *parent = old;
592                 return;
593         }
594 
595         trace.func = self_addr;
596 
597         /* Only trace if the calling function expects to */
598         if (!ftrace_graph_entry(&trace)) {
599                 current->curr_ret_stack--;
600                 *parent = old;
601         }
602 }
603 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
604 
605 #if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_PPC64)
606 unsigned long __init arch_syscall_addr(int nr)
607 {
608         return sys_call_table[nr*2];
609 }
610 #endif /* CONFIG_FTRACE_SYSCALLS && CONFIG_PPC64 */
611 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp