~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/s390/kernel/kprobes.c

Version: ~ [ linux-5.2-rc1 ] ~ [ linux-5.1.2 ] ~ [ linux-5.0.16 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.43 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.119 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.176 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.179 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.139 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.67 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0+
  2 /*
  3  *  Kernel Probes (KProbes)
  4  *
  5  * Copyright IBM Corp. 2002, 2006
  6  *
  7  * s390 port, used ppc64 as template. Mike Grundy <grundym@us.ibm.com>
  8  */
  9 
 10 #include <linux/kprobes.h>
 11 #include <linux/ptrace.h>
 12 #include <linux/preempt.h>
 13 #include <linux/stop_machine.h>
 14 #include <linux/kdebug.h>
 15 #include <linux/uaccess.h>
 16 #include <linux/extable.h>
 17 #include <linux/module.h>
 18 #include <linux/slab.h>
 19 #include <linux/hardirq.h>
 20 #include <linux/ftrace.h>
 21 #include <asm/set_memory.h>
 22 #include <asm/sections.h>
 23 #include <asm/dis.h>
 24 
 25 DEFINE_PER_CPU(struct kprobe *, current_kprobe);
 26 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
 27 
 28 struct kretprobe_blackpoint kretprobe_blacklist[] = { };
 29 
 30 DEFINE_INSN_CACHE_OPS(dmainsn);
 31 
 32 static void *alloc_dmainsn_page(void)
 33 {
 34         void *page;
 35 
 36         page = (void *) __get_free_page(GFP_KERNEL | GFP_DMA);
 37         if (page)
 38                 set_memory_x((unsigned long) page, 1);
 39         return page;
 40 }
 41 
 42 static void free_dmainsn_page(void *page)
 43 {
 44         set_memory_nx((unsigned long) page, 1);
 45         free_page((unsigned long)page);
 46 }
 47 
 48 struct kprobe_insn_cache kprobe_dmainsn_slots = {
 49         .mutex = __MUTEX_INITIALIZER(kprobe_dmainsn_slots.mutex),
 50         .alloc = alloc_dmainsn_page,
 51         .free = free_dmainsn_page,
 52         .pages = LIST_HEAD_INIT(kprobe_dmainsn_slots.pages),
 53         .insn_size = MAX_INSN_SIZE,
 54 };
 55 
 56 static void copy_instruction(struct kprobe *p)
 57 {
 58         unsigned long ip = (unsigned long) p->addr;
 59         s64 disp, new_disp;
 60         u64 addr, new_addr;
 61 
 62         if (ftrace_location(ip) == ip) {
 63                 /*
 64                  * If kprobes patches the instruction that is morphed by
 65                  * ftrace make sure that kprobes always sees the branch
 66                  * "jg .+24" that skips the mcount block or the "brcl 0,0"
 67                  * in case of hotpatch.
 68                  */
 69                 ftrace_generate_nop_insn((struct ftrace_insn *)p->ainsn.insn);
 70                 p->ainsn.is_ftrace_insn = 1;
 71         } else
 72                 memcpy(p->ainsn.insn, p->addr, insn_length(*p->addr >> 8));
 73         p->opcode = p->ainsn.insn[0];
 74         if (!probe_is_insn_relative_long(p->ainsn.insn))
 75                 return;
 76         /*
 77          * For pc-relative instructions in RIL-b or RIL-c format patch the
 78          * RI2 displacement field. We have already made sure that the insn
 79          * slot for the patched instruction is within the same 2GB area
 80          * as the original instruction (either kernel image or module area).
 81          * Therefore the new displacement will always fit.
 82          */
 83         disp = *(s32 *)&p->ainsn.insn[1];
 84         addr = (u64)(unsigned long)p->addr;
 85         new_addr = (u64)(unsigned long)p->ainsn.insn;
 86         new_disp = ((addr + (disp * 2)) - new_addr) / 2;
 87         *(s32 *)&p->ainsn.insn[1] = new_disp;
 88 }
 89 NOKPROBE_SYMBOL(copy_instruction);
 90 
 91 static inline int is_kernel_addr(void *addr)
 92 {
 93         return addr < (void *)_end;
 94 }
 95 
 96 static int s390_get_insn_slot(struct kprobe *p)
 97 {
 98         /*
 99          * Get an insn slot that is within the same 2GB area like the original
100          * instruction. That way instructions with a 32bit signed displacement
101          * field can be patched and executed within the insn slot.
102          */
103         p->ainsn.insn = NULL;
104         if (is_kernel_addr(p->addr))
105                 p->ainsn.insn = get_dmainsn_slot();
106         else if (is_module_addr(p->addr))
107                 p->ainsn.insn = get_insn_slot();
108         return p->ainsn.insn ? 0 : -ENOMEM;
109 }
110 NOKPROBE_SYMBOL(s390_get_insn_slot);
111 
112 static void s390_free_insn_slot(struct kprobe *p)
113 {
114         if (!p->ainsn.insn)
115                 return;
116         if (is_kernel_addr(p->addr))
117                 free_dmainsn_slot(p->ainsn.insn, 0);
118         else
119                 free_insn_slot(p->ainsn.insn, 0);
120         p->ainsn.insn = NULL;
121 }
122 NOKPROBE_SYMBOL(s390_free_insn_slot);
123 
124 int arch_prepare_kprobe(struct kprobe *p)
125 {
126         if ((unsigned long) p->addr & 0x01)
127                 return -EINVAL;
128         /* Make sure the probe isn't going on a difficult instruction */
129         if (probe_is_prohibited_opcode(p->addr))
130                 return -EINVAL;
131         if (s390_get_insn_slot(p))
132                 return -ENOMEM;
133         copy_instruction(p);
134         return 0;
135 }
136 NOKPROBE_SYMBOL(arch_prepare_kprobe);
137 
138 int arch_check_ftrace_location(struct kprobe *p)
139 {
140         return 0;
141 }
142 
143 struct swap_insn_args {
144         struct kprobe *p;
145         unsigned int arm_kprobe : 1;
146 };
147 
148 static int swap_instruction(void *data)
149 {
150         struct swap_insn_args *args = data;
151         struct ftrace_insn new_insn, *insn;
152         struct kprobe *p = args->p;
153         size_t len;
154 
155         new_insn.opc = args->arm_kprobe ? BREAKPOINT_INSTRUCTION : p->opcode;
156         len = sizeof(new_insn.opc);
157         if (!p->ainsn.is_ftrace_insn)
158                 goto skip_ftrace;
159         len = sizeof(new_insn);
160         insn = (struct ftrace_insn *) p->addr;
161         if (args->arm_kprobe) {
162                 if (is_ftrace_nop(insn))
163                         new_insn.disp = KPROBE_ON_FTRACE_NOP;
164                 else
165                         new_insn.disp = KPROBE_ON_FTRACE_CALL;
166         } else {
167                 ftrace_generate_call_insn(&new_insn, (unsigned long)p->addr);
168                 if (insn->disp == KPROBE_ON_FTRACE_NOP)
169                         ftrace_generate_nop_insn(&new_insn);
170         }
171 skip_ftrace:
172         s390_kernel_write(p->addr, &new_insn, len);
173         return 0;
174 }
175 NOKPROBE_SYMBOL(swap_instruction);
176 
177 void arch_arm_kprobe(struct kprobe *p)
178 {
179         struct swap_insn_args args = {.p = p, .arm_kprobe = 1};
180 
181         stop_machine_cpuslocked(swap_instruction, &args, NULL);
182 }
183 NOKPROBE_SYMBOL(arch_arm_kprobe);
184 
185 void arch_disarm_kprobe(struct kprobe *p)
186 {
187         struct swap_insn_args args = {.p = p, .arm_kprobe = 0};
188 
189         stop_machine_cpuslocked(swap_instruction, &args, NULL);
190 }
191 NOKPROBE_SYMBOL(arch_disarm_kprobe);
192 
193 void arch_remove_kprobe(struct kprobe *p)
194 {
195         s390_free_insn_slot(p);
196 }
197 NOKPROBE_SYMBOL(arch_remove_kprobe);
198 
199 static void enable_singlestep(struct kprobe_ctlblk *kcb,
200                               struct pt_regs *regs,
201                               unsigned long ip)
202 {
203         struct per_regs per_kprobe;
204 
205         /* Set up the PER control registers %cr9-%cr11 */
206         per_kprobe.control = PER_EVENT_IFETCH;
207         per_kprobe.start = ip;
208         per_kprobe.end = ip;
209 
210         /* Save control regs and psw mask */
211         __ctl_store(kcb->kprobe_saved_ctl, 9, 11);
212         kcb->kprobe_saved_imask = regs->psw.mask &
213                 (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT);
214 
215         /* Set PER control regs, turns on single step for the given address */
216         __ctl_load(per_kprobe, 9, 11);
217         regs->psw.mask |= PSW_MASK_PER;
218         regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT);
219         regs->psw.addr = ip;
220 }
221 NOKPROBE_SYMBOL(enable_singlestep);
222 
223 static void disable_singlestep(struct kprobe_ctlblk *kcb,
224                                struct pt_regs *regs,
225                                unsigned long ip)
226 {
227         /* Restore control regs and psw mask, set new psw address */
228         __ctl_load(kcb->kprobe_saved_ctl, 9, 11);
229         regs->psw.mask &= ~PSW_MASK_PER;
230         regs->psw.mask |= kcb->kprobe_saved_imask;
231         regs->psw.addr = ip;
232 }
233 NOKPROBE_SYMBOL(disable_singlestep);
234 
235 /*
236  * Activate a kprobe by storing its pointer to current_kprobe. The
237  * previous kprobe is stored in kcb->prev_kprobe. A stack of up to
238  * two kprobes can be active, see KPROBE_REENTER.
239  */
240 static void push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p)
241 {
242         kcb->prev_kprobe.kp = __this_cpu_read(current_kprobe);
243         kcb->prev_kprobe.status = kcb->kprobe_status;
244         __this_cpu_write(current_kprobe, p);
245 }
246 NOKPROBE_SYMBOL(push_kprobe);
247 
248 /*
249  * Deactivate a kprobe by backing up to the previous state. If the
250  * current state is KPROBE_REENTER prev_kprobe.kp will be non-NULL,
251  * for any other state prev_kprobe.kp will be NULL.
252  */
253 static void pop_kprobe(struct kprobe_ctlblk *kcb)
254 {
255         __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
256         kcb->kprobe_status = kcb->prev_kprobe.status;
257 }
258 NOKPROBE_SYMBOL(pop_kprobe);
259 
260 void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
261 {
262         ri->ret_addr = (kprobe_opcode_t *) regs->gprs[14];
263 
264         /* Replace the return addr with trampoline addr */
265         regs->gprs[14] = (unsigned long) &kretprobe_trampoline;
266 }
267 NOKPROBE_SYMBOL(arch_prepare_kretprobe);
268 
269 static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p)
270 {
271         switch (kcb->kprobe_status) {
272         case KPROBE_HIT_SSDONE:
273         case KPROBE_HIT_ACTIVE:
274                 kprobes_inc_nmissed_count(p);
275                 break;
276         case KPROBE_HIT_SS:
277         case KPROBE_REENTER:
278         default:
279                 /*
280                  * A kprobe on the code path to single step an instruction
281                  * is a BUG. The code path resides in the .kprobes.text
282                  * section and is executed with interrupts disabled.
283                  */
284                 pr_err("Invalid kprobe detected.\n");
285                 dump_kprobe(p);
286                 BUG();
287         }
288 }
289 NOKPROBE_SYMBOL(kprobe_reenter_check);
290 
291 static int kprobe_handler(struct pt_regs *regs)
292 {
293         struct kprobe_ctlblk *kcb;
294         struct kprobe *p;
295 
296         /*
297          * We want to disable preemption for the entire duration of kprobe
298          * processing. That includes the calls to the pre/post handlers
299          * and single stepping the kprobe instruction.
300          */
301         preempt_disable();
302         kcb = get_kprobe_ctlblk();
303         p = get_kprobe((void *)(regs->psw.addr - 2));
304 
305         if (p) {
306                 if (kprobe_running()) {
307                         /*
308                          * We have hit a kprobe while another is still
309                          * active. This can happen in the pre and post
310                          * handler. Single step the instruction of the
311                          * new probe but do not call any handler function
312                          * of this secondary kprobe.
313                          * push_kprobe and pop_kprobe saves and restores
314                          * the currently active kprobe.
315                          */
316                         kprobe_reenter_check(kcb, p);
317                         push_kprobe(kcb, p);
318                         kcb->kprobe_status = KPROBE_REENTER;
319                 } else {
320                         /*
321                          * If we have no pre-handler or it returned 0, we
322                          * continue with single stepping. If we have a
323                          * pre-handler and it returned non-zero, it prepped
324                          * for calling the break_handler below on re-entry
325                          * for jprobe processing, so get out doing nothing
326                          * more here.
327                          */
328                         push_kprobe(kcb, p);
329                         kcb->kprobe_status = KPROBE_HIT_ACTIVE;
330                         if (p->pre_handler && p->pre_handler(p, regs))
331                                 return 1;
332                         kcb->kprobe_status = KPROBE_HIT_SS;
333                 }
334                 enable_singlestep(kcb, regs, (unsigned long) p->ainsn.insn);
335                 return 1;
336         } else if (kprobe_running()) {
337                 p = __this_cpu_read(current_kprobe);
338                 if (p->break_handler && p->break_handler(p, regs)) {
339                         /*
340                          * Continuation after the jprobe completed and
341                          * caused the jprobe_return trap. The jprobe
342                          * break_handler "returns" to the original
343                          * function that still has the kprobe breakpoint
344                          * installed. We continue with single stepping.
345                          */
346                         kcb->kprobe_status = KPROBE_HIT_SS;
347                         enable_singlestep(kcb, regs,
348                                           (unsigned long) p->ainsn.insn);
349                         return 1;
350                 } /* else:
351                    * No kprobe at this address and the current kprobe
352                    * has no break handler (no jprobe!). The kernel just
353                    * exploded, let the standard trap handler pick up the
354                    * pieces.
355                    */
356         } /* else:
357            * No kprobe at this address and no active kprobe. The trap has
358            * not been caused by a kprobe breakpoint. The race of breakpoint
359            * vs. kprobe remove does not exist because on s390 as we use
360            * stop_machine to arm/disarm the breakpoints.
361            */
362         preempt_enable_no_resched();
363         return 0;
364 }
365 NOKPROBE_SYMBOL(kprobe_handler);
366 
367 /*
368  * Function return probe trampoline:
369  *      - init_kprobes() establishes a probepoint here
370  *      - When the probed function returns, this probe
371  *              causes the handlers to fire
372  */
373 static void __used kretprobe_trampoline_holder(void)
374 {
375         asm volatile(".global kretprobe_trampoline\n"
376                      "kretprobe_trampoline: bcr 0,0\n");
377 }
378 
379 /*
380  * Called when the probe at kretprobe trampoline is hit
381  */
382 static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
383 {
384         struct kretprobe_instance *ri;
385         struct hlist_head *head, empty_rp;
386         struct hlist_node *tmp;
387         unsigned long flags, orig_ret_address;
388         unsigned long trampoline_address;
389         kprobe_opcode_t *correct_ret_addr;
390 
391         INIT_HLIST_HEAD(&empty_rp);
392         kretprobe_hash_lock(current, &head, &flags);
393 
394         /*
395          * It is possible to have multiple instances associated with a given
396          * task either because an multiple functions in the call path
397          * have a return probe installed on them, and/or more than one return
398          * return probe was registered for a target function.
399          *
400          * We can handle this because:
401          *     - instances are always inserted at the head of the list
402          *     - when multiple return probes are registered for the same
403          *       function, the first instance's ret_addr will point to the
404          *       real return address, and all the rest will point to
405          *       kretprobe_trampoline
406          */
407         ri = NULL;
408         orig_ret_address = 0;
409         correct_ret_addr = NULL;
410         trampoline_address = (unsigned long) &kretprobe_trampoline;
411         hlist_for_each_entry_safe(ri, tmp, head, hlist) {
412                 if (ri->task != current)
413                         /* another task is sharing our hash bucket */
414                         continue;
415 
416                 orig_ret_address = (unsigned long) ri->ret_addr;
417 
418                 if (orig_ret_address != trampoline_address)
419                         /*
420                          * This is the real return address. Any other
421                          * instances associated with this task are for
422                          * other calls deeper on the call stack
423                          */
424                         break;
425         }
426 
427         kretprobe_assert(ri, orig_ret_address, trampoline_address);
428 
429         correct_ret_addr = ri->ret_addr;
430         hlist_for_each_entry_safe(ri, tmp, head, hlist) {
431                 if (ri->task != current)
432                         /* another task is sharing our hash bucket */
433                         continue;
434 
435                 orig_ret_address = (unsigned long) ri->ret_addr;
436 
437                 if (ri->rp && ri->rp->handler) {
438                         ri->ret_addr = correct_ret_addr;
439                         ri->rp->handler(ri, regs);
440                 }
441 
442                 recycle_rp_inst(ri, &empty_rp);
443 
444                 if (orig_ret_address != trampoline_address)
445                         /*
446                          * This is the real return address. Any other
447                          * instances associated with this task are for
448                          * other calls deeper on the call stack
449                          */
450                         break;
451         }
452 
453         regs->psw.addr = orig_ret_address;
454 
455         pop_kprobe(get_kprobe_ctlblk());
456         kretprobe_hash_unlock(current, &flags);
457         preempt_enable_no_resched();
458 
459         hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
460                 hlist_del(&ri->hlist);
461                 kfree(ri);
462         }
463         /*
464          * By returning a non-zero value, we are telling
465          * kprobe_handler() that we don't want the post_handler
466          * to run (and have re-enabled preemption)
467          */
468         return 1;
469 }
470 NOKPROBE_SYMBOL(trampoline_probe_handler);
471 
472 /*
473  * Called after single-stepping.  p->addr is the address of the
474  * instruction whose first byte has been replaced by the "breakpoint"
475  * instruction.  To avoid the SMP problems that can occur when we
476  * temporarily put back the original opcode to single-step, we
477  * single-stepped a copy of the instruction.  The address of this
478  * copy is p->ainsn.insn.
479  */
480 static void resume_execution(struct kprobe *p, struct pt_regs *regs)
481 {
482         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
483         unsigned long ip = regs->psw.addr;
484         int fixup = probe_get_fixup_type(p->ainsn.insn);
485 
486         /* Check if the kprobes location is an enabled ftrace caller */
487         if (p->ainsn.is_ftrace_insn) {
488                 struct ftrace_insn *insn = (struct ftrace_insn *) p->addr;
489                 struct ftrace_insn call_insn;
490 
491                 ftrace_generate_call_insn(&call_insn, (unsigned long) p->addr);
492                 /*
493                  * A kprobe on an enabled ftrace call site actually single
494                  * stepped an unconditional branch (ftrace nop equivalent).
495                  * Now we need to fixup things and pretend that a brasl r0,...
496                  * was executed instead.
497                  */
498                 if (insn->disp == KPROBE_ON_FTRACE_CALL) {
499                         ip += call_insn.disp * 2 - MCOUNT_INSN_SIZE;
500                         regs->gprs[0] = (unsigned long)p->addr + sizeof(*insn);
501                 }
502         }
503 
504         if (fixup & FIXUP_PSW_NORMAL)
505                 ip += (unsigned long) p->addr - (unsigned long) p->ainsn.insn;
506 
507         if (fixup & FIXUP_BRANCH_NOT_TAKEN) {
508                 int ilen = insn_length(p->ainsn.insn[0] >> 8);
509                 if (ip - (unsigned long) p->ainsn.insn == ilen)
510                         ip = (unsigned long) p->addr + ilen;
511         }
512 
513         if (fixup & FIXUP_RETURN_REGISTER) {
514                 int reg = (p->ainsn.insn[0] & 0xf0) >> 4;
515                 regs->gprs[reg] += (unsigned long) p->addr -
516                                    (unsigned long) p->ainsn.insn;
517         }
518 
519         disable_singlestep(kcb, regs, ip);
520 }
521 NOKPROBE_SYMBOL(resume_execution);
522 
523 static int post_kprobe_handler(struct pt_regs *regs)
524 {
525         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
526         struct kprobe *p = kprobe_running();
527 
528         if (!p)
529                 return 0;
530 
531         if (kcb->kprobe_status != KPROBE_REENTER && p->post_handler) {
532                 kcb->kprobe_status = KPROBE_HIT_SSDONE;
533                 p->post_handler(p, regs, 0);
534         }
535 
536         resume_execution(p, regs);
537         pop_kprobe(kcb);
538         preempt_enable_no_resched();
539 
540         /*
541          * if somebody else is singlestepping across a probe point, psw mask
542          * will have PER set, in which case, continue the remaining processing
543          * of do_single_step, as if this is not a probe hit.
544          */
545         if (regs->psw.mask & PSW_MASK_PER)
546                 return 0;
547 
548         return 1;
549 }
550 NOKPROBE_SYMBOL(post_kprobe_handler);
551 
552 static int kprobe_trap_handler(struct pt_regs *regs, int trapnr)
553 {
554         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
555         struct kprobe *p = kprobe_running();
556         const struct exception_table_entry *entry;
557 
558         switch(kcb->kprobe_status) {
559         case KPROBE_HIT_SS:
560         case KPROBE_REENTER:
561                 /*
562                  * We are here because the instruction being single
563                  * stepped caused a page fault. We reset the current
564                  * kprobe and the nip points back to the probe address
565                  * and allow the page fault handler to continue as a
566                  * normal page fault.
567                  */
568                 disable_singlestep(kcb, regs, (unsigned long) p->addr);
569                 pop_kprobe(kcb);
570                 preempt_enable_no_resched();
571                 break;
572         case KPROBE_HIT_ACTIVE:
573         case KPROBE_HIT_SSDONE:
574                 /*
575                  * We increment the nmissed count for accounting,
576                  * we can also use npre/npostfault count for accounting
577                  * these specific fault cases.
578                  */
579                 kprobes_inc_nmissed_count(p);
580 
581                 /*
582                  * We come here because instructions in the pre/post
583                  * handler caused the page_fault, this could happen
584                  * if handler tries to access user space by
585                  * copy_from_user(), get_user() etc. Let the
586                  * user-specified handler try to fix it first.
587                  */
588                 if (p->fault_handler && p->fault_handler(p, regs, trapnr))
589                         return 1;
590 
591                 /*
592                  * In case the user-specified fault handler returned
593                  * zero, try to fix up.
594                  */
595                 entry = search_exception_tables(regs->psw.addr);
596                 if (entry) {
597                         regs->psw.addr = extable_fixup(entry);
598                         return 1;
599                 }
600 
601                 /*
602                  * fixup_exception() could not handle it,
603                  * Let do_page_fault() fix it.
604                  */
605                 break;
606         default:
607                 break;
608         }
609         return 0;
610 }
611 NOKPROBE_SYMBOL(kprobe_trap_handler);
612 
613 int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
614 {
615         int ret;
616 
617         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
618                 local_irq_disable();
619         ret = kprobe_trap_handler(regs, trapnr);
620         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
621                 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
622         return ret;
623 }
624 NOKPROBE_SYMBOL(kprobe_fault_handler);
625 
626 /*
627  * Wrapper routine to for handling exceptions.
628  */
629 int kprobe_exceptions_notify(struct notifier_block *self,
630                              unsigned long val, void *data)
631 {
632         struct die_args *args = (struct die_args *) data;
633         struct pt_regs *regs = args->regs;
634         int ret = NOTIFY_DONE;
635 
636         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
637                 local_irq_disable();
638 
639         switch (val) {
640         case DIE_BPT:
641                 if (kprobe_handler(regs))
642                         ret = NOTIFY_STOP;
643                 break;
644         case DIE_SSTEP:
645                 if (post_kprobe_handler(regs))
646                         ret = NOTIFY_STOP;
647                 break;
648         case DIE_TRAP:
649                 if (!preemptible() && kprobe_running() &&
650                     kprobe_trap_handler(regs, args->trapnr))
651                         ret = NOTIFY_STOP;
652                 break;
653         default:
654                 break;
655         }
656 
657         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
658                 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
659 
660         return ret;
661 }
662 NOKPROBE_SYMBOL(kprobe_exceptions_notify);
663 
664 int setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
665 {
666         struct jprobe *jp = container_of(p, struct jprobe, kp);
667         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
668         unsigned long stack;
669 
670         memcpy(&kcb->jprobe_saved_regs, regs, sizeof(struct pt_regs));
671 
672         /* setup return addr to the jprobe handler routine */
673         regs->psw.addr = (unsigned long) jp->entry;
674         regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT);
675 
676         /* r15 is the stack pointer */
677         stack = (unsigned long) regs->gprs[15];
678 
679         memcpy(kcb->jprobes_stack, (void *) stack, MIN_STACK_SIZE(stack));
680 
681         /*
682          * jprobes use jprobe_return() which skips the normal return
683          * path of the function, and this messes up the accounting of the
684          * function graph tracer to get messed up.
685          *
686          * Pause function graph tracing while performing the jprobe function.
687          */
688         pause_graph_tracing();
689         return 1;
690 }
691 NOKPROBE_SYMBOL(setjmp_pre_handler);
692 
693 void jprobe_return(void)
694 {
695         asm volatile(".word 0x0002");
696 }
697 NOKPROBE_SYMBOL(jprobe_return);
698 
699 int longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
700 {
701         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
702         unsigned long stack;
703 
704         /* It's OK to start function graph tracing again */
705         unpause_graph_tracing();
706 
707         stack = (unsigned long) kcb->jprobe_saved_regs.gprs[15];
708 
709         /* Put the regs back */
710         memcpy(regs, &kcb->jprobe_saved_regs, sizeof(struct pt_regs));
711         /* put the stack back */
712         memcpy((void *) stack, kcb->jprobes_stack, MIN_STACK_SIZE(stack));
713         preempt_enable_no_resched();
714         return 1;
715 }
716 NOKPROBE_SYMBOL(longjmp_break_handler);
717 
718 static struct kprobe trampoline = {
719         .addr = (kprobe_opcode_t *) &kretprobe_trampoline,
720         .pre_handler = trampoline_probe_handler
721 };
722 
723 int __init arch_init_kprobes(void)
724 {
725         return register_kprobe(&trampoline);
726 }
727 
728 int arch_trampoline_kprobe(struct kprobe *p)
729 {
730         return p->addr == (kprobe_opcode_t *) &kretprobe_trampoline;
731 }
732 NOKPROBE_SYMBOL(arch_trampoline_kprobe);
733 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp