~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm/mm/fault.c

Version: ~ [ linux-5.16-rc3 ] ~ [ linux-5.15.5 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.82 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.162 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.218 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.256 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.291 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.293 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-only
  2 /*
  3  *  linux/arch/arm/mm/fault.c
  4  *
  5  *  Copyright (C) 1995  Linus Torvalds
  6  *  Modifications for ARM processor (c) 1995-2004 Russell King
  7  */
  8 #include <linux/extable.h>
  9 #include <linux/signal.h>
 10 #include <linux/mm.h>
 11 #include <linux/hardirq.h>
 12 #include <linux/init.h>
 13 #include <linux/kprobes.h>
 14 #include <linux/uaccess.h>
 15 #include <linux/page-flags.h>
 16 #include <linux/sched/signal.h>
 17 #include <linux/sched/debug.h>
 18 #include <linux/highmem.h>
 19 #include <linux/perf_event.h>
 20 
 21 #include <asm/system_misc.h>
 22 #include <asm/system_info.h>
 23 #include <asm/tlbflush.h>
 24 
 25 #include "fault.h"
 26 
 27 #ifdef CONFIG_MMU
 28 
 29 /*
 30  * This is useful to dump out the page tables associated with
 31  * 'addr' in mm 'mm'.
 32  */
 33 void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)
 34 {
 35         pgd_t *pgd;
 36 
 37         if (!mm)
 38                 mm = &init_mm;
 39 
 40         printk("%spgd = %p\n", lvl, mm->pgd);
 41         pgd = pgd_offset(mm, addr);
 42         printk("%s[%08lx] *pgd=%08llx", lvl, addr, (long long)pgd_val(*pgd));
 43 
 44         do {
 45                 p4d_t *p4d;
 46                 pud_t *pud;
 47                 pmd_t *pmd;
 48                 pte_t *pte;
 49 
 50                 p4d = p4d_offset(pgd, addr);
 51                 if (p4d_none(*p4d))
 52                         break;
 53 
 54                 if (p4d_bad(*p4d)) {
 55                         pr_cont("(bad)");
 56                         break;
 57                 }
 58 
 59                 pud = pud_offset(p4d, addr);
 60                 if (PTRS_PER_PUD != 1)
 61                         pr_cont(", *pud=%08llx", (long long)pud_val(*pud));
 62 
 63                 if (pud_none(*pud))
 64                         break;
 65 
 66                 if (pud_bad(*pud)) {
 67                         pr_cont("(bad)");
 68                         break;
 69                 }
 70 
 71                 pmd = pmd_offset(pud, addr);
 72                 if (PTRS_PER_PMD != 1)
 73                         pr_cont(", *pmd=%08llx", (long long)pmd_val(*pmd));
 74 
 75                 if (pmd_none(*pmd))
 76                         break;
 77 
 78                 if (pmd_bad(*pmd)) {
 79                         pr_cont("(bad)");
 80                         break;
 81                 }
 82 
 83                 /* We must not map this if we have highmem enabled */
 84                 if (PageHighMem(pfn_to_page(pmd_val(*pmd) >> PAGE_SHIFT)))
 85                         break;
 86 
 87                 pte = pte_offset_map(pmd, addr);
 88                 pr_cont(", *pte=%08llx", (long long)pte_val(*pte));
 89 #ifndef CONFIG_ARM_LPAE
 90                 pr_cont(", *ppte=%08llx",
 91                        (long long)pte_val(pte[PTE_HWTABLE_PTRS]));
 92 #endif
 93                 pte_unmap(pte);
 94         } while(0);
 95 
 96         pr_cont("\n");
 97 }
 98 #else                                   /* CONFIG_MMU */
 99 void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)
100 { }
101 #endif                                  /* CONFIG_MMU */
102 
103 /*
104  * Oops.  The kernel tried to access some page that wasn't present.
105  */
106 static void
107 __do_kernel_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr,
108                   struct pt_regs *regs)
109 {
110         /*
111          * Are we prepared to handle this kernel fault?
112          */
113         if (fixup_exception(regs))
114                 return;
115 
116         /*
117          * No handler, we'll have to terminate things with extreme prejudice.
118          */
119         bust_spinlocks(1);
120         pr_alert("8<--- cut here ---\n");
121         pr_alert("Unable to handle kernel %s at virtual address %08lx\n",
122                  (addr < PAGE_SIZE) ? "NULL pointer dereference" :
123                  "paging request", addr);
124 
125         show_pte(KERN_ALERT, mm, addr);
126         die("Oops", regs, fsr);
127         bust_spinlocks(0);
128         do_exit(SIGKILL);
129 }
130 
131 /*
132  * Something tried to access memory that isn't in our memory map..
133  * User mode accesses just cause a SIGSEGV
134  */
135 static void
136 __do_user_fault(unsigned long addr, unsigned int fsr, unsigned int sig,
137                 int code, struct pt_regs *regs)
138 {
139         struct task_struct *tsk = current;
140 
141         if (addr > TASK_SIZE)
142                 harden_branch_predictor();
143 
144 #ifdef CONFIG_DEBUG_USER
145         if (((user_debug & UDBG_SEGV) && (sig == SIGSEGV)) ||
146             ((user_debug & UDBG_BUS)  && (sig == SIGBUS))) {
147                 pr_err("8<--- cut here ---\n");
148                 pr_err("%s: unhandled page fault (%d) at 0x%08lx, code 0x%03x\n",
149                        tsk->comm, sig, addr, fsr);
150                 show_pte(KERN_ERR, tsk->mm, addr);
151                 show_regs(regs);
152         }
153 #endif
154 #ifndef CONFIG_KUSER_HELPERS
155         if ((sig == SIGSEGV) && ((addr & PAGE_MASK) == 0xffff0000))
156                 printk_ratelimited(KERN_DEBUG
157                                    "%s: CONFIG_KUSER_HELPERS disabled at 0x%08lx\n",
158                                    tsk->comm, addr);
159 #endif
160 
161         tsk->thread.address = addr;
162         tsk->thread.error_code = fsr;
163         tsk->thread.trap_no = 14;
164         force_sig_fault(sig, code, (void __user *)addr);
165 }
166 
167 void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
168 {
169         struct task_struct *tsk = current;
170         struct mm_struct *mm = tsk->active_mm;
171 
172         /*
173          * If we are in kernel mode at this point, we
174          * have no context to handle this fault with.
175          */
176         if (user_mode(regs))
177                 __do_user_fault(addr, fsr, SIGSEGV, SEGV_MAPERR, regs);
178         else
179                 __do_kernel_fault(mm, addr, fsr, regs);
180 }
181 
182 #ifdef CONFIG_MMU
183 #define VM_FAULT_BADMAP         0x010000
184 #define VM_FAULT_BADACCESS      0x020000
185 
186 /*
187  * Check that the permissions on the VMA allow for the fault which occurred.
188  * If we encountered a write fault, we must have write permission, otherwise
189  * we allow any permission.
190  */
191 static inline bool access_error(unsigned int fsr, struct vm_area_struct *vma)
192 {
193         unsigned int mask = VM_ACCESS_FLAGS;
194 
195         if ((fsr & FSR_WRITE) && !(fsr & FSR_CM))
196                 mask = VM_WRITE;
197         if (fsr & FSR_LNX_PF)
198                 mask = VM_EXEC;
199 
200         return vma->vm_flags & mask ? false : true;
201 }
202 
203 static vm_fault_t __kprobes
204 __do_page_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr,
205                 unsigned int flags, struct task_struct *tsk,
206                 struct pt_regs *regs)
207 {
208         struct vm_area_struct *vma;
209         vm_fault_t fault;
210 
211         vma = find_vma(mm, addr);
212         fault = VM_FAULT_BADMAP;
213         if (unlikely(!vma))
214                 goto out;
215         if (unlikely(vma->vm_start > addr))
216                 goto check_stack;
217 
218         /*
219          * Ok, we have a good vm_area for this
220          * memory access, so we can handle it.
221          */
222 good_area:
223         if (access_error(fsr, vma)) {
224                 fault = VM_FAULT_BADACCESS;
225                 goto out;
226         }
227 
228         return handle_mm_fault(vma, addr & PAGE_MASK, flags, regs);
229 
230 check_stack:
231         /* Don't allow expansion below FIRST_USER_ADDRESS */
232         if (vma->vm_flags & VM_GROWSDOWN &&
233             addr >= FIRST_USER_ADDRESS && !expand_stack(vma, addr))
234                 goto good_area;
235 out:
236         return fault;
237 }
238 
239 static int __kprobes
240 do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
241 {
242         struct task_struct *tsk;
243         struct mm_struct *mm;
244         int sig, code;
245         vm_fault_t fault;
246         unsigned int flags = FAULT_FLAG_DEFAULT;
247 
248         if (kprobe_page_fault(regs, fsr))
249                 return 0;
250 
251         tsk = current;
252         mm  = tsk->mm;
253 
254         /* Enable interrupts if they were enabled in the parent context. */
255         if (interrupts_enabled(regs))
256                 local_irq_enable();
257 
258         /*
259          * If we're in an interrupt or have no user
260          * context, we must not take the fault..
261          */
262         if (faulthandler_disabled() || !mm)
263                 goto no_context;
264 
265         if (user_mode(regs))
266                 flags |= FAULT_FLAG_USER;
267         if ((fsr & FSR_WRITE) && !(fsr & FSR_CM))
268                 flags |= FAULT_FLAG_WRITE;
269 
270         perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
271 
272         /*
273          * As per x86, we may deadlock here.  However, since the kernel only
274          * validly references user space from well defined areas of the code,
275          * we can bug out early if this is from code which shouldn't.
276          */
277         if (!mmap_read_trylock(mm)) {
278                 if (!user_mode(regs) && !search_exception_tables(regs->ARM_pc))
279                         goto no_context;
280 retry:
281                 mmap_read_lock(mm);
282         } else {
283                 /*
284                  * The above down_read_trylock() might have succeeded in
285                  * which case, we'll have missed the might_sleep() from
286                  * down_read()
287                  */
288                 might_sleep();
289 #ifdef CONFIG_DEBUG_VM
290                 if (!user_mode(regs) &&
291                     !search_exception_tables(regs->ARM_pc))
292                         goto no_context;
293 #endif
294         }
295 
296         fault = __do_page_fault(mm, addr, fsr, flags, tsk, regs);
297 
298         /* If we need to retry but a fatal signal is pending, handle the
299          * signal first. We do not need to release the mmap_lock because
300          * it would already be released in __lock_page_or_retry in
301          * mm/filemap.c. */
302         if (fault_signal_pending(fault, regs)) {
303                 if (!user_mode(regs))
304                         goto no_context;
305                 return 0;
306         }
307 
308         if (!(fault & VM_FAULT_ERROR) && flags & FAULT_FLAG_ALLOW_RETRY) {
309                 if (fault & VM_FAULT_RETRY) {
310                         flags |= FAULT_FLAG_TRIED;
311                         goto retry;
312                 }
313         }
314 
315         mmap_read_unlock(mm);
316 
317         /*
318          * Handle the "normal" case first - VM_FAULT_MAJOR
319          */
320         if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | VM_FAULT_BADACCESS))))
321                 return 0;
322 
323         /*
324          * If we are in kernel mode at this point, we
325          * have no context to handle this fault with.
326          */
327         if (!user_mode(regs))
328                 goto no_context;
329 
330         if (fault & VM_FAULT_OOM) {
331                 /*
332                  * We ran out of memory, call the OOM killer, and return to
333                  * userspace (which will retry the fault, or kill us if we
334                  * got oom-killed)
335                  */
336                 pagefault_out_of_memory();
337                 return 0;
338         }
339 
340         if (fault & VM_FAULT_SIGBUS) {
341                 /*
342                  * We had some memory, but were unable to
343                  * successfully fix up this page fault.
344                  */
345                 sig = SIGBUS;
346                 code = BUS_ADRERR;
347         } else {
348                 /*
349                  * Something tried to access memory that
350                  * isn't in our memory map..
351                  */
352                 sig = SIGSEGV;
353                 code = fault == VM_FAULT_BADACCESS ?
354                         SEGV_ACCERR : SEGV_MAPERR;
355         }
356 
357         __do_user_fault(addr, fsr, sig, code, regs);
358         return 0;
359 
360 no_context:
361         __do_kernel_fault(mm, addr, fsr, regs);
362         return 0;
363 }
364 #else                                   /* CONFIG_MMU */
365 static int
366 do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
367 {
368         return 0;
369 }
370 #endif                                  /* CONFIG_MMU */
371 
372 /*
373  * First Level Translation Fault Handler
374  *
375  * We enter here because the first level page table doesn't contain
376  * a valid entry for the address.
377  *
378  * If the address is in kernel space (>= TASK_SIZE), then we are
379  * probably faulting in the vmalloc() area.
380  *
381  * If the init_task's first level page tables contains the relevant
382  * entry, we copy the it to this task.  If not, we send the process
383  * a signal, fixup the exception, or oops the kernel.
384  *
385  * NOTE! We MUST NOT take any locks for this case. We may be in an
386  * interrupt or a critical region, and should only copy the information
387  * from the master page table, nothing more.
388  */
389 #ifdef CONFIG_MMU
390 static int __kprobes
391 do_translation_fault(unsigned long addr, unsigned int fsr,
392                      struct pt_regs *regs)
393 {
394         unsigned int index;
395         pgd_t *pgd, *pgd_k;
396         p4d_t *p4d, *p4d_k;
397         pud_t *pud, *pud_k;
398         pmd_t *pmd, *pmd_k;
399 
400         if (addr < TASK_SIZE)
401                 return do_page_fault(addr, fsr, regs);
402 
403         if (user_mode(regs))
404                 goto bad_area;
405 
406         index = pgd_index(addr);
407 
408         pgd = cpu_get_pgd() + index;
409         pgd_k = init_mm.pgd + index;
410 
411         p4d = p4d_offset(pgd, addr);
412         p4d_k = p4d_offset(pgd_k, addr);
413 
414         if (p4d_none(*p4d_k))
415                 goto bad_area;
416         if (!p4d_present(*p4d))
417                 set_p4d(p4d, *p4d_k);
418 
419         pud = pud_offset(p4d, addr);
420         pud_k = pud_offset(p4d_k, addr);
421 
422         if (pud_none(*pud_k))
423                 goto bad_area;
424         if (!pud_present(*pud))
425                 set_pud(pud, *pud_k);
426 
427         pmd = pmd_offset(pud, addr);
428         pmd_k = pmd_offset(pud_k, addr);
429 
430 #ifdef CONFIG_ARM_LPAE
431         /*
432          * Only one hardware entry per PMD with LPAE.
433          */
434         index = 0;
435 #else
436         /*
437          * On ARM one Linux PGD entry contains two hardware entries (see page
438          * tables layout in pgtable.h). We normally guarantee that we always
439          * fill both L1 entries. But create_mapping() doesn't follow the rule.
440          * It can create inidividual L1 entries, so here we have to call
441          * pmd_none() check for the entry really corresponded to address, not
442          * for the first of pair.
443          */
444         index = (addr >> SECTION_SHIFT) & 1;
445 #endif
446         if (pmd_none(pmd_k[index]))
447                 goto bad_area;
448 
449         copy_pmd(pmd, pmd_k);
450         return 0;
451 
452 bad_area:
453         do_bad_area(addr, fsr, regs);
454         return 0;
455 }
456 #else                                   /* CONFIG_MMU */
457 static int
458 do_translation_fault(unsigned long addr, unsigned int fsr,
459                      struct pt_regs *regs)
460 {
461         return 0;
462 }
463 #endif                                  /* CONFIG_MMU */
464 
465 /*
466  * Some section permission faults need to be handled gracefully.
467  * They can happen due to a __{get,put}_user during an oops.
468  */
469 #ifndef CONFIG_ARM_LPAE
470 static int
471 do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
472 {
473         do_bad_area(addr, fsr, regs);
474         return 0;
475 }
476 #endif /* CONFIG_ARM_LPAE */
477 
478 /*
479  * This abort handler always returns "fault".
480  */
481 static int
482 do_bad(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
483 {
484         return 1;
485 }
486 
487 struct fsr_info {
488         int     (*fn)(unsigned long addr, unsigned int fsr, struct pt_regs *regs);
489         int     sig;
490         int     code;
491         const char *name;
492 };
493 
494 /* FSR definition */
495 #ifdef CONFIG_ARM_LPAE
496 #include "fsr-3level.c"
497 #else
498 #include "fsr-2level.c"
499 #endif
500 
501 void __init
502 hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
503                 int sig, int code, const char *name)
504 {
505         if (nr < 0 || nr >= ARRAY_SIZE(fsr_info))
506                 BUG();
507 
508         fsr_info[nr].fn   = fn;
509         fsr_info[nr].sig  = sig;
510         fsr_info[nr].code = code;
511         fsr_info[nr].name = name;
512 }
513 
514 /*
515  * Dispatch a data abort to the relevant handler.
516  */
517 asmlinkage void
518 do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
519 {
520         const struct fsr_info *inf = fsr_info + fsr_fs(fsr);
521 
522         if (!inf->fn(addr, fsr & ~FSR_LNX_PF, regs))
523                 return;
524 
525         pr_alert("8<--- cut here ---\n");
526         pr_alert("Unhandled fault: %s (0x%03x) at 0x%08lx\n",
527                 inf->name, fsr, addr);
528         show_pte(KERN_ALERT, current->mm, addr);
529 
530         arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr,
531                        fsr, 0);
532 }
533 
534 void __init
535 hook_ifault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
536                  int sig, int code, const char *name)
537 {
538         if (nr < 0 || nr >= ARRAY_SIZE(ifsr_info))
539                 BUG();
540 
541         ifsr_info[nr].fn   = fn;
542         ifsr_info[nr].sig  = sig;
543         ifsr_info[nr].code = code;
544         ifsr_info[nr].name = name;
545 }
546 
547 asmlinkage void
548 do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs)
549 {
550         const struct fsr_info *inf = ifsr_info + fsr_fs(ifsr);
551 
552         if (!inf->fn(addr, ifsr | FSR_LNX_PF, regs))
553                 return;
554 
555         pr_alert("Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n",
556                 inf->name, ifsr, addr);
557 
558         arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr,
559                        ifsr, 0);
560 }
561 
562 /*
563  * Abort handler to be used only during first unmasking of asynchronous aborts
564  * on the boot CPU. This makes sure that the machine will not die if the
565  * firmware/bootloader left an imprecise abort pending for us to trip over.
566  */
567 static int __init early_abort_handler(unsigned long addr, unsigned int fsr,
568                                       struct pt_regs *regs)
569 {
570         pr_warn("Hit pending asynchronous external abort (FSR=0x%08x) during "
571                 "first unmask, this is most likely caused by a "
572                 "firmware/bootloader bug.\n", fsr);
573 
574         return 0;
575 }
576 
577 void __init early_abt_enable(void)
578 {
579         fsr_info[FSR_FS_AEA].fn = early_abort_handler;
580         local_abt_enable();
581         fsr_info[FSR_FS_AEA].fn = do_bad;
582 }
583 
584 #ifndef CONFIG_ARM_LPAE
585 static int __init exceptions_init(void)
586 {
587         if (cpu_architecture() >= CPU_ARCH_ARMv6) {
588                 hook_fault_code(4, do_translation_fault, SIGSEGV, SEGV_MAPERR,
589                                 "I-cache maintenance fault");
590         }
591 
592         if (cpu_architecture() >= CPU_ARCH_ARMv7) {
593                 /*
594                  * TODO: Access flag faults introduced in ARMv6K.
595                  * Runtime check for 'K' extension is needed
596                  */
597                 hook_fault_code(3, do_bad, SIGSEGV, SEGV_MAPERR,
598                                 "section access flag fault");
599                 hook_fault_code(6, do_bad, SIGSEGV, SEGV_MAPERR,
600                                 "section access flag fault");
601         }
602 
603         return 0;
604 }
605 
606 arch_initcall(exceptions_init);
607 #endif
608 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp