~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/tile/mm/fault.c

Version: ~ [ linux-5.4-rc7 ] ~ [ linux-5.3.11 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.84 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.154 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.201 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.201 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.77 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Copyright 2010 Tilera Corporation. All Rights Reserved.
  3  *
  4  *   This program is free software; you can redistribute it and/or
  5  *   modify it under the terms of the GNU General Public License
  6  *   as published by the Free Software Foundation, version 2.
  7  *
  8  *   This program is distributed in the hope that it will be useful, but
  9  *   WITHOUT ANY WARRANTY; without even the implied warranty of
 10  *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
 11  *   NON INFRINGEMENT.  See the GNU General Public License for
 12  *   more details.
 13  *
 14  * From i386 code copyright (C) 1995  Linus Torvalds
 15  */
 16 
 17 #include <linux/signal.h>
 18 #include <linux/sched.h>
 19 #include <linux/kernel.h>
 20 #include <linux/errno.h>
 21 #include <linux/string.h>
 22 #include <linux/types.h>
 23 #include <linux/ptrace.h>
 24 #include <linux/mman.h>
 25 #include <linux/mm.h>
 26 #include <linux/smp.h>
 27 #include <linux/interrupt.h>
 28 #include <linux/init.h>
 29 #include <linux/tty.h>
 30 #include <linux/vt_kern.h>              /* For unblank_screen() */
 31 #include <linux/highmem.h>
 32 #include <linux/module.h>
 33 #include <linux/kprobes.h>
 34 #include <linux/hugetlb.h>
 35 #include <linux/syscalls.h>
 36 #include <linux/uaccess.h>
 37 #include <linux/kdebug.h>
 38 #include <linux/context_tracking.h>
 39 
 40 #include <asm/pgalloc.h>
 41 #include <asm/sections.h>
 42 #include <asm/traps.h>
 43 #include <asm/syscalls.h>
 44 
 45 #include <arch/interrupts.h>
 46 
 47 static noinline void force_sig_info_fault(const char *type, int si_signo,
 48                                           int si_code, unsigned long address,
 49                                           int fault_num,
 50                                           struct task_struct *tsk,
 51                                           struct pt_regs *regs)
 52 {
 53         siginfo_t info;
 54 
 55         if (unlikely(tsk->pid < 2)) {
 56                 panic("Signal %d (code %d) at %#lx sent to %s!",
 57                       si_signo, si_code & 0xffff, address,
 58                       is_idle_task(tsk) ? "the idle task" : "init");
 59         }
 60 
 61         info.si_signo = si_signo;
 62         info.si_errno = 0;
 63         info.si_code = si_code;
 64         info.si_addr = (void __user *)address;
 65         info.si_trapno = fault_num;
 66         trace_unhandled_signal(type, regs, address, si_signo);
 67         force_sig_info(si_signo, &info, tsk);
 68 }
 69 
 70 #ifndef __tilegx__
 71 /*
 72  * Synthesize the fault a PL0 process would get by doing a word-load of
 73  * an unaligned address or a high kernel address.
 74  */
 75 SYSCALL_DEFINE1(cmpxchg_badaddr, unsigned long, address)
 76 {
 77         struct pt_regs *regs = current_pt_regs();
 78 
 79         if (address >= PAGE_OFFSET)
 80                 force_sig_info_fault("atomic segfault", SIGSEGV, SEGV_MAPERR,
 81                                      address, INT_DTLB_MISS, current, regs);
 82         else
 83                 force_sig_info_fault("atomic alignment fault", SIGBUS,
 84                                      BUS_ADRALN, address,
 85                                      INT_UNALIGN_DATA, current, regs);
 86 
 87         /*
 88          * Adjust pc to point at the actual instruction, which is unusual
 89          * for syscalls normally, but is appropriate when we are claiming
 90          * that a syscall swint1 caused a page fault or bus error.
 91          */
 92         regs->pc -= 8;
 93 
 94         /*
 95          * Mark this as a caller-save interrupt, like a normal page fault,
 96          * so that when we go through the signal handler path we will
 97          * properly restore r0, r1, and r2 for the signal handler arguments.
 98          */
 99         regs->flags |= PT_FLAGS_CALLER_SAVES;
100 
101         return 0;
102 }
103 #endif
104 
105 static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
106 {
107         unsigned index = pgd_index(address);
108         pgd_t *pgd_k;
109         pud_t *pud, *pud_k;
110         pmd_t *pmd, *pmd_k;
111 
112         pgd += index;
113         pgd_k = init_mm.pgd + index;
114 
115         if (!pgd_present(*pgd_k))
116                 return NULL;
117 
118         pud = pud_offset(pgd, address);
119         pud_k = pud_offset(pgd_k, address);
120         if (!pud_present(*pud_k))
121                 return NULL;
122 
123         pmd = pmd_offset(pud, address);
124         pmd_k = pmd_offset(pud_k, address);
125         if (!pmd_present(*pmd_k))
126                 return NULL;
127         if (!pmd_present(*pmd))
128                 set_pmd(pmd, *pmd_k);
129         else
130                 BUG_ON(pmd_ptfn(*pmd) != pmd_ptfn(*pmd_k));
131         return pmd_k;
132 }
133 
134 /*
135  * Handle a fault on the vmalloc area.
136  */
137 static inline int vmalloc_fault(pgd_t *pgd, unsigned long address)
138 {
139         pmd_t *pmd_k;
140         pte_t *pte_k;
141 
142         /* Make sure we are in vmalloc area */
143         if (!(address >= VMALLOC_START && address < VMALLOC_END))
144                 return -1;
145 
146         /*
147          * Synchronize this task's top level page-table
148          * with the 'reference' page table.
149          */
150         pmd_k = vmalloc_sync_one(pgd, address);
151         if (!pmd_k)
152                 return -1;
153         pte_k = pte_offset_kernel(pmd_k, address);
154         if (!pte_present(*pte_k))
155                 return -1;
156         return 0;
157 }
158 
159 /* Wait until this PTE has completed migration. */
160 static void wait_for_migration(pte_t *pte)
161 {
162         if (pte_migrating(*pte)) {
163                 /*
164                  * Wait until the migrater fixes up this pte.
165                  * We scale the loop count by the clock rate so we'll wait for
166                  * a few seconds here.
167                  */
168                 int retries = 0;
169                 int bound = get_clock_rate();
170                 while (pte_migrating(*pte)) {
171                         barrier();
172                         if (++retries > bound)
173                                 panic("Hit migrating PTE (%#llx) and page PFN %#lx still migrating",
174                                       pte->val, pte_pfn(*pte));
175                 }
176         }
177 }
178 
179 /*
180  * It's not generally safe to use "current" to get the page table pointer,
181  * since we might be running an oprofile interrupt in the middle of a
182  * task switch.
183  */
184 static pgd_t *get_current_pgd(void)
185 {
186         HV_Context ctx = hv_inquire_context();
187         unsigned long pgd_pfn = ctx.page_table >> PAGE_SHIFT;
188         struct page *pgd_page = pfn_to_page(pgd_pfn);
189         BUG_ON(PageHighMem(pgd_page));
190         return (pgd_t *) __va(ctx.page_table);
191 }
192 
193 /*
194  * We can receive a page fault from a migrating PTE at any time.
195  * Handle it by just waiting until the fault resolves.
196  *
197  * It's also possible to get a migrating kernel PTE that resolves
198  * itself during the downcall from hypervisor to Linux.  We just check
199  * here to see if the PTE seems valid, and if so we retry it.
200  *
201  * NOTE! We MUST NOT take any locks for this case.  We may be in an
202  * interrupt or a critical region, and must do as little as possible.
203  * Similarly, we can't use atomic ops here, since we may be handling a
204  * fault caused by an atomic op access.
205  *
206  * If we find a migrating PTE while we're in an NMI context, and we're
207  * at a PC that has a registered exception handler, we don't wait,
208  * since this thread may (e.g.) have been interrupted while migrating
209  * its own stack, which would then cause us to self-deadlock.
210  */
211 static int handle_migrating_pte(pgd_t *pgd, int fault_num,
212                                 unsigned long address, unsigned long pc,
213                                 int is_kernel_mode, int write)
214 {
215         pud_t *pud;
216         pmd_t *pmd;
217         pte_t *pte;
218         pte_t pteval;
219 
220         if (pgd_addr_invalid(address))
221                 return 0;
222 
223         pgd += pgd_index(address);
224         pud = pud_offset(pgd, address);
225         if (!pud || !pud_present(*pud))
226                 return 0;
227         pmd = pmd_offset(pud, address);
228         if (!pmd || !pmd_present(*pmd))
229                 return 0;
230         pte = pmd_huge_page(*pmd) ? ((pte_t *)pmd) :
231                 pte_offset_kernel(pmd, address);
232         pteval = *pte;
233         if (pte_migrating(pteval)) {
234                 if (in_nmi() && search_exception_tables(pc))
235                         return 0;
236                 wait_for_migration(pte);
237                 return 1;
238         }
239 
240         if (!is_kernel_mode || !pte_present(pteval))
241                 return 0;
242         if (fault_num == INT_ITLB_MISS) {
243                 if (pte_exec(pteval))
244                         return 1;
245         } else if (write) {
246                 if (pte_write(pteval))
247                         return 1;
248         } else {
249                 if (pte_read(pteval))
250                         return 1;
251         }
252 
253         return 0;
254 }
255 
256 /*
257  * This routine is responsible for faulting in user pages.
258  * It passes the work off to one of the appropriate routines.
259  * It returns true if the fault was successfully handled.
260  */
261 static int handle_page_fault(struct pt_regs *regs,
262                              int fault_num,
263                              int is_page_fault,
264                              unsigned long address,
265                              int write)
266 {
267         struct task_struct *tsk;
268         struct mm_struct *mm;
269         struct vm_area_struct *vma;
270         unsigned long stack_offset;
271         int fault;
272         int si_code;
273         int is_kernel_mode;
274         pgd_t *pgd;
275         unsigned int flags;
276 
277         /* on TILE, protection faults are always writes */
278         if (!is_page_fault)
279                 write = 1;
280 
281         flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
282 
283         is_kernel_mode = !user_mode(regs);
284 
285         tsk = validate_current();
286 
287         /*
288          * Check to see if we might be overwriting the stack, and bail
289          * out if so.  The page fault code is a relatively likely
290          * place to get trapped in an infinite regress, and once we
291          * overwrite the whole stack, it becomes very hard to recover.
292          */
293         stack_offset = stack_pointer & (THREAD_SIZE-1);
294         if (stack_offset < THREAD_SIZE / 8) {
295                 pr_alert("Potential stack overrun: sp %#lx\n", stack_pointer);
296                 show_regs(regs);
297                 pr_alert("Killing current process %d/%s\n",
298                          tsk->pid, tsk->comm);
299                 do_group_exit(SIGKILL);
300         }
301 
302         /*
303          * Early on, we need to check for migrating PTE entries;
304          * see homecache.c.  If we find a migrating PTE, we wait until
305          * the backing page claims to be done migrating, then we proceed.
306          * For kernel PTEs, we rewrite the PTE and return and retry.
307          * Otherwise, we treat the fault like a normal "no PTE" fault,
308          * rather than trying to patch up the existing PTE.
309          */
310         pgd = get_current_pgd();
311         if (handle_migrating_pte(pgd, fault_num, address, regs->pc,
312                                  is_kernel_mode, write))
313                 return 1;
314 
315         si_code = SEGV_MAPERR;
316 
317         /*
318          * We fault-in kernel-space virtual memory on-demand. The
319          * 'reference' page table is init_mm.pgd.
320          *
321          * NOTE! We MUST NOT take any locks for this case. We may
322          * be in an interrupt or a critical region, and should
323          * only copy the information from the master page table,
324          * nothing more.
325          *
326          * This verifies that the fault happens in kernel space
327          * and that the fault was not a protection fault.
328          */
329         if (unlikely(address >= TASK_SIZE &&
330                      !is_arch_mappable_range(address, 0))) {
331                 if (is_kernel_mode && is_page_fault &&
332                     vmalloc_fault(pgd, address) >= 0)
333                         return 1;
334                 /*
335                  * Don't take the mm semaphore here. If we fixup a prefetch
336                  * fault we could otherwise deadlock.
337                  */
338                 mm = NULL;  /* happy compiler */
339                 vma = NULL;
340                 goto bad_area_nosemaphore;
341         }
342 
343         /*
344          * If we're trying to touch user-space addresses, we must
345          * be either at PL0, or else with interrupts enabled in the
346          * kernel, so either way we can re-enable interrupts here
347          * unless we are doing atomic access to user space with
348          * interrupts disabled.
349          */
350         if (!(regs->flags & PT_FLAGS_DISABLE_IRQ))
351                 local_irq_enable();
352 
353         mm = tsk->mm;
354 
355         /*
356          * If we're in an interrupt, have no user context or are running in an
357          * atomic region then we must not take the fault.
358          */
359         if (in_atomic() || !mm) {
360                 vma = NULL;  /* happy compiler */
361                 goto bad_area_nosemaphore;
362         }
363 
364         if (!is_kernel_mode)
365                 flags |= FAULT_FLAG_USER;
366 
367         /*
368          * When running in the kernel we expect faults to occur only to
369          * addresses in user space.  All other faults represent errors in the
370          * kernel and should generate an OOPS.  Unfortunately, in the case of an
371          * erroneous fault occurring in a code path which already holds mmap_sem
372          * we will deadlock attempting to validate the fault against the
373          * address space.  Luckily the kernel only validly references user
374          * space from well defined areas of code, which are listed in the
375          * exceptions table.
376          *
377          * As the vast majority of faults will be valid we will only perform
378          * the source reference check when there is a possibility of a deadlock.
379          * Attempt to lock the address space, if we cannot we then validate the
380          * source.  If this is invalid we can skip the address space check,
381          * thus avoiding the deadlock.
382          */
383         if (!down_read_trylock(&mm->mmap_sem)) {
384                 if (is_kernel_mode &&
385                     !search_exception_tables(regs->pc)) {
386                         vma = NULL;  /* happy compiler */
387                         goto bad_area_nosemaphore;
388                 }
389 
390 retry:
391                 down_read(&mm->mmap_sem);
392         }
393 
394         vma = find_vma(mm, address);
395         if (!vma)
396                 goto bad_area;
397         if (vma->vm_start <= address)
398                 goto good_area;
399         if (!(vma->vm_flags & VM_GROWSDOWN))
400                 goto bad_area;
401         if (regs->sp < PAGE_OFFSET) {
402                 /*
403                  * accessing the stack below sp is always a bug.
404                  */
405                 if (address < regs->sp)
406                         goto bad_area;
407         }
408         if (expand_stack(vma, address))
409                 goto bad_area;
410 
411 /*
412  * Ok, we have a good vm_area for this memory access, so
413  * we can handle it..
414  */
415 good_area:
416         si_code = SEGV_ACCERR;
417         if (fault_num == INT_ITLB_MISS) {
418                 if (!(vma->vm_flags & VM_EXEC))
419                         goto bad_area;
420         } else if (write) {
421 #ifdef TEST_VERIFY_AREA
422                 if (!is_page_fault && regs->cs == KERNEL_CS)
423                         pr_err("WP fault at " REGFMT "\n", regs->eip);
424 #endif
425                 if (!(vma->vm_flags & VM_WRITE))
426                         goto bad_area;
427                 flags |= FAULT_FLAG_WRITE;
428         } else {
429                 if (!is_page_fault || !(vma->vm_flags & VM_READ))
430                         goto bad_area;
431         }
432 
433         /*
434          * If for any reason at all we couldn't handle the fault,
435          * make sure we exit gracefully rather than endlessly redo
436          * the fault.
437          */
438         fault = handle_mm_fault(mm, vma, address, flags);
439 
440         if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
441                 return 0;
442 
443         if (unlikely(fault & VM_FAULT_ERROR)) {
444                 if (fault & VM_FAULT_OOM)
445                         goto out_of_memory;
446                 else if (fault & VM_FAULT_SIGSEGV)
447                         goto bad_area;
448                 else if (fault & VM_FAULT_SIGBUS)
449                         goto do_sigbus;
450                 BUG();
451         }
452         if (flags & FAULT_FLAG_ALLOW_RETRY) {
453                 if (fault & VM_FAULT_MAJOR)
454                         tsk->maj_flt++;
455                 else
456                         tsk->min_flt++;
457                 if (fault & VM_FAULT_RETRY) {
458                         flags &= ~FAULT_FLAG_ALLOW_RETRY;
459                         flags |= FAULT_FLAG_TRIED;
460 
461                          /*
462                           * No need to up_read(&mm->mmap_sem) as we would
463                           * have already released it in __lock_page_or_retry
464                           * in mm/filemap.c.
465                           */
466                         goto retry;
467                 }
468         }
469 
470 #if CHIP_HAS_TILE_DMA()
471         /* If this was a DMA TLB fault, restart the DMA engine. */
472         switch (fault_num) {
473         case INT_DMATLB_MISS:
474         case INT_DMATLB_MISS_DWNCL:
475         case INT_DMATLB_ACCESS:
476         case INT_DMATLB_ACCESS_DWNCL:
477                 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK);
478                 break;
479         }
480 #endif
481 
482         up_read(&mm->mmap_sem);
483         return 1;
484 
485 /*
486  * Something tried to access memory that isn't in our memory map..
487  * Fix it, but check if it's kernel or user first..
488  */
489 bad_area:
490         up_read(&mm->mmap_sem);
491 
492 bad_area_nosemaphore:
493         /* User mode accesses just cause a SIGSEGV */
494         if (!is_kernel_mode) {
495                 /*
496                  * It's possible to have interrupts off here.
497                  */
498                 local_irq_enable();
499 
500                 force_sig_info_fault("segfault", SIGSEGV, si_code, address,
501                                      fault_num, tsk, regs);
502                 return 0;
503         }
504 
505 no_context:
506         /* Are we prepared to handle this kernel fault?  */
507         if (fixup_exception(regs))
508                 return 0;
509 
510 /*
511  * Oops. The kernel tried to access some bad page. We'll have to
512  * terminate things with extreme prejudice.
513  */
514 
515         bust_spinlocks(1);
516 
517         /* FIXME: no lookup_address() yet */
518 #ifdef SUPPORT_LOOKUP_ADDRESS
519         if (fault_num == INT_ITLB_MISS) {
520                 pte_t *pte = lookup_address(address);
521 
522                 if (pte && pte_present(*pte) && !pte_exec_kernel(*pte))
523                         pr_crit("kernel tried to execute non-executable page - exploit attempt? (uid: %d)\n",
524                                 current->uid);
525         }
526 #endif
527         if (address < PAGE_SIZE)
528                 pr_alert("Unable to handle kernel NULL pointer dereference\n");
529         else
530                 pr_alert("Unable to handle kernel paging request\n");
531         pr_alert(" at virtual address " REGFMT ", pc " REGFMT "\n",
532                  address, regs->pc);
533 
534         show_regs(regs);
535 
536         if (unlikely(tsk->pid < 2)) {
537                 panic("Kernel page fault running %s!",
538                       is_idle_task(tsk) ? "the idle task" : "init");
539         }
540 
541         /*
542          * More FIXME: we should probably copy the i386 here and
543          * implement a generic die() routine.  Not today.
544          */
545 #ifdef SUPPORT_DIE
546         die("Oops", regs);
547 #endif
548         bust_spinlocks(1);
549 
550         do_group_exit(SIGKILL);
551 
552 /*
553  * We ran out of memory, or some other thing happened to us that made
554  * us unable to handle the page fault gracefully.
555  */
556 out_of_memory:
557         up_read(&mm->mmap_sem);
558         if (is_kernel_mode)
559                 goto no_context;
560         pagefault_out_of_memory();
561         return 0;
562 
563 do_sigbus:
564         up_read(&mm->mmap_sem);
565 
566         /* Kernel mode? Handle exceptions or die */
567         if (is_kernel_mode)
568                 goto no_context;
569 
570         force_sig_info_fault("bus error", SIGBUS, BUS_ADRERR, address,
571                              fault_num, tsk, regs);
572         return 0;
573 }
574 
575 #ifndef __tilegx__
576 
577 /* We must release ICS before panicking or we won't get anywhere. */
578 #define ics_panic(fmt, ...)                                     \
579 do {                                                            \
580         __insn_mtspr(SPR_INTERRUPT_CRITICAL_SECTION, 0);        \
581         panic(fmt, ##__VA_ARGS__);                              \
582 } while (0)
583 
584 /*
585  * When we take an ITLB or DTLB fault or access violation in the
586  * supervisor while the critical section bit is set, the hypervisor is
587  * reluctant to write new values into the EX_CONTEXT_K_x registers,
588  * since that might indicate we have not yet squirreled the SPR
589  * contents away and can thus safely take a recursive interrupt.
590  * Accordingly, the hypervisor passes us the PC via SYSTEM_SAVE_K_2.
591  *
592  * Note that this routine is called before homecache_tlb_defer_enter(),
593  * which means that we can properly unlock any atomics that might
594  * be used there (good), but also means we must be very sensitive
595  * to not touch any data structures that might be located in memory
596  * that could migrate, as we could be entering the kernel on a dataplane
597  * cpu that has been deferring kernel TLB updates.  This means, for
598  * example, that we can't migrate init_mm or its pgd.
599  */
600 struct intvec_state do_page_fault_ics(struct pt_regs *regs, int fault_num,
601                                       unsigned long address,
602                                       unsigned long info)
603 {
604         unsigned long pc = info & ~1;
605         int write = info & 1;
606         pgd_t *pgd = get_current_pgd();
607 
608         /* Retval is 1 at first since we will handle the fault fully. */
609         struct intvec_state state = {
610                 do_page_fault, fault_num, address, write, 1
611         };
612 
613         /* Validate that we are plausibly in the right routine. */
614         if ((pc & 0x7) != 0 || pc < PAGE_OFFSET ||
615             (fault_num != INT_DTLB_MISS &&
616              fault_num != INT_DTLB_ACCESS)) {
617                 unsigned long old_pc = regs->pc;
618                 regs->pc = pc;
619                 ics_panic("Bad ICS page fault args: old PC %#lx, fault %d/%d at %#lx",
620                           old_pc, fault_num, write, address);
621         }
622 
623         /* We might be faulting on a vmalloc page, so check that first. */
624         if (fault_num != INT_DTLB_ACCESS && vmalloc_fault(pgd, address) >= 0)
625                 return state;
626 
627         /*
628          * If we faulted with ICS set in sys_cmpxchg, we are providing
629          * a user syscall service that should generate a signal on
630          * fault.  We didn't set up a kernel stack on initial entry to
631          * sys_cmpxchg, but instead had one set up by the fault, which
632          * (because sys_cmpxchg never releases ICS) came to us via the
633          * SYSTEM_SAVE_K_2 mechanism, and thus EX_CONTEXT_K_[01] are
634          * still referencing the original user code.  We release the
635          * atomic lock and rewrite pt_regs so that it appears that we
636          * came from user-space directly, and after we finish the
637          * fault we'll go back to user space and re-issue the swint.
638          * This way the backtrace information is correct if we need to
639          * emit a stack dump at any point while handling this.
640          *
641          * Must match register use in sys_cmpxchg().
642          */
643         if (pc >= (unsigned long) sys_cmpxchg &&
644             pc < (unsigned long) __sys_cmpxchg_end) {
645 #ifdef CONFIG_SMP
646                 /* Don't unlock before we could have locked. */
647                 if (pc >= (unsigned long)__sys_cmpxchg_grab_lock) {
648                         int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]);
649                         __atomic_fault_unlock(lock_ptr);
650                 }
651 #endif
652                 regs->sp = regs->regs[27];
653         }
654 
655         /*
656          * We can also fault in the atomic assembly, in which
657          * case we use the exception table to do the first-level fixup.
658          * We may re-fixup again in the real fault handler if it
659          * turns out the faulting address is just bad, and not,
660          * for example, migrating.
661          */
662         else if (pc >= (unsigned long) __start_atomic_asm_code &&
663                    pc < (unsigned long) __end_atomic_asm_code) {
664                 const struct exception_table_entry *fixup;
665 #ifdef CONFIG_SMP
666                 /* Unlock the atomic lock. */
667                 int *lock_ptr = (int *)(regs->regs[ATOMIC_LOCK_REG]);
668                 __atomic_fault_unlock(lock_ptr);
669 #endif
670                 fixup = search_exception_tables(pc);
671                 if (!fixup)
672                         ics_panic("ICS atomic fault not in table: PC %#lx, fault %d",
673                                   pc, fault_num);
674                 regs->pc = fixup->fixup;
675                 regs->ex1 = PL_ICS_EX1(KERNEL_PL, 0);
676         }
677 
678         /*
679          * Now that we have released the atomic lock (if necessary),
680          * it's safe to spin if the PTE that caused the fault was migrating.
681          */
682         if (fault_num == INT_DTLB_ACCESS)
683                 write = 1;
684         if (handle_migrating_pte(pgd, fault_num, address, pc, 1, write))
685                 return state;
686 
687         /* Return zero so that we continue on with normal fault handling. */
688         state.retval = 0;
689         return state;
690 }
691 
692 #endif /* !__tilegx__ */
693 
694 /*
695  * This routine handles page faults.  It determines the address, and the
696  * problem, and then passes it handle_page_fault() for normal DTLB and
697  * ITLB issues, and for DMA or SN processor faults when we are in user
698  * space.  For the latter, if we're in kernel mode, we just save the
699  * interrupt away appropriately and return immediately.  We can't do
700  * page faults for user code while in kernel mode.
701  */
702 void do_page_fault(struct pt_regs *regs, int fault_num,
703                    unsigned long address, unsigned long write)
704 {
705         int is_page_fault;
706         enum ctx_state prev_state = exception_enter();
707 
708 #ifdef CONFIG_KPROBES
709         /*
710          * This is to notify the fault handler of the kprobes.  The
711          * exception code is redundant as it is also carried in REGS,
712          * but we pass it anyhow.
713          */
714         if (notify_die(DIE_PAGE_FAULT, "page fault", regs, -1,
715                        regs->faultnum, SIGSEGV) == NOTIFY_STOP)
716                 goto done;
717 #endif
718 
719 #ifdef __tilegx__
720         /*
721          * We don't need early do_page_fault_ics() support, since unlike
722          * Pro we don't need to worry about unlocking the atomic locks.
723          * There is only one current case in GX where we touch any memory
724          * under ICS other than our own kernel stack, and we handle that
725          * here.  (If we crash due to trying to touch our own stack,
726          * we're in too much trouble for C code to help out anyway.)
727          */
728         if (write & ~1) {
729                 unsigned long pc = write & ~1;
730                 if (pc >= (unsigned long) __start_unalign_asm_code &&
731                     pc < (unsigned long) __end_unalign_asm_code) {
732                         struct thread_info *ti = current_thread_info();
733                         /*
734                          * Our EX_CONTEXT is still what it was from the
735                          * initial unalign exception, but now we've faulted
736                          * on the JIT page.  We would like to complete the
737                          * page fault however is appropriate, and then retry
738                          * the instruction that caused the unalign exception.
739                          * Our state has been "corrupted" by setting the low
740                          * bit in "sp", and stashing r0..r3 in the
741                          * thread_info area, so we revert all of that, then
742                          * continue as if this were a normal page fault.
743                          */
744                         regs->sp &= ~1UL;
745                         regs->regs[0] = ti->unalign_jit_tmp[0];
746                         regs->regs[1] = ti->unalign_jit_tmp[1];
747                         regs->regs[2] = ti->unalign_jit_tmp[2];
748                         regs->regs[3] = ti->unalign_jit_tmp[3];
749                         write &= 1;
750                 } else {
751                         pr_alert("%s/%d: ICS set at page fault at %#lx: %#lx\n",
752                                  current->comm, current->pid, pc, address);
753                         show_regs(regs);
754                         do_group_exit(SIGKILL);
755                 }
756         }
757 #else
758         /* This case should have been handled by do_page_fault_ics(). */
759         BUG_ON(write & ~1);
760 #endif
761 
762 #if CHIP_HAS_TILE_DMA()
763         /*
764          * If it's a DMA fault, suspend the transfer while we're
765          * handling the miss; we'll restart after it's handled.  If we
766          * don't suspend, it's possible that this process could swap
767          * out and back in, and restart the engine since the DMA is
768          * still 'running'.
769          */
770         if (fault_num == INT_DMATLB_MISS ||
771             fault_num == INT_DMATLB_ACCESS ||
772             fault_num == INT_DMATLB_MISS_DWNCL ||
773             fault_num == INT_DMATLB_ACCESS_DWNCL) {
774                 __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__SUSPEND_MASK);
775                 while (__insn_mfspr(SPR_DMA_USER_STATUS) &
776                        SPR_DMA_STATUS__BUSY_MASK)
777                         ;
778         }
779 #endif
780 
781         /* Validate fault num and decide if this is a first-time page fault. */
782         switch (fault_num) {
783         case INT_ITLB_MISS:
784         case INT_DTLB_MISS:
785 #if CHIP_HAS_TILE_DMA()
786         case INT_DMATLB_MISS:
787         case INT_DMATLB_MISS_DWNCL:
788 #endif
789                 is_page_fault = 1;
790                 break;
791 
792         case INT_DTLB_ACCESS:
793 #if CHIP_HAS_TILE_DMA()
794         case INT_DMATLB_ACCESS:
795         case INT_DMATLB_ACCESS_DWNCL:
796 #endif
797                 is_page_fault = 0;
798                 break;
799 
800         default:
801                 panic("Bad fault number %d in do_page_fault", fault_num);
802         }
803 
804 #if CHIP_HAS_TILE_DMA()
805         if (!user_mode(regs)) {
806                 struct async_tlb *async;
807                 switch (fault_num) {
808 #if CHIP_HAS_TILE_DMA()
809                 case INT_DMATLB_MISS:
810                 case INT_DMATLB_ACCESS:
811                 case INT_DMATLB_MISS_DWNCL:
812                 case INT_DMATLB_ACCESS_DWNCL:
813                         async = &current->thread.dma_async_tlb;
814                         break;
815 #endif
816                 default:
817                         async = NULL;
818                 }
819                 if (async) {
820 
821                         /*
822                          * No vmalloc check required, so we can allow
823                          * interrupts immediately at this point.
824                          */
825                         local_irq_enable();
826 
827                         set_thread_flag(TIF_ASYNC_TLB);
828                         if (async->fault_num != 0) {
829                                 panic("Second async fault %d; old fault was %d (%#lx/%ld)",
830                                       fault_num, async->fault_num,
831                                       address, write);
832                         }
833                         BUG_ON(fault_num == 0);
834                         async->fault_num = fault_num;
835                         async->is_fault = is_page_fault;
836                         async->is_write = write;
837                         async->address = address;
838                         goto done;
839                 }
840         }
841 #endif
842 
843         handle_page_fault(regs, fault_num, is_page_fault, address, write);
844 
845 done:
846         exception_exit(prev_state);
847 }
848 
849 
850 #if CHIP_HAS_TILE_DMA()
851 /*
852  * This routine effectively re-issues asynchronous page faults
853  * when we are returning to user space.
854  */
855 void do_async_page_fault(struct pt_regs *regs)
856 {
857         struct async_tlb *async = &current->thread.dma_async_tlb;
858 
859         /*
860          * Clear thread flag early.  If we re-interrupt while processing
861          * code here, we will reset it and recall this routine before
862          * returning to user space.
863          */
864         clear_thread_flag(TIF_ASYNC_TLB);
865 
866         if (async->fault_num) {
867                 /*
868                  * Clear async->fault_num before calling the page-fault
869                  * handler so that if we re-interrupt before returning
870                  * from the function we have somewhere to put the
871                  * information from the new interrupt.
872                  */
873                 int fault_num = async->fault_num;
874                 async->fault_num = 0;
875                 handle_page_fault(regs, fault_num, async->is_fault,
876                                   async->address, async->is_write);
877         }
878 }
879 #endif /* CHIP_HAS_TILE_DMA() */
880 
881 
882 void vmalloc_sync_all(void)
883 {
884 #ifdef __tilegx__
885         /* Currently all L1 kernel pmd's are static and shared. */
886         BUILD_BUG_ON(pgd_index(VMALLOC_END - PAGE_SIZE) !=
887                      pgd_index(VMALLOC_START));
888 #else
889         /*
890          * Note that races in the updates of insync and start aren't
891          * problematic: insync can only get set bits added, and updates to
892          * start are only improving performance (without affecting correctness
893          * if undone).
894          */
895         static DECLARE_BITMAP(insync, PTRS_PER_PGD);
896         static unsigned long start = PAGE_OFFSET;
897         unsigned long address;
898 
899         BUILD_BUG_ON(PAGE_OFFSET & ~PGDIR_MASK);
900         for (address = start; address >= PAGE_OFFSET; address += PGDIR_SIZE) {
901                 if (!test_bit(pgd_index(address), insync)) {
902                         unsigned long flags;
903                         struct list_head *pos;
904 
905                         spin_lock_irqsave(&pgd_lock, flags);
906                         list_for_each(pos, &pgd_list)
907                                 if (!vmalloc_sync_one(list_to_pgd(pos),
908                                                                 address)) {
909                                         /* Must be at first entry in list. */
910                                         BUG_ON(pos != pgd_list.next);
911                                         break;
912                                 }
913                         spin_unlock_irqrestore(&pgd_lock, flags);
914                         if (pos != pgd_list.next)
915                                 set_bit(pgd_index(address), insync);
916                 }
917                 if (address == start && test_bit(pgd_index(address), insync))
918                         start = address + PGDIR_SIZE;
919         }
920 #endif
921 }
922 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp