~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/i386/mm/fault.c

Version: ~ [ linux-5.8 ] ~ [ linux-5.7.12 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.55 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.136 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.191 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.232 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.232 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *  linux/arch/i386/mm/fault.c
  3  *
  4  *  Copyright (C) 1995  Linus Torvalds
  5  */
  6 
  7 #include <linux/signal.h>
  8 #include <linux/sched.h>
  9 #include <linux/kernel.h>
 10 #include <linux/errno.h>
 11 #include <linux/string.h>
 12 #include <linux/types.h>
 13 #include <linux/ptrace.h>
 14 #include <linux/mman.h>
 15 #include <linux/mm.h>
 16 #include <linux/smp.h>
 17 #include <linux/smp_lock.h>
 18 #include <linux/interrupt.h>
 19 #include <linux/init.h>
 20 #include <linux/tty.h>
 21 #include <linux/vt_kern.h>              /* For unblank_screen() */
 22 #include <linux/highmem.h>
 23 #include <linux/module.h>
 24 
 25 #include <asm/system.h>
 26 #include <asm/uaccess.h>
 27 #include <asm/pgalloc.h>
 28 #include <asm/hardirq.h>
 29 #include <asm/desc.h>
 30 
 31 extern void die(const char *,struct pt_regs *,long);
 32 
 33 /*
 34  * Unlock any spinlocks which will prevent us from getting the
 35  * message out 
 36  */
 37 void bust_spinlocks(int yes)
 38 {
 39         int loglevel_save = console_loglevel;
 40 
 41         if (yes) {
 42                 oops_in_progress = 1;
 43                 return;
 44         }
 45 #ifdef CONFIG_VT
 46         unblank_screen();
 47 #endif
 48         oops_in_progress = 0;
 49         /*
 50          * OK, the message is on the console.  Now we call printk()
 51          * without oops_in_progress set so that printk will give klogd
 52          * a poke.  Hold onto your hats...
 53          */
 54         console_loglevel = 15;          /* NMI oopser may have shut the console up */
 55         printk(" ");
 56         console_loglevel = loglevel_save;
 57 }
 58 
 59 /*
 60  * Return EIP plus the CS segment base.  The segment limit is also
 61  * adjusted, clamped to the kernel/user address space (whichever is
 62  * appropriate), and returned in *eip_limit.
 63  *
 64  * The segment is checked, because it might have been changed by another
 65  * task between the original faulting instruction and here.
 66  *
 67  * If CS is no longer a valid code segment, or if EIP is beyond the
 68  * limit, or if it is a kernel address when CS is not a kernel segment,
 69  * then the returned value will be greater than *eip_limit.
 70  * 
 71  * This is slow, but is very rarely executed.
 72  */
 73 static inline unsigned long get_segment_eip(struct pt_regs *regs,
 74                                             unsigned long *eip_limit)
 75 {
 76         unsigned long eip = regs->eip;
 77         unsigned seg = regs->xcs & 0xffff;
 78         u32 seg_ar, seg_limit, base, *desc;
 79 
 80         /* The standard kernel/user address space limit. */
 81         *eip_limit = (seg & 3) ? USER_DS.seg : KERNEL_DS.seg;
 82 
 83         /* Unlikely, but must come before segment checks. */
 84         if (unlikely((regs->eflags & VM_MASK) != 0))
 85                 return eip + (seg << 4);
 86         
 87         /* By far the most common cases. */
 88         if (likely(seg == __USER_CS || seg == __KERNEL_CS))
 89                 return eip;
 90 
 91         /* Check the segment exists, is within the current LDT/GDT size,
 92            that kernel/user (ring 0..3) has the appropriate privilege,
 93            that it's a code segment, and get the limit. */
 94         __asm__ ("larl %3,%0; lsll %3,%1"
 95                  : "=&r" (seg_ar), "=r" (seg_limit) : "" (0), "rm" (seg));
 96         if ((~seg_ar & 0x9800) || eip > seg_limit) {
 97                 *eip_limit = 0;
 98                 return 1;        /* So that returned eip > *eip_limit. */
 99         }
100 
101         /* Get the GDT/LDT descriptor base. 
102            When you look for races in this code remember that
103            LDT and other horrors are only used in user space. */
104         if (seg & (1<<2)) {
105                 /* Must lock the LDT while reading it. */
106                 down(&current->mm->context.sem);
107                 desc = current->mm->context.ldt;
108                 desc = (void *)desc + (seg & ~7);
109         } else {
110                 /* Must disable preemption while reading the GDT. */
111                 desc = (u32 *)&cpu_gdt_table[get_cpu()];
112                 desc = (void *)desc + (seg & ~7);
113         }
114 
115         /* Decode the code segment base from the descriptor */
116         base =   (desc[0] >> 16) |
117                 ((desc[1] & 0xff) << 16) |
118                  (desc[1] & 0xff000000);
119 
120         if (seg & (1<<2)) { 
121                 up(&current->mm->context.sem);
122         } else
123                 put_cpu();
124 
125         /* Adjust EIP and segment limit, and clamp at the kernel limit.
126            It's legitimate for segments to wrap at 0xffffffff. */
127         seg_limit += base;
128         if (seg_limit < *eip_limit && seg_limit >= base)
129                 *eip_limit = seg_limit;
130         return eip + base;
131 }
132 
133 /* 
134  * Sometimes AMD Athlon/Opteron CPUs report invalid exceptions on prefetch.
135  * Check that here and ignore it.
136  */
137 static int __is_prefetch(struct pt_regs *regs, unsigned long addr)
138 { 
139         unsigned long limit;
140         unsigned long instr = get_segment_eip (regs, &limit);
141         int scan_more = 1;
142         int prefetch = 0; 
143         int i;
144 
145         for (i = 0; scan_more && i < 15; i++) { 
146                 unsigned char opcode;
147                 unsigned char instr_hi;
148                 unsigned char instr_lo;
149 
150                 if (instr > limit)
151                         break;
152                 if (__get_user(opcode, (unsigned char *) instr))
153                         break; 
154 
155                 instr_hi = opcode & 0xf0; 
156                 instr_lo = opcode & 0x0f; 
157                 instr++;
158 
159                 switch (instr_hi) { 
160                 case 0x20:
161                 case 0x30:
162                         /* Values 0x26,0x2E,0x36,0x3E are valid x86 prefixes. */
163                         scan_more = ((instr_lo & 7) == 0x6);
164                         break;
165                         
166                 case 0x60:
167                         /* 0x64 thru 0x67 are valid prefixes in all modes. */
168                         scan_more = (instr_lo & 0xC) == 0x4;
169                         break;          
170                 case 0xF0:
171                         /* 0xF0, 0xF2, and 0xF3 are valid prefixes */
172                         scan_more = !instr_lo || (instr_lo>>1) == 1;
173                         break;                  
174                 case 0x00:
175                         /* Prefetch instruction is 0x0F0D or 0x0F18 */
176                         scan_more = 0;
177                         if (instr > limit)
178                                 break;
179                         if (__get_user(opcode, (unsigned char *) instr)) 
180                                 break;
181                         prefetch = (instr_lo == 0xF) &&
182                                 (opcode == 0x0D || opcode == 0x18);
183                         break;                  
184                 default:
185                         scan_more = 0;
186                         break;
187                 } 
188         }
189         return prefetch;
190 }
191 
192 static inline int is_prefetch(struct pt_regs *regs, unsigned long addr)
193 {
194         if (unlikely(boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
195                      boot_cpu_data.x86 >= 6))
196                 return __is_prefetch(regs, addr);
197         return 0;
198 } 
199 
200 asmlinkage void do_invalid_op(struct pt_regs *, unsigned long);
201 
202 /*
203  * This routine handles page faults.  It determines the address,
204  * and the problem, and then passes it off to one of the appropriate
205  * routines.
206  *
207  * error_code:
208  *      bit 0 == 0 means no page found, 1 means protection fault
209  *      bit 1 == 0 means read, 1 means write
210  *      bit 2 == 0 means kernel, 1 means user-mode
211  */
212 asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code)
213 {
214         struct task_struct *tsk;
215         struct mm_struct *mm;
216         struct vm_area_struct * vma;
217         unsigned long address;
218         unsigned long page;
219         int write;
220         siginfo_t info;
221 
222         /* get the address */
223         __asm__("movl %%cr2,%0":"=r" (address));
224 
225         /* It's safe to allow irq's after cr2 has been saved */
226         if (regs->eflags & (X86_EFLAGS_IF|VM_MASK))
227                 local_irq_enable();
228 
229         tsk = current;
230 
231         info.si_code = SEGV_MAPERR;
232 
233         /*
234          * We fault-in kernel-space virtual memory on-demand. The
235          * 'reference' page table is init_mm.pgd.
236          *
237          * NOTE! We MUST NOT take any locks for this case. We may
238          * be in an interrupt or a critical region, and should
239          * only copy the information from the master page table,
240          * nothing more.
241          *
242          * This verifies that the fault happens in kernel space
243          * (error_code & 4) == 0, and that the fault was not a
244          * protection error (error_code & 1) == 0.
245          */
246         if (unlikely(address >= TASK_SIZE)) { 
247                 if (!(error_code & 5))
248                         goto vmalloc_fault;
249                 /* 
250                  * Don't take the mm semaphore here. If we fixup a prefetch
251                  * fault we could otherwise deadlock.
252                  */
253                 goto bad_area_nosemaphore;
254         } 
255 
256         mm = tsk->mm;
257 
258         /*
259          * If we're in an interrupt, have no user context or are running in an
260          * atomic region then we must not take the fault..
261          */
262         if (in_atomic() || !mm)
263                 goto bad_area_nosemaphore;
264 
265         down_read(&mm->mmap_sem);
266 
267         vma = find_vma(mm, address);
268         if (!vma)
269                 goto bad_area;
270         if (vma->vm_start <= address)
271                 goto good_area;
272         if (!(vma->vm_flags & VM_GROWSDOWN))
273                 goto bad_area;
274         if (error_code & 4) {
275                 /*
276                  * accessing the stack below %esp is always a bug.
277                  * The "+ 32" is there due to some instructions (like
278                  * pusha) doing post-decrement on the stack and that
279                  * doesn't show up until later..
280                  */
281                 if (address + 32 < regs->esp)
282                         goto bad_area;
283         }
284         if (expand_stack(vma, address))
285                 goto bad_area;
286 /*
287  * Ok, we have a good vm_area for this memory access, so
288  * we can handle it..
289  */
290 good_area:
291         info.si_code = SEGV_ACCERR;
292         write = 0;
293         switch (error_code & 3) {
294                 default:        /* 3: write, present */
295 #ifdef TEST_VERIFY_AREA
296                         if (regs->cs == KERNEL_CS)
297                                 printk("WP fault at %08lx\n", regs->eip);
298 #endif
299                         /* fall through */
300                 case 2:         /* write, not present */
301                         if (!(vma->vm_flags & VM_WRITE))
302                                 goto bad_area;
303                         write++;
304                         break;
305                 case 1:         /* read, present */
306                         goto bad_area;
307                 case 0:         /* read, not present */
308                         if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
309                                 goto bad_area;
310         }
311 
312  survive:
313         /*
314          * If for any reason at all we couldn't handle the fault,
315          * make sure we exit gracefully rather than endlessly redo
316          * the fault.
317          */
318         switch (handle_mm_fault(mm, vma, address, write)) {
319                 case VM_FAULT_MINOR:
320                         tsk->min_flt++;
321                         break;
322                 case VM_FAULT_MAJOR:
323                         tsk->maj_flt++;
324                         break;
325                 case VM_FAULT_SIGBUS:
326                         goto do_sigbus;
327                 case VM_FAULT_OOM:
328                         goto out_of_memory;
329                 default:
330                         BUG();
331         }
332 
333         /*
334          * Did it hit the DOS screen memory VA from vm86 mode?
335          */
336         if (regs->eflags & VM_MASK) {
337                 unsigned long bit = (address - 0xA0000) >> PAGE_SHIFT;
338                 if (bit < 32)
339                         tsk->thread.screen_bitmap |= 1 << bit;
340         }
341         up_read(&mm->mmap_sem);
342         return;
343 
344 /*
345  * Something tried to access memory that isn't in our memory map..
346  * Fix it, but check if it's kernel or user first..
347  */
348 bad_area:
349         up_read(&mm->mmap_sem);
350 
351 bad_area_nosemaphore:
352         /* User mode accesses just cause a SIGSEGV */
353         if (error_code & 4) {
354                 /* 
355                  * Valid to do another page fault here because this one came 
356                  * from user space.
357                  */
358                 if (is_prefetch(regs, address))
359                         return;
360 
361                 tsk->thread.cr2 = address;
362                 /* Kernel addresses are always protection faults */
363                 tsk->thread.error_code = error_code | (address >= TASK_SIZE);
364                 tsk->thread.trap_no = 14;
365                 info.si_signo = SIGSEGV;
366                 info.si_errno = 0;
367                 /* info.si_code has been set above */
368                 info.si_addr = (void *)address;
369                 force_sig_info(SIGSEGV, &info, tsk);
370                 return;
371         }
372 
373 #ifdef CONFIG_X86_F00F_BUG
374         /*
375          * Pentium F0 0F C7 C8 bug workaround.
376          */
377         if (boot_cpu_data.f00f_bug) {
378                 unsigned long nr;
379                 
380                 nr = (address - idt_descr.address) >> 3;
381 
382                 if (nr == 6) {
383                         do_invalid_op(regs, 0);
384                         return;
385                 }
386         }
387 #endif
388 
389 no_context:
390         /* Are we prepared to handle this kernel fault?  */
391         if (fixup_exception(regs))
392                 return;
393 
394         /* 
395          * Valid to do another page fault here, because if this fault
396          * had been triggered by is_prefetch fixup_exception would have 
397          * handled it.
398          */
399         if (is_prefetch(regs, address))
400                 return;
401 
402 /*
403  * Oops. The kernel tried to access some bad page. We'll have to
404  * terminate things with extreme prejudice.
405  */
406 
407         bust_spinlocks(1);
408 
409         if (address < PAGE_SIZE)
410                 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
411         else
412                 printk(KERN_ALERT "Unable to handle kernel paging request");
413         printk(" at virtual address %08lx\n",address);
414         printk(" printing eip:\n");
415         printk("%08lx\n", regs->eip);
416         asm("movl %%cr3,%0":"=r" (page));
417         page = ((unsigned long *) __va(page))[address >> 22];
418         printk(KERN_ALERT "*pde = %08lx\n", page);
419         /*
420          * We must not directly access the pte in the highpte
421          * case, the page table might be allocated in highmem.
422          * And lets rather not kmap-atomic the pte, just in case
423          * it's allocated already.
424          */
425 #ifndef CONFIG_HIGHPTE
426         if (page & 1) {
427                 page &= PAGE_MASK;
428                 address &= 0x003ff000;
429                 page = ((unsigned long *) __va(page))[address >> PAGE_SHIFT];
430                 printk(KERN_ALERT "*pte = %08lx\n", page);
431         }
432 #endif
433         die("Oops", regs, error_code);
434         bust_spinlocks(0);
435         do_exit(SIGKILL);
436 
437 /*
438  * We ran out of memory, or some other thing happened to us that made
439  * us unable to handle the page fault gracefully.
440  */
441 out_of_memory:
442         up_read(&mm->mmap_sem);
443         if (tsk->pid == 1) {
444                 yield();
445                 down_read(&mm->mmap_sem);
446                 goto survive;
447         }
448         printk("VM: killing process %s\n", tsk->comm);
449         if (error_code & 4)
450                 do_exit(SIGKILL);
451         goto no_context;
452 
453 do_sigbus:
454         up_read(&mm->mmap_sem);
455 
456         /* Kernel mode? Handle exceptions or die */
457         if (!(error_code & 4))
458                 goto no_context;
459 
460         /* User space => ok to do another page fault */
461         if (is_prefetch(regs, address))
462                 return;
463 
464         tsk->thread.cr2 = address;
465         tsk->thread.error_code = error_code;
466         tsk->thread.trap_no = 14;
467         info.si_signo = SIGBUS;
468         info.si_errno = 0;
469         info.si_code = BUS_ADRERR;
470         info.si_addr = (void *)address;
471         force_sig_info(SIGBUS, &info, tsk);
472         return;
473 
474 vmalloc_fault:
475         {
476                 /*
477                  * Synchronize this task's top level page-table
478                  * with the 'reference' page table.
479                  *
480                  * Do _not_ use "tsk" here. We might be inside
481                  * an interrupt in the middle of a task switch..
482                  */
483                 int index = pgd_index(address);
484                 pgd_t *pgd, *pgd_k;
485                 pmd_t *pmd, *pmd_k;
486                 pte_t *pte_k;
487 
488                 asm("movl %%cr3,%0":"=r" (pgd));
489                 pgd = index + (pgd_t *)__va(pgd);
490                 pgd_k = init_mm.pgd + index;
491 
492                 if (!pgd_present(*pgd_k))
493                         goto no_context;
494 
495                 /*
496                  * set_pgd(pgd, *pgd_k); here would be useless on PAE
497                  * and redundant with the set_pmd() on non-PAE.
498                  */
499 
500                 pmd = pmd_offset(pgd, address);
501                 pmd_k = pmd_offset(pgd_k, address);
502                 if (!pmd_present(*pmd_k))
503                         goto no_context;
504                 set_pmd(pmd, *pmd_k);
505 
506                 pte_k = pte_offset_kernel(pmd_k, address);
507                 if (!pte_present(*pte_k))
508                         goto no_context;
509                 return;
510         }
511 }
512 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp