~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/nommu.c

Version: ~ [ linux-5.8-rc3 ] ~ [ linux-5.7.5 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.48 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.129 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.185 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.228 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.228 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *  linux/mm/nommu.c
  3  *
  4  *  Replacement code for mm functions to support CPU's that don't
  5  *  have any form of memory management unit (thus no virtual memory).
  6  *
  7  *  See Documentation/nommu-mmap.txt
  8  *
  9  *  Copyright (c) 2004-2008 David Howells <dhowells@redhat.com>
 10  *  Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com>
 11  *  Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org>
 12  *  Copyright (c) 2002      Greg Ungerer <gerg@snapgear.com>
 13  *  Copyright (c) 2007-2010 Paul Mundt <lethal@linux-sh.org>
 14  */
 15 
 16 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 17 
 18 #include <linux/export.h>
 19 #include <linux/mm.h>
 20 #include <linux/vmacache.h>
 21 #include <linux/mman.h>
 22 #include <linux/swap.h>
 23 #include <linux/file.h>
 24 #include <linux/highmem.h>
 25 #include <linux/pagemap.h>
 26 #include <linux/slab.h>
 27 #include <linux/vmalloc.h>
 28 #include <linux/blkdev.h>
 29 #include <linux/backing-dev.h>
 30 #include <linux/compiler.h>
 31 #include <linux/mount.h>
 32 #include <linux/personality.h>
 33 #include <linux/security.h>
 34 #include <linux/syscalls.h>
 35 #include <linux/audit.h>
 36 #include <linux/printk.h>
 37 
 38 #include <asm/uaccess.h>
 39 #include <asm/tlb.h>
 40 #include <asm/tlbflush.h>
 41 #include <asm/mmu_context.h>
 42 #include "internal.h"
 43 
 44 void *high_memory;
 45 EXPORT_SYMBOL(high_memory);
 46 struct page *mem_map;
 47 unsigned long max_mapnr;
 48 EXPORT_SYMBOL(max_mapnr);
 49 unsigned long highest_memmap_pfn;
 50 int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS;
 51 int heap_stack_gap = 0;
 52 
 53 atomic_long_t mmap_pages_allocated;
 54 
 55 EXPORT_SYMBOL(mem_map);
 56 
 57 /* list of mapped, potentially shareable regions */
 58 static struct kmem_cache *vm_region_jar;
 59 struct rb_root nommu_region_tree = RB_ROOT;
 60 DECLARE_RWSEM(nommu_region_sem);
 61 
 62 const struct vm_operations_struct generic_file_vm_ops = {
 63 };
 64 
 65 /*
 66  * Return the total memory allocated for this pointer, not
 67  * just what the caller asked for.
 68  *
 69  * Doesn't have to be accurate, i.e. may have races.
 70  */
 71 unsigned int kobjsize(const void *objp)
 72 {
 73         struct page *page;
 74 
 75         /*
 76          * If the object we have should not have ksize performed on it,
 77          * return size of 0
 78          */
 79         if (!objp || !virt_addr_valid(objp))
 80                 return 0;
 81 
 82         page = virt_to_head_page(objp);
 83 
 84         /*
 85          * If the allocator sets PageSlab, we know the pointer came from
 86          * kmalloc().
 87          */
 88         if (PageSlab(page))
 89                 return ksize(objp);
 90 
 91         /*
 92          * If it's not a compound page, see if we have a matching VMA
 93          * region. This test is intentionally done in reverse order,
 94          * so if there's no VMA, we still fall through and hand back
 95          * PAGE_SIZE for 0-order pages.
 96          */
 97         if (!PageCompound(page)) {
 98                 struct vm_area_struct *vma;
 99 
100                 vma = find_vma(current->mm, (unsigned long)objp);
101                 if (vma)
102                         return vma->vm_end - vma->vm_start;
103         }
104 
105         /*
106          * The ksize() function is only guaranteed to work for pointers
107          * returned by kmalloc(). So handle arbitrary pointers here.
108          */
109         return PAGE_SIZE << compound_order(page);
110 }
111 
112 long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
113                       unsigned long start, unsigned long nr_pages,
114                       unsigned int foll_flags, struct page **pages,
115                       struct vm_area_struct **vmas, int *nonblocking)
116 {
117         struct vm_area_struct *vma;
118         unsigned long vm_flags;
119         int i;
120 
121         /* calculate required read or write permissions.
122          * If FOLL_FORCE is set, we only require the "MAY" flags.
123          */
124         vm_flags  = (foll_flags & FOLL_WRITE) ?
125                         (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
126         vm_flags &= (foll_flags & FOLL_FORCE) ?
127                         (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
128 
129         for (i = 0; i < nr_pages; i++) {
130                 vma = find_vma(mm, start);
131                 if (!vma)
132                         goto finish_or_fault;
133 
134                 /* protect what we can, including chardevs */
135                 if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
136                     !(vm_flags & vma->vm_flags))
137                         goto finish_or_fault;
138 
139                 if (pages) {
140                         pages[i] = virt_to_page(start);
141                         if (pages[i])
142                                 get_page(pages[i]);
143                 }
144                 if (vmas)
145                         vmas[i] = vma;
146                 start = (start + PAGE_SIZE) & PAGE_MASK;
147         }
148 
149         return i;
150 
151 finish_or_fault:
152         return i ? : -EFAULT;
153 }
154 
155 /*
156  * get a list of pages in an address range belonging to the specified process
157  * and indicate the VMA that covers each page
158  * - this is potentially dodgy as we may end incrementing the page count of a
159  *   slab page or a secondary page from a compound page
160  * - don't permit access to VMAs that don't support it, such as I/O mappings
161  */
162 long get_user_pages(unsigned long start, unsigned long nr_pages,
163                     int write, int force, struct page **pages,
164                     struct vm_area_struct **vmas)
165 {
166         int flags = 0;
167 
168         if (write)
169                 flags |= FOLL_WRITE;
170         if (force)
171                 flags |= FOLL_FORCE;
172 
173         return __get_user_pages(current, current->mm, start, nr_pages, flags,
174                                 pages, vmas, NULL);
175 }
176 EXPORT_SYMBOL(get_user_pages);
177 
178 long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
179                             int write, int force, struct page **pages,
180                             int *locked)
181 {
182         return get_user_pages(start, nr_pages, write, force, pages, NULL);
183 }
184 EXPORT_SYMBOL(get_user_pages_locked);
185 
186 long __get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
187                                unsigned long start, unsigned long nr_pages,
188                                int write, int force, struct page **pages,
189                                unsigned int gup_flags)
190 {
191         long ret;
192         down_read(&mm->mmap_sem);
193         ret = __get_user_pages(tsk, mm, start, nr_pages, gup_flags, pages,
194                                 NULL, NULL);
195         up_read(&mm->mmap_sem);
196         return ret;
197 }
198 EXPORT_SYMBOL(__get_user_pages_unlocked);
199 
200 long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
201                              int write, int force, struct page **pages)
202 {
203         return __get_user_pages_unlocked(current, current->mm, start, nr_pages,
204                                          write, force, pages, 0);
205 }
206 EXPORT_SYMBOL(get_user_pages_unlocked);
207 
208 /**
209  * follow_pfn - look up PFN at a user virtual address
210  * @vma: memory mapping
211  * @address: user virtual address
212  * @pfn: location to store found PFN
213  *
214  * Only IO mappings and raw PFN mappings are allowed.
215  *
216  * Returns zero and the pfn at @pfn on success, -ve otherwise.
217  */
218 int follow_pfn(struct vm_area_struct *vma, unsigned long address,
219         unsigned long *pfn)
220 {
221         if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
222                 return -EINVAL;
223 
224         *pfn = address >> PAGE_SHIFT;
225         return 0;
226 }
227 EXPORT_SYMBOL(follow_pfn);
228 
229 LIST_HEAD(vmap_area_list);
230 
231 void vfree(const void *addr)
232 {
233         kfree(addr);
234 }
235 EXPORT_SYMBOL(vfree);
236 
237 void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
238 {
239         /*
240          *  You can't specify __GFP_HIGHMEM with kmalloc() since kmalloc()
241          * returns only a logical address.
242          */
243         return kmalloc(size, (gfp_mask | __GFP_COMP) & ~__GFP_HIGHMEM);
244 }
245 EXPORT_SYMBOL(__vmalloc);
246 
247 void *vmalloc_user(unsigned long size)
248 {
249         void *ret;
250 
251         ret = __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
252                         PAGE_KERNEL);
253         if (ret) {
254                 struct vm_area_struct *vma;
255 
256                 down_write(&current->mm->mmap_sem);
257                 vma = find_vma(current->mm, (unsigned long)ret);
258                 if (vma)
259                         vma->vm_flags |= VM_USERMAP;
260                 up_write(&current->mm->mmap_sem);
261         }
262 
263         return ret;
264 }
265 EXPORT_SYMBOL(vmalloc_user);
266 
267 struct page *vmalloc_to_page(const void *addr)
268 {
269         return virt_to_page(addr);
270 }
271 EXPORT_SYMBOL(vmalloc_to_page);
272 
273 unsigned long vmalloc_to_pfn(const void *addr)
274 {
275         return page_to_pfn(virt_to_page(addr));
276 }
277 EXPORT_SYMBOL(vmalloc_to_pfn);
278 
279 long vread(char *buf, char *addr, unsigned long count)
280 {
281         /* Don't allow overflow */
282         if ((unsigned long) buf + count < count)
283                 count = -(unsigned long) buf;
284 
285         memcpy(buf, addr, count);
286         return count;
287 }
288 
289 long vwrite(char *buf, char *addr, unsigned long count)
290 {
291         /* Don't allow overflow */
292         if ((unsigned long) addr + count < count)
293                 count = -(unsigned long) addr;
294 
295         memcpy(addr, buf, count);
296         return count;
297 }
298 
299 /*
300  *      vmalloc  -  allocate virtually contiguous memory
301  *
302  *      @size:          allocation size
303  *
304  *      Allocate enough pages to cover @size from the page level
305  *      allocator and map them into contiguous kernel virtual space.
306  *
307  *      For tight control over page level allocator and protection flags
308  *      use __vmalloc() instead.
309  */
310 void *vmalloc(unsigned long size)
311 {
312        return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL);
313 }
314 EXPORT_SYMBOL(vmalloc);
315 
316 /*
317  *      vzalloc - allocate virtually contiguous memory with zero fill
318  *
319  *      @size:          allocation size
320  *
321  *      Allocate enough pages to cover @size from the page level
322  *      allocator and map them into contiguous kernel virtual space.
323  *      The memory allocated is set to zero.
324  *
325  *      For tight control over page level allocator and protection flags
326  *      use __vmalloc() instead.
327  */
328 void *vzalloc(unsigned long size)
329 {
330         return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
331                         PAGE_KERNEL);
332 }
333 EXPORT_SYMBOL(vzalloc);
334 
335 /**
336  * vmalloc_node - allocate memory on a specific node
337  * @size:       allocation size
338  * @node:       numa node
339  *
340  * Allocate enough pages to cover @size from the page level
341  * allocator and map them into contiguous kernel virtual space.
342  *
343  * For tight control over page level allocator and protection flags
344  * use __vmalloc() instead.
345  */
346 void *vmalloc_node(unsigned long size, int node)
347 {
348         return vmalloc(size);
349 }
350 EXPORT_SYMBOL(vmalloc_node);
351 
352 /**
353  * vzalloc_node - allocate memory on a specific node with zero fill
354  * @size:       allocation size
355  * @node:       numa node
356  *
357  * Allocate enough pages to cover @size from the page level
358  * allocator and map them into contiguous kernel virtual space.
359  * The memory allocated is set to zero.
360  *
361  * For tight control over page level allocator and protection flags
362  * use __vmalloc() instead.
363  */
364 void *vzalloc_node(unsigned long size, int node)
365 {
366         return vzalloc(size);
367 }
368 EXPORT_SYMBOL(vzalloc_node);
369 
370 #ifndef PAGE_KERNEL_EXEC
371 # define PAGE_KERNEL_EXEC PAGE_KERNEL
372 #endif
373 
374 /**
375  *      vmalloc_exec  -  allocate virtually contiguous, executable memory
376  *      @size:          allocation size
377  *
378  *      Kernel-internal function to allocate enough pages to cover @size
379  *      the page level allocator and map them into contiguous and
380  *      executable kernel virtual space.
381  *
382  *      For tight control over page level allocator and protection flags
383  *      use __vmalloc() instead.
384  */
385 
386 void *vmalloc_exec(unsigned long size)
387 {
388         return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
389 }
390 
391 /**
392  * vmalloc_32  -  allocate virtually contiguous memory (32bit addressable)
393  *      @size:          allocation size
394  *
395  *      Allocate enough 32bit PA addressable pages to cover @size from the
396  *      page level allocator and map them into contiguous kernel virtual space.
397  */
398 void *vmalloc_32(unsigned long size)
399 {
400         return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL);
401 }
402 EXPORT_SYMBOL(vmalloc_32);
403 
404 /**
405  * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
406  *      @size:          allocation size
407  *
408  * The resulting memory area is 32bit addressable and zeroed so it can be
409  * mapped to userspace without leaking data.
410  *
411  * VM_USERMAP is set on the corresponding VMA so that subsequent calls to
412  * remap_vmalloc_range() are permissible.
413  */
414 void *vmalloc_32_user(unsigned long size)
415 {
416         /*
417          * We'll have to sort out the ZONE_DMA bits for 64-bit,
418          * but for now this can simply use vmalloc_user() directly.
419          */
420         return vmalloc_user(size);
421 }
422 EXPORT_SYMBOL(vmalloc_32_user);
423 
424 void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot)
425 {
426         BUG();
427         return NULL;
428 }
429 EXPORT_SYMBOL(vmap);
430 
431 void vunmap(const void *addr)
432 {
433         BUG();
434 }
435 EXPORT_SYMBOL(vunmap);
436 
437 void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot)
438 {
439         BUG();
440         return NULL;
441 }
442 EXPORT_SYMBOL(vm_map_ram);
443 
444 void vm_unmap_ram(const void *mem, unsigned int count)
445 {
446         BUG();
447 }
448 EXPORT_SYMBOL(vm_unmap_ram);
449 
450 void vm_unmap_aliases(void)
451 {
452 }
453 EXPORT_SYMBOL_GPL(vm_unmap_aliases);
454 
455 /*
456  * Implement a stub for vmalloc_sync_all() if the architecture chose not to
457  * have one.
458  */
459 void __weak vmalloc_sync_all(void)
460 {
461 }
462 
463 /**
464  *      alloc_vm_area - allocate a range of kernel address space
465  *      @size:          size of the area
466  *
467  *      Returns:        NULL on failure, vm_struct on success
468  *
469  *      This function reserves a range of kernel address space, and
470  *      allocates pagetables to map that range.  No actual mappings
471  *      are created.  If the kernel address space is not shared
472  *      between processes, it syncs the pagetable across all
473  *      processes.
474  */
475 struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes)
476 {
477         BUG();
478         return NULL;
479 }
480 EXPORT_SYMBOL_GPL(alloc_vm_area);
481 
482 void free_vm_area(struct vm_struct *area)
483 {
484         BUG();
485 }
486 EXPORT_SYMBOL_GPL(free_vm_area);
487 
488 int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
489                    struct page *page)
490 {
491         return -EINVAL;
492 }
493 EXPORT_SYMBOL(vm_insert_page);
494 
495 /*
496  *  sys_brk() for the most part doesn't need the global kernel
497  *  lock, except when an application is doing something nasty
498  *  like trying to un-brk an area that has already been mapped
499  *  to a regular file.  in this case, the unmapping will need
500  *  to invoke file system routines that need the global lock.
501  */
502 SYSCALL_DEFINE1(brk, unsigned long, brk)
503 {
504         struct mm_struct *mm = current->mm;
505 
506         if (brk < mm->start_brk || brk > mm->context.end_brk)
507                 return mm->brk;
508 
509         if (mm->brk == brk)
510                 return mm->brk;
511 
512         /*
513          * Always allow shrinking brk
514          */
515         if (brk <= mm->brk) {
516                 mm->brk = brk;
517                 return brk;
518         }
519 
520         /*
521          * Ok, looks good - let it rip.
522          */
523         flush_icache_range(mm->brk, brk);
524         return mm->brk = brk;
525 }
526 
527 /*
528  * initialise the VMA and region record slabs
529  */
530 void __init mmap_init(void)
531 {
532         int ret;
533 
534         ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
535         VM_BUG_ON(ret);
536         vm_region_jar = KMEM_CACHE(vm_region, SLAB_PANIC|SLAB_ACCOUNT);
537 }
538 
539 /*
540  * validate the region tree
541  * - the caller must hold the region lock
542  */
543 #ifdef CONFIG_DEBUG_NOMMU_REGIONS
544 static noinline void validate_nommu_regions(void)
545 {
546         struct vm_region *region, *last;
547         struct rb_node *p, *lastp;
548 
549         lastp = rb_first(&nommu_region_tree);
550         if (!lastp)
551                 return;
552 
553         last = rb_entry(lastp, struct vm_region, vm_rb);
554         BUG_ON(last->vm_end <= last->vm_start);
555         BUG_ON(last->vm_top < last->vm_end);
556 
557         while ((p = rb_next(lastp))) {
558                 region = rb_entry(p, struct vm_region, vm_rb);
559                 last = rb_entry(lastp, struct vm_region, vm_rb);
560 
561                 BUG_ON(region->vm_end <= region->vm_start);
562                 BUG_ON(region->vm_top < region->vm_end);
563                 BUG_ON(region->vm_start < last->vm_top);
564 
565                 lastp = p;
566         }
567 }
568 #else
569 static void validate_nommu_regions(void)
570 {
571 }
572 #endif
573 
574 /*
575  * add a region into the global tree
576  */
577 static void add_nommu_region(struct vm_region *region)
578 {
579         struct vm_region *pregion;
580         struct rb_node **p, *parent;
581 
582         validate_nommu_regions();
583 
584         parent = NULL;
585         p = &nommu_region_tree.rb_node;
586         while (*p) {
587                 parent = *p;
588                 pregion = rb_entry(parent, struct vm_region, vm_rb);
589                 if (region->vm_start < pregion->vm_start)
590                         p = &(*p)->rb_left;
591                 else if (region->vm_start > pregion->vm_start)
592                         p = &(*p)->rb_right;
593                 else if (pregion == region)
594                         return;
595                 else
596                         BUG();
597         }
598 
599         rb_link_node(&region->vm_rb, parent, p);
600         rb_insert_color(&region->vm_rb, &nommu_region_tree);
601 
602         validate_nommu_regions();
603 }
604 
605 /*
606  * delete a region from the global tree
607  */
608 static void delete_nommu_region(struct vm_region *region)
609 {
610         BUG_ON(!nommu_region_tree.rb_node);
611 
612         validate_nommu_regions();
613         rb_erase(&region->vm_rb, &nommu_region_tree);
614         validate_nommu_regions();
615 }
616 
617 /*
618  * free a contiguous series of pages
619  */
620 static void free_page_series(unsigned long from, unsigned long to)
621 {
622         for (; from < to; from += PAGE_SIZE) {
623                 struct page *page = virt_to_page(from);
624 
625                 atomic_long_dec(&mmap_pages_allocated);
626                 put_page(page);
627         }
628 }
629 
630 /*
631  * release a reference to a region
632  * - the caller must hold the region semaphore for writing, which this releases
633  * - the region may not have been added to the tree yet, in which case vm_top
634  *   will equal vm_start
635  */
636 static void __put_nommu_region(struct vm_region *region)
637         __releases(nommu_region_sem)
638 {
639         BUG_ON(!nommu_region_tree.rb_node);
640 
641         if (--region->vm_usage == 0) {
642                 if (region->vm_top > region->vm_start)
643                         delete_nommu_region(region);
644                 up_write(&nommu_region_sem);
645 
646                 if (region->vm_file)
647                         fput(region->vm_file);
648 
649                 /* IO memory and memory shared directly out of the pagecache
650                  * from ramfs/tmpfs mustn't be released here */
651                 if (region->vm_flags & VM_MAPPED_COPY)
652                         free_page_series(region->vm_start, region->vm_top);
653                 kmem_cache_free(vm_region_jar, region);
654         } else {
655                 up_write(&nommu_region_sem);
656         }
657 }
658 
659 /*
660  * release a reference to a region
661  */
662 static void put_nommu_region(struct vm_region *region)
663 {
664         down_write(&nommu_region_sem);
665         __put_nommu_region(region);
666 }
667 
668 /*
669  * update protection on a vma
670  */
671 static void protect_vma(struct vm_area_struct *vma, unsigned long flags)
672 {
673 #ifdef CONFIG_MPU
674         struct mm_struct *mm = vma->vm_mm;
675         long start = vma->vm_start & PAGE_MASK;
676         while (start < vma->vm_end) {
677                 protect_page(mm, start, flags);
678                 start += PAGE_SIZE;
679         }
680         update_protections(mm);
681 #endif
682 }
683 
684 /*
685  * add a VMA into a process's mm_struct in the appropriate place in the list
686  * and tree and add to the address space's page tree also if not an anonymous
687  * page
688  * - should be called with mm->mmap_sem held writelocked
689  */
690 static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
691 {
692         struct vm_area_struct *pvma, *prev;
693         struct address_space *mapping;
694         struct rb_node **p, *parent, *rb_prev;
695 
696         BUG_ON(!vma->vm_region);
697 
698         mm->map_count++;
699         vma->vm_mm = mm;
700 
701         protect_vma(vma, vma->vm_flags);
702 
703         /* add the VMA to the mapping */
704         if (vma->vm_file) {
705                 mapping = vma->vm_file->f_mapping;
706 
707                 i_mmap_lock_write(mapping);
708                 flush_dcache_mmap_lock(mapping);
709                 vma_interval_tree_insert(vma, &mapping->i_mmap);
710                 flush_dcache_mmap_unlock(mapping);
711                 i_mmap_unlock_write(mapping);
712         }
713 
714         /* add the VMA to the tree */
715         parent = rb_prev = NULL;
716         p = &mm->mm_rb.rb_node;
717         while (*p) {
718                 parent = *p;
719                 pvma = rb_entry(parent, struct vm_area_struct, vm_rb);
720 
721                 /* sort by: start addr, end addr, VMA struct addr in that order
722                  * (the latter is necessary as we may get identical VMAs) */
723                 if (vma->vm_start < pvma->vm_start)
724                         p = &(*p)->rb_left;
725                 else if (vma->vm_start > pvma->vm_start) {
726                         rb_prev = parent;
727                         p = &(*p)->rb_right;
728                 } else if (vma->vm_end < pvma->vm_end)
729                         p = &(*p)->rb_left;
730                 else if (vma->vm_end > pvma->vm_end) {
731                         rb_prev = parent;
732                         p = &(*p)->rb_right;
733                 } else if (vma < pvma)
734                         p = &(*p)->rb_left;
735                 else if (vma > pvma) {
736                         rb_prev = parent;
737                         p = &(*p)->rb_right;
738                 } else
739                         BUG();
740         }
741 
742         rb_link_node(&vma->vm_rb, parent, p);
743         rb_insert_color(&vma->vm_rb, &mm->mm_rb);
744 
745         /* add VMA to the VMA list also */
746         prev = NULL;
747         if (rb_prev)
748                 prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
749 
750         __vma_link_list(mm, vma, prev, parent);
751 }
752 
753 /*
754  * delete a VMA from its owning mm_struct and address space
755  */
756 static void delete_vma_from_mm(struct vm_area_struct *vma)
757 {
758         int i;
759         struct address_space *mapping;
760         struct mm_struct *mm = vma->vm_mm;
761         struct task_struct *curr = current;
762 
763         protect_vma(vma, 0);
764 
765         mm->map_count--;
766         for (i = 0; i < VMACACHE_SIZE; i++) {
767                 /* if the vma is cached, invalidate the entire cache */
768                 if (curr->vmacache[i] == vma) {
769                         vmacache_invalidate(mm);
770                         break;
771                 }
772         }
773 
774         /* remove the VMA from the mapping */
775         if (vma->vm_file) {
776                 mapping = vma->vm_file->f_mapping;
777 
778                 i_mmap_lock_write(mapping);
779                 flush_dcache_mmap_lock(mapping);
780                 vma_interval_tree_remove(vma, &mapping->i_mmap);
781                 flush_dcache_mmap_unlock(mapping);
782                 i_mmap_unlock_write(mapping);
783         }
784 
785         /* remove from the MM's tree and list */
786         rb_erase(&vma->vm_rb, &mm->mm_rb);
787 
788         if (vma->vm_prev)
789                 vma->vm_prev->vm_next = vma->vm_next;
790         else
791                 mm->mmap = vma->vm_next;
792 
793         if (vma->vm_next)
794                 vma->vm_next->vm_prev = vma->vm_prev;
795 }
796 
797 /*
798  * destroy a VMA record
799  */
800 static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma)
801 {
802         if (vma->vm_ops && vma->vm_ops->close)
803                 vma->vm_ops->close(vma);
804         if (vma->vm_file)
805                 fput(vma->vm_file);
806         put_nommu_region(vma->vm_region);
807         kmem_cache_free(vm_area_cachep, vma);
808 }
809 
810 /*
811  * look up the first VMA in which addr resides, NULL if none
812  * - should be called with mm->mmap_sem at least held readlocked
813  */
814 struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
815 {
816         struct vm_area_struct *vma;
817 
818         /* check the cache first */
819         vma = vmacache_find(mm, addr);
820         if (likely(vma))
821                 return vma;
822 
823         /* trawl the list (there may be multiple mappings in which addr
824          * resides) */
825         for (vma = mm->mmap; vma; vma = vma->vm_next) {
826                 if (vma->vm_start > addr)
827                         return NULL;
828                 if (vma->vm_end > addr) {
829                         vmacache_update(addr, vma);
830                         return vma;
831                 }
832         }
833 
834         return NULL;
835 }
836 EXPORT_SYMBOL(find_vma);
837 
838 /*
839  * find a VMA
840  * - we don't extend stack VMAs under NOMMU conditions
841  */
842 struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr)
843 {
844         return find_vma(mm, addr);
845 }
846 
847 /*
848  * expand a stack to a given address
849  * - not supported under NOMMU conditions
850  */
851 int expand_stack(struct vm_area_struct *vma, unsigned long address)
852 {
853         return -ENOMEM;
854 }
855 
856 /*
857  * look up the first VMA exactly that exactly matches addr
858  * - should be called with mm->mmap_sem at least held readlocked
859  */
860 static struct vm_area_struct *find_vma_exact(struct mm_struct *mm,
861                                              unsigned long addr,
862                                              unsigned long len)
863 {
864         struct vm_area_struct *vma;
865         unsigned long end = addr + len;
866 
867         /* check the cache first */
868         vma = vmacache_find_exact(mm, addr, end);
869         if (vma)
870                 return vma;
871 
872         /* trawl the list (there may be multiple mappings in which addr
873          * resides) */
874         for (vma = mm->mmap; vma; vma = vma->vm_next) {
875                 if (vma->vm_start < addr)
876                         continue;
877                 if (vma->vm_start > addr)
878                         return NULL;
879                 if (vma->vm_end == end) {
880                         vmacache_update(addr, vma);
881                         return vma;
882                 }
883         }
884 
885         return NULL;
886 }
887 
888 /*
889  * determine whether a mapping should be permitted and, if so, what sort of
890  * mapping we're capable of supporting
891  */
892 static int validate_mmap_request(struct file *file,
893                                  unsigned long addr,
894                                  unsigned long len,
895                                  unsigned long prot,
896                                  unsigned long flags,
897                                  unsigned long pgoff,
898                                  unsigned long *_capabilities)
899 {
900         unsigned long capabilities, rlen;
901         int ret;
902 
903         /* do the simple checks first */
904         if (flags & MAP_FIXED)
905                 return -EINVAL;
906 
907         if ((flags & MAP_TYPE) != MAP_PRIVATE &&
908             (flags & MAP_TYPE) != MAP_SHARED)
909                 return -EINVAL;
910 
911         if (!len)
912                 return -EINVAL;
913 
914         /* Careful about overflows.. */
915         rlen = PAGE_ALIGN(len);
916         if (!rlen || rlen > TASK_SIZE)
917                 return -ENOMEM;
918 
919         /* offset overflow? */
920         if ((pgoff + (rlen >> PAGE_SHIFT)) < pgoff)
921                 return -EOVERFLOW;
922 
923         if (file) {
924                 /* files must support mmap */
925                 if (!file->f_op->mmap)
926                         return -ENODEV;
927 
928                 /* work out if what we've got could possibly be shared
929                  * - we support chardevs that provide their own "memory"
930                  * - we support files/blockdevs that are memory backed
931                  */
932                 if (file->f_op->mmap_capabilities) {
933                         capabilities = file->f_op->mmap_capabilities(file);
934                 } else {
935                         /* no explicit capabilities set, so assume some
936                          * defaults */
937                         switch (file_inode(file)->i_mode & S_IFMT) {
938                         case S_IFREG:
939                         case S_IFBLK:
940                                 capabilities = NOMMU_MAP_COPY;
941                                 break;
942 
943                         case S_IFCHR:
944                                 capabilities =
945                                         NOMMU_MAP_DIRECT |
946                                         NOMMU_MAP_READ |
947                                         NOMMU_MAP_WRITE;
948                                 break;
949 
950                         default:
951                                 return -EINVAL;
952                         }
953                 }
954 
955                 /* eliminate any capabilities that we can't support on this
956                  * device */
957                 if (!file->f_op->get_unmapped_area)
958                         capabilities &= ~NOMMU_MAP_DIRECT;
959                 if (!(file->f_mode & FMODE_CAN_READ))
960                         capabilities &= ~NOMMU_MAP_COPY;
961 
962                 /* The file shall have been opened with read permission. */
963                 if (!(file->f_mode & FMODE_READ))
964                         return -EACCES;
965 
966                 if (flags & MAP_SHARED) {
967                         /* do checks for writing, appending and locking */
968                         if ((prot & PROT_WRITE) &&
969                             !(file->f_mode & FMODE_WRITE))
970                                 return -EACCES;
971 
972                         if (IS_APPEND(file_inode(file)) &&
973                             (file->f_mode & FMODE_WRITE))
974                                 return -EACCES;
975 
976                         if (locks_verify_locked(file))
977                                 return -EAGAIN;
978 
979                         if (!(capabilities & NOMMU_MAP_DIRECT))
980                                 return -ENODEV;
981 
982                         /* we mustn't privatise shared mappings */
983                         capabilities &= ~NOMMU_MAP_COPY;
984                 } else {
985                         /* we're going to read the file into private memory we
986                          * allocate */
987                         if (!(capabilities & NOMMU_MAP_COPY))
988                                 return -ENODEV;
989 
990                         /* we don't permit a private writable mapping to be
991                          * shared with the backing device */
992                         if (prot & PROT_WRITE)
993                                 capabilities &= ~NOMMU_MAP_DIRECT;
994                 }
995 
996                 if (capabilities & NOMMU_MAP_DIRECT) {
997                         if (((prot & PROT_READ)  && !(capabilities & NOMMU_MAP_READ))  ||
998                             ((prot & PROT_WRITE) && !(capabilities & NOMMU_MAP_WRITE)) ||
999                             ((prot & PROT_EXEC)  && !(capabilities & NOMMU_MAP_EXEC))
1000                             ) {
1001                                 capabilities &= ~NOMMU_MAP_DIRECT;
1002                                 if (flags & MAP_SHARED) {
1003                                         pr_warn("MAP_SHARED not completely supported on !MMU\n");
1004                                         return -EINVAL;
1005                                 }
1006                         }
1007                 }
1008 
1009                 /* handle executable mappings and implied executable
1010                  * mappings */
1011                 if (path_noexec(&file->f_path)) {
1012                         if (prot & PROT_EXEC)
1013                                 return -EPERM;
1014                 } else if ((prot & PROT_READ) && !(prot & PROT_EXEC)) {
1015                         /* handle implication of PROT_EXEC by PROT_READ */
1016                         if (current->personality & READ_IMPLIES_EXEC) {
1017                                 if (capabilities & NOMMU_MAP_EXEC)
1018                                         prot |= PROT_EXEC;
1019                         }
1020                 } else if ((prot & PROT_READ) &&
1021                          (prot & PROT_EXEC) &&
1022                          !(capabilities & NOMMU_MAP_EXEC)
1023                          ) {
1024                         /* backing file is not executable, try to copy */
1025                         capabilities &= ~NOMMU_MAP_DIRECT;
1026                 }
1027         } else {
1028                 /* anonymous mappings are always memory backed and can be
1029                  * privately mapped
1030                  */
1031                 capabilities = NOMMU_MAP_COPY;
1032 
1033                 /* handle PROT_EXEC implication by PROT_READ */
1034                 if ((prot & PROT_READ) &&
1035                     (current->personality & READ_IMPLIES_EXEC))
1036                         prot |= PROT_EXEC;
1037         }
1038 
1039         /* allow the security API to have its say */
1040         ret = security_mmap_addr(addr);
1041         if (ret < 0)
1042                 return ret;
1043 
1044         /* looks okay */
1045         *_capabilities = capabilities;
1046         return 0;
1047 }
1048 
1049 /*
1050  * we've determined that we can make the mapping, now translate what we
1051  * now know into VMA flags
1052  */
1053 static unsigned long determine_vm_flags(struct file *file,
1054                                         unsigned long prot,
1055                                         unsigned long flags,
1056                                         unsigned long capabilities)
1057 {
1058         unsigned long vm_flags;
1059 
1060         vm_flags = calc_vm_prot_bits(prot, 0) | calc_vm_flag_bits(flags);
1061         /* vm_flags |= mm->def_flags; */
1062 
1063         if (!(capabilities & NOMMU_MAP_DIRECT)) {
1064                 /* attempt to share read-only copies of mapped file chunks */
1065                 vm_flags |= VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
1066                 if (file && !(prot & PROT_WRITE))
1067                         vm_flags |= VM_MAYSHARE;
1068         } else {
1069                 /* overlay a shareable mapping on the backing device or inode
1070                  * if possible - used for chardevs, ramfs/tmpfs/shmfs and
1071                  * romfs/cramfs */
1072                 vm_flags |= VM_MAYSHARE | (capabilities & NOMMU_VMFLAGS);
1073                 if (flags & MAP_SHARED)
1074                         vm_flags |= VM_SHARED;
1075         }
1076 
1077         /* refuse to let anyone share private mappings with this process if
1078          * it's being traced - otherwise breakpoints set in it may interfere
1079          * with another untraced process
1080          */
1081         if ((flags & MAP_PRIVATE) && current->ptrace)
1082                 vm_flags &= ~VM_MAYSHARE;
1083 
1084         return vm_flags;
1085 }
1086 
1087 /*
1088  * set up a shared mapping on a file (the driver or filesystem provides and
1089  * pins the storage)
1090  */
1091 static int do_mmap_shared_file(struct vm_area_struct *vma)
1092 {
1093         int ret;
1094 
1095         ret = vma->vm_file->f_op->mmap(vma->vm_file, vma);
1096         if (ret == 0) {
1097                 vma->vm_region->vm_top = vma->vm_region->vm_end;
1098                 return 0;
1099         }
1100         if (ret != -ENOSYS)
1101                 return ret;
1102 
1103         /* getting -ENOSYS indicates that direct mmap isn't possible (as
1104          * opposed to tried but failed) so we can only give a suitable error as
1105          * it's not possible to make a private copy if MAP_SHARED was given */
1106         return -ENODEV;
1107 }
1108 
1109 /*
1110  * set up a private mapping or an anonymous shared mapping
1111  */
1112 static int do_mmap_private(struct vm_area_struct *vma,
1113                            struct vm_region *region,
1114                            unsigned long len,
1115                            unsigned long capabilities)
1116 {
1117         unsigned long total, point;
1118         void *base;
1119         int ret, order;
1120 
1121         /* invoke the file's mapping function so that it can keep track of
1122          * shared mappings on devices or memory
1123          * - VM_MAYSHARE will be set if it may attempt to share
1124          */
1125         if (capabilities & NOMMU_MAP_DIRECT) {
1126                 ret = vma->vm_file->f_op->mmap(vma->vm_file, vma);
1127                 if (ret == 0) {
1128                         /* shouldn't return success if we're not sharing */
1129                         BUG_ON(!(vma->vm_flags & VM_MAYSHARE));
1130                         vma->vm_region->vm_top = vma->vm_region->vm_end;
1131                         return 0;
1132                 }
1133                 if (ret != -ENOSYS)
1134                         return ret;
1135 
1136                 /* getting an ENOSYS error indicates that direct mmap isn't
1137                  * possible (as opposed to tried but failed) so we'll try to
1138                  * make a private copy of the data and map that instead */
1139         }
1140 
1141 
1142         /* allocate some memory to hold the mapping
1143          * - note that this may not return a page-aligned address if the object
1144          *   we're allocating is smaller than a page
1145          */
1146         order = get_order(len);
1147         total = 1 << order;
1148         point = len >> PAGE_SHIFT;
1149 
1150         /* we don't want to allocate a power-of-2 sized page set */
1151         if (sysctl_nr_trim_pages && total - point >= sysctl_nr_trim_pages)
1152                 total = point;
1153 
1154         base = alloc_pages_exact(total << PAGE_SHIFT, GFP_KERNEL);
1155         if (!base)
1156                 goto enomem;
1157 
1158         atomic_long_add(total, &mmap_pages_allocated);
1159 
1160         region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
1161         region->vm_start = (unsigned long) base;
1162         region->vm_end   = region->vm_start + len;
1163         region->vm_top   = region->vm_start + (total << PAGE_SHIFT);
1164 
1165         vma->vm_start = region->vm_start;
1166         vma->vm_end   = region->vm_start + len;
1167 
1168         if (vma->vm_file) {
1169                 /* read the contents of a file into the copy */
1170                 mm_segment_t old_fs;
1171                 loff_t fpos;
1172 
1173                 fpos = vma->vm_pgoff;
1174                 fpos <<= PAGE_SHIFT;
1175 
1176                 old_fs = get_fs();
1177                 set_fs(KERNEL_DS);
1178                 ret = __vfs_read(vma->vm_file, base, len, &fpos);
1179                 set_fs(old_fs);
1180 
1181                 if (ret < 0)
1182                         goto error_free;
1183 
1184                 /* clear the last little bit */
1185                 if (ret < len)
1186                         memset(base + ret, 0, len - ret);
1187 
1188         }
1189 
1190         return 0;
1191 
1192 error_free:
1193         free_page_series(region->vm_start, region->vm_top);
1194         region->vm_start = vma->vm_start = 0;
1195         region->vm_end   = vma->vm_end = 0;
1196         region->vm_top   = 0;
1197         return ret;
1198 
1199 enomem:
1200         pr_err("Allocation of length %lu from process %d (%s) failed\n",
1201                len, current->pid, current->comm);
1202         show_free_areas(0);
1203         return -ENOMEM;
1204 }
1205 
1206 /*
1207  * handle mapping creation for uClinux
1208  */
1209 unsigned long do_mmap(struct file *file,
1210                         unsigned long addr,
1211                         unsigned long len,
1212                         unsigned long prot,
1213                         unsigned long flags,
1214                         vm_flags_t vm_flags,
1215                         unsigned long pgoff,
1216                         unsigned long *populate)
1217 {
1218         struct vm_area_struct *vma;
1219         struct vm_region *region;
1220         struct rb_node *rb;
1221         unsigned long capabilities, result;
1222         int ret;
1223 
1224         *populate = 0;
1225 
1226         /* decide whether we should attempt the mapping, and if so what sort of
1227          * mapping */
1228         ret = validate_mmap_request(file, addr, len, prot, flags, pgoff,
1229                                     &capabilities);
1230         if (ret < 0)
1231                 return ret;
1232 
1233         /* we ignore the address hint */
1234         addr = 0;
1235         len = PAGE_ALIGN(len);
1236 
1237         /* we've determined that we can make the mapping, now translate what we
1238          * now know into VMA flags */
1239         vm_flags |= determine_vm_flags(file, prot, flags, capabilities);
1240 
1241         /* we're going to need to record the mapping */
1242         region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL);
1243         if (!region)
1244                 goto error_getting_region;
1245 
1246         vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
1247         if (!vma)
1248                 goto error_getting_vma;
1249 
1250         region->vm_usage = 1;
1251         region->vm_flags = vm_flags;
1252         region->vm_pgoff = pgoff;
1253 
1254         INIT_LIST_HEAD(&vma->anon_vma_chain);
1255         vma->vm_flags = vm_flags;
1256         vma->vm_pgoff = pgoff;
1257 
1258         if (file) {
1259                 region->vm_file = get_file(file);
1260                 vma->vm_file = get_file(file);
1261         }
1262 
1263         down_write(&nommu_region_sem);
1264 
1265         /* if we want to share, we need to check for regions created by other
1266          * mmap() calls that overlap with our proposed mapping
1267          * - we can only share with a superset match on most regular files
1268          * - shared mappings on character devices and memory backed files are
1269          *   permitted to overlap inexactly as far as we are concerned for in
1270          *   these cases, sharing is handled in the driver or filesystem rather
1271          *   than here
1272          */
1273         if (vm_flags & VM_MAYSHARE) {
1274                 struct vm_region *pregion;
1275                 unsigned long pglen, rpglen, pgend, rpgend, start;
1276 
1277                 pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1278                 pgend = pgoff + pglen;
1279 
1280                 for (rb = rb_first(&nommu_region_tree); rb; rb = rb_next(rb)) {
1281                         pregion = rb_entry(rb, struct vm_region, vm_rb);
1282 
1283                         if (!(pregion->vm_flags & VM_MAYSHARE))
1284                                 continue;
1285 
1286                         /* search for overlapping mappings on the same file */
1287                         if (file_inode(pregion->vm_file) !=
1288                             file_inode(file))
1289                                 continue;
1290 
1291                         if (pregion->vm_pgoff >= pgend)
1292                                 continue;
1293 
1294                         rpglen = pregion->vm_end - pregion->vm_start;
1295                         rpglen = (rpglen + PAGE_SIZE - 1) >> PAGE_SHIFT;
1296                         rpgend = pregion->vm_pgoff + rpglen;
1297                         if (pgoff >= rpgend)
1298                                 continue;
1299 
1300                         /* handle inexactly overlapping matches between
1301                          * mappings */
1302                         if ((pregion->vm_pgoff != pgoff || rpglen != pglen) &&
1303                             !(pgoff >= pregion->vm_pgoff && pgend <= rpgend)) {
1304                                 /* new mapping is not a subset of the region */
1305                                 if (!(capabilities & NOMMU_MAP_DIRECT))
1306                                         goto sharing_violation;
1307                                 continue;
1308                         }
1309 
1310                         /* we've found a region we can share */
1311                         pregion->vm_usage++;
1312                         vma->vm_region = pregion;
1313                         start = pregion->vm_start;
1314                         start += (pgoff - pregion->vm_pgoff) << PAGE_SHIFT;
1315                         vma->vm_start = start;
1316                         vma->vm_end = start + len;
1317 
1318                         if (pregion->vm_flags & VM_MAPPED_COPY)
1319                                 vma->vm_flags |= VM_MAPPED_COPY;
1320                         else {
1321                                 ret = do_mmap_shared_file(vma);
1322                                 if (ret < 0) {
1323                                         vma->vm_region = NULL;
1324                                         vma->vm_start = 0;
1325                                         vma->vm_end = 0;
1326                                         pregion->vm_usage--;
1327                                         pregion = NULL;
1328                                         goto error_just_free;
1329                                 }
1330                         }
1331                         fput(region->vm_file);
1332                         kmem_cache_free(vm_region_jar, region);
1333                         region = pregion;
1334                         result = start;
1335                         goto share;
1336                 }
1337 
1338                 /* obtain the address at which to make a shared mapping
1339                  * - this is the hook for quasi-memory character devices to
1340                  *   tell us the location of a shared mapping
1341                  */
1342                 if (capabilities & NOMMU_MAP_DIRECT) {
1343                         addr = file->f_op->get_unmapped_area(file, addr, len,
1344                                                              pgoff, flags);
1345                         if (IS_ERR_VALUE(addr)) {
1346                                 ret = addr;
1347                                 if (ret != -ENOSYS)
1348                                         goto error_just_free;
1349 
1350                                 /* the driver refused to tell us where to site
1351                                  * the mapping so we'll have to attempt to copy
1352                                  * it */
1353                                 ret = -ENODEV;
1354                                 if (!(capabilities & NOMMU_MAP_COPY))
1355                                         goto error_just_free;
1356 
1357                                 capabilities &= ~NOMMU_MAP_DIRECT;
1358                         } else {
1359                                 vma->vm_start = region->vm_start = addr;
1360                                 vma->vm_end = region->vm_end = addr + len;
1361                         }
1362                 }
1363         }
1364 
1365         vma->vm_region = region;
1366 
1367         /* set up the mapping
1368          * - the region is filled in if NOMMU_MAP_DIRECT is still set
1369          */
1370         if (file && vma->vm_flags & VM_SHARED)
1371                 ret = do_mmap_shared_file(vma);
1372         else
1373                 ret = do_mmap_private(vma, region, len, capabilities);
1374         if (ret < 0)
1375                 goto error_just_free;
1376         add_nommu_region(region);
1377 
1378         /* clear anonymous mappings that don't ask for uninitialized data */
1379         if (!vma->vm_file && !(flags & MAP_UNINITIALIZED))
1380                 memset((void *)region->vm_start, 0,
1381                        region->vm_end - region->vm_start);
1382 
1383         /* okay... we have a mapping; now we have to register it */
1384         result = vma->vm_start;
1385 
1386         current->mm->total_vm += len >> PAGE_SHIFT;
1387 
1388 share:
1389         add_vma_to_mm(current->mm, vma);
1390 
1391         /* we flush the region from the icache only when the first executable
1392          * mapping of it is made  */
1393         if (vma->vm_flags & VM_EXEC && !region->vm_icache_flushed) {
1394                 flush_icache_range(region->vm_start, region->vm_end);
1395                 region->vm_icache_flushed = true;
1396         }
1397 
1398         up_write(&nommu_region_sem);
1399 
1400         return result;
1401 
1402 error_just_free:
1403         up_write(&nommu_region_sem);
1404 error:
1405         if (region->vm_file)
1406                 fput(region->vm_file);
1407         kmem_cache_free(vm_region_jar, region);
1408         if (vma->vm_file)
1409                 fput(vma->vm_file);
1410         kmem_cache_free(vm_area_cachep, vma);
1411         return ret;
1412 
1413 sharing_violation:
1414         up_write(&nommu_region_sem);
1415         pr_warn("Attempt to share mismatched mappings\n");
1416         ret = -EINVAL;
1417         goto error;
1418 
1419 error_getting_vma:
1420         kmem_cache_free(vm_region_jar, region);
1421         pr_warn("Allocation of vma for %lu byte allocation from process %d failed\n",
1422                         len, current->pid);
1423         show_free_areas(0);
1424         return -ENOMEM;
1425 
1426 error_getting_region:
1427         pr_warn("Allocation of vm region for %lu byte allocation from process %d failed\n",
1428                         len, current->pid);
1429         show_free_areas(0);
1430         return -ENOMEM;
1431 }
1432 
1433 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
1434                 unsigned long, prot, unsigned long, flags,
1435                 unsigned long, fd, unsigned long, pgoff)
1436 {
1437         struct file *file = NULL;
1438         unsigned long retval = -EBADF;
1439 
1440         audit_mmap_fd(fd, flags);
1441         if (!(flags & MAP_ANONYMOUS)) {
1442                 file = fget(fd);
1443                 if (!file)
1444                         goto out;
1445         }
1446 
1447         flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
1448 
1449         retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
1450 
1451         if (file)
1452                 fput(file);
1453 out:
1454         return retval;
1455 }
1456 
1457 #ifdef __ARCH_WANT_SYS_OLD_MMAP
1458 struct mmap_arg_struct {
1459         unsigned long addr;
1460         unsigned long len;
1461         unsigned long prot;
1462         unsigned long flags;
1463         unsigned long fd;
1464         unsigned long offset;
1465 };
1466 
1467 SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
1468 {
1469         struct mmap_arg_struct a;
1470 
1471         if (copy_from_user(&a, arg, sizeof(a)))
1472                 return -EFAULT;
1473         if (offset_in_page(a.offset))
1474                 return -EINVAL;
1475 
1476         return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
1477                               a.offset >> PAGE_SHIFT);
1478 }
1479 #endif /* __ARCH_WANT_SYS_OLD_MMAP */
1480 
1481 /*
1482  * split a vma into two pieces at address 'addr', a new vma is allocated either
1483  * for the first part or the tail.
1484  */
1485 int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
1486               unsigned long addr, int new_below)
1487 {
1488         struct vm_area_struct *new;
1489         struct vm_region *region;
1490         unsigned long npages;
1491 
1492         /* we're only permitted to split anonymous regions (these should have
1493          * only a single usage on the region) */
1494         if (vma->vm_file)
1495                 return -ENOMEM;
1496 
1497         if (mm->map_count >= sysctl_max_map_count)
1498                 return -ENOMEM;
1499 
1500         region = kmem_cache_alloc(vm_region_jar, GFP_KERNEL);
1501         if (!region)
1502                 return -ENOMEM;
1503 
1504         new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
1505         if (!new) {
1506                 kmem_cache_free(vm_region_jar, region);
1507                 return -ENOMEM;
1508         }
1509 
1510         /* most fields are the same, copy all, and then fixup */
1511         *new = *vma;
1512         *region = *vma->vm_region;
1513         new->vm_region = region;
1514 
1515         npages = (addr - vma->vm_start) >> PAGE_SHIFT;
1516 
1517         if (new_below) {
1518                 region->vm_top = region->vm_end = new->vm_end = addr;
1519         } else {
1520                 region->vm_start = new->vm_start = addr;
1521                 region->vm_pgoff = new->vm_pgoff += npages;
1522         }
1523 
1524         if (new->vm_ops && new->vm_ops->open)
1525                 new->vm_ops->open(new);
1526 
1527         delete_vma_from_mm(vma);
1528         down_write(&nommu_region_sem);
1529         delete_nommu_region(vma->vm_region);
1530         if (new_below) {
1531                 vma->vm_region->vm_start = vma->vm_start = addr;
1532                 vma->vm_region->vm_pgoff = vma->vm_pgoff += npages;
1533         } else {
1534                 vma->vm_region->vm_end = vma->vm_end = addr;
1535                 vma->vm_region->vm_top = addr;
1536         }
1537         add_nommu_region(vma->vm_region);
1538         add_nommu_region(new->vm_region);
1539         up_write(&nommu_region_sem);
1540         add_vma_to_mm(mm, vma);
1541         add_vma_to_mm(mm, new);
1542         return 0;
1543 }
1544 
1545 /*
1546  * shrink a VMA by removing the specified chunk from either the beginning or
1547  * the end
1548  */
1549 static int shrink_vma(struct mm_struct *mm,
1550                       struct vm_area_struct *vma,
1551                       unsigned long from, unsigned long to)
1552 {
1553         struct vm_region *region;
1554 
1555         /* adjust the VMA's pointers, which may reposition it in the MM's tree
1556          * and list */
1557         delete_vma_from_mm(vma);
1558         if (from > vma->vm_start)
1559                 vma->vm_end = from;
1560         else
1561                 vma->vm_start = to;
1562         add_vma_to_mm(mm, vma);
1563 
1564         /* cut the backing region down to size */
1565         region = vma->vm_region;
1566         BUG_ON(region->vm_usage != 1);
1567 
1568         down_write(&nommu_region_sem);
1569         delete_nommu_region(region);
1570         if (from > region->vm_start) {
1571                 to = region->vm_top;
1572                 region->vm_top = region->vm_end = from;
1573         } else {
1574                 region->vm_start = to;
1575         }
1576         add_nommu_region(region);
1577         up_write(&nommu_region_sem);
1578 
1579         free_page_series(from, to);
1580         return 0;
1581 }
1582 
1583 /*
1584  * release a mapping
1585  * - under NOMMU conditions the chunk to be unmapped must be backed by a single
1586  *   VMA, though it need not cover the whole VMA
1587  */
1588 int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
1589 {
1590         struct vm_area_struct *vma;
1591         unsigned long end;
1592         int ret;
1593 
1594         len = PAGE_ALIGN(len);
1595         if (len == 0)
1596                 return -EINVAL;
1597 
1598         end = start + len;
1599 
1600         /* find the first potentially overlapping VMA */
1601         vma = find_vma(mm, start);
1602         if (!vma) {
1603                 static int limit;
1604                 if (limit < 5) {
1605                         pr_warn("munmap of memory not mmapped by process %d (%s): 0x%lx-0x%lx\n",
1606                                         current->pid, current->comm,
1607                                         start, start + len - 1);
1608                         limit++;
1609                 }
1610                 return -EINVAL;
1611         }
1612 
1613         /* we're allowed to split an anonymous VMA but not a file-backed one */
1614         if (vma->vm_file) {
1615                 do {
1616                         if (start > vma->vm_start)
1617                                 return -EINVAL;
1618                         if (end == vma->vm_end)
1619                                 goto erase_whole_vma;
1620                         vma = vma->vm_next;
1621                 } while (vma);
1622                 return -EINVAL;
1623         } else {
1624                 /* the chunk must be a subset of the VMA found */
1625                 if (start == vma->vm_start && end == vma->vm_end)
1626                         goto erase_whole_vma;
1627                 if (start < vma->vm_start || end > vma->vm_end)
1628                         return -EINVAL;
1629                 if (offset_in_page(start))
1630                         return -EINVAL;
1631                 if (end != vma->vm_end && offset_in_page(end))
1632                         return -EINVAL;
1633                 if (start != vma->vm_start && end != vma->vm_end) {
1634                         ret = split_vma(mm, vma, start, 1);
1635                         if (ret < 0)
1636                                 return ret;
1637                 }
1638                 return shrink_vma(mm, vma, start, end);
1639         }
1640 
1641 erase_whole_vma:
1642         delete_vma_from_mm(vma);
1643         delete_vma(mm, vma);
1644         return 0;
1645 }
1646 EXPORT_SYMBOL(do_munmap);
1647 
1648 int vm_munmap(unsigned long addr, size_t len)
1649 {
1650         struct mm_struct *mm = current->mm;
1651         int ret;
1652 
1653         down_write(&mm->mmap_sem);
1654         ret = do_munmap(mm, addr, len);
1655         up_write(&mm->mmap_sem);
1656         return ret;
1657 }
1658 EXPORT_SYMBOL(vm_munmap);
1659 
1660 SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
1661 {
1662         return vm_munmap(addr, len);
1663 }
1664 
1665 /*
1666  * release all the mappings made in a process's VM space
1667  */
1668 void exit_mmap(struct mm_struct *mm)
1669 {
1670         struct vm_area_struct *vma;
1671 
1672         if (!mm)
1673                 return;
1674 
1675         mm->total_vm = 0;
1676 
1677         while ((vma = mm->mmap)) {
1678                 mm->mmap = vma->vm_next;
1679                 delete_vma_from_mm(vma);
1680                 delete_vma(mm, vma);
1681                 cond_resched();
1682         }
1683 }
1684 
1685 int vm_brk(unsigned long addr, unsigned long len)
1686 {
1687         return -ENOMEM;
1688 }
1689 
1690 /*
1691  * expand (or shrink) an existing mapping, potentially moving it at the same
1692  * time (controlled by the MREMAP_MAYMOVE flag and available VM space)
1693  *
1694  * under NOMMU conditions, we only permit changing a mapping's size, and only
1695  * as long as it stays within the region allocated by do_mmap_private() and the
1696  * block is not shareable
1697  *
1698  * MREMAP_FIXED is not supported under NOMMU conditions
1699  */
1700 static unsigned long do_mremap(unsigned long addr,
1701                         unsigned long old_len, unsigned long new_len,
1702                         unsigned long flags, unsigned long new_addr)
1703 {
1704         struct vm_area_struct *vma;
1705 
1706         /* insanity checks first */
1707         old_len = PAGE_ALIGN(old_len);
1708         new_len = PAGE_ALIGN(new_len);
1709         if (old_len == 0 || new_len == 0)
1710                 return (unsigned long) -EINVAL;
1711 
1712         if (offset_in_page(addr))
1713                 return -EINVAL;
1714 
1715         if (flags & MREMAP_FIXED && new_addr != addr)
1716                 return (unsigned long) -EINVAL;
1717 
1718         vma = find_vma_exact(current->mm, addr, old_len);
1719         if (!vma)
1720                 return (unsigned long) -EINVAL;
1721 
1722         if (vma->vm_end != vma->vm_start + old_len)
1723                 return (unsigned long) -EFAULT;
1724 
1725         if (vma->vm_flags & VM_MAYSHARE)
1726                 return (unsigned long) -EPERM;
1727 
1728         if (new_len > vma->vm_region->vm_end - vma->vm_region->vm_start)
1729                 return (unsigned long) -ENOMEM;
1730 
1731         /* all checks complete - do it */
1732         vma->vm_end = vma->vm_start + new_len;
1733         return vma->vm_start;
1734 }
1735 
1736 SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
1737                 unsigned long, new_len, unsigned long, flags,
1738                 unsigned long, new_addr)
1739 {
1740         unsigned long ret;
1741 
1742         down_write(&current->mm->mmap_sem);
1743         ret = do_mremap(addr, old_len, new_len, flags, new_addr);
1744         up_write(&current->mm->mmap_sem);
1745         return ret;
1746 }
1747 
1748 struct page *follow_page_mask(struct vm_area_struct *vma,
1749                               unsigned long address, unsigned int flags,
1750                               unsigned int *page_mask)
1751 {
1752         *page_mask = 0;
1753         return NULL;
1754 }
1755 
1756 int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
1757                 unsigned long pfn, unsigned long size, pgprot_t prot)
1758 {
1759         if (addr != (pfn << PAGE_SHIFT))
1760                 return -EINVAL;
1761 
1762         vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
1763         return 0;
1764 }
1765 EXPORT_SYMBOL(remap_pfn_range);
1766 
1767 int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
1768 {
1769         unsigned long pfn = start >> PAGE_SHIFT;
1770         unsigned long vm_len = vma->vm_end - vma->vm_start;
1771 
1772         pfn += vma->vm_pgoff;
1773         return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
1774 }
1775 EXPORT_SYMBOL(vm_iomap_memory);
1776 
1777 int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
1778                         unsigned long pgoff)
1779 {
1780         unsigned int size = vma->vm_end - vma->vm_start;
1781 
1782         if (!(vma->vm_flags & VM_USERMAP))
1783                 return -EINVAL;
1784 
1785         vma->vm_start = (unsigned long)(addr + (pgoff << PAGE_SHIFT));
1786         vma->vm_end = vma->vm_start + size;
1787 
1788         return 0;
1789 }
1790 EXPORT_SYMBOL(remap_vmalloc_range);
1791 
1792 unsigned long arch_get_unmapped_area(struct file *file, unsigned long addr,
1793         unsigned long len, unsigned long pgoff, unsigned long flags)
1794 {
1795         return -ENOMEM;
1796 }
1797 
1798 void unmap_mapping_range(struct address_space *mapping,
1799                          loff_t const holebegin, loff_t const holelen,
1800                          int even_cows)
1801 {
1802 }
1803 EXPORT_SYMBOL(unmap_mapping_range);
1804 
1805 int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1806 {
1807         BUG();
1808         return 0;
1809 }
1810 EXPORT_SYMBOL(filemap_fault);
1811 
1812 void filemap_map_pages(struct vm_area_struct *vma, struct vm_fault *vmf)
1813 {
1814         BUG();
1815 }
1816 EXPORT_SYMBOL(filemap_map_pages);
1817 
1818 static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
1819                 unsigned long addr, void *buf, int len, int write)
1820 {
1821         struct vm_area_struct *vma;
1822 
1823         down_read(&mm->mmap_sem);
1824 
1825         /* the access must start within one of the target process's mappings */
1826         vma = find_vma(mm, addr);
1827         if (vma) {
1828                 /* don't overrun this mapping */
1829                 if (addr + len >= vma->vm_end)
1830                         len = vma->vm_end - addr;
1831 
1832                 /* only read or write mappings where it is permitted */
1833                 if (write && vma->vm_flags & VM_MAYWRITE)
1834                         copy_to_user_page(vma, NULL, addr,
1835                                          (void *) addr, buf, len);
1836                 else if (!write && vma->vm_flags & VM_MAYREAD)
1837                         copy_from_user_page(vma, NULL, addr,
1838                                             buf, (void *) addr, len);
1839                 else
1840                         len = 0;
1841         } else {
1842                 len = 0;
1843         }
1844 
1845         up_read(&mm->mmap_sem);
1846 
1847         return len;
1848 }
1849 
1850 /**
1851  * @access_remote_vm - access another process' address space
1852  * @mm:         the mm_struct of the target address space
1853  * @addr:       start address to access
1854  * @buf:        source or destination buffer
1855  * @len:        number of bytes to transfer
1856  * @write:      whether the access is a write
1857  *
1858  * The caller must hold a reference on @mm.
1859  */
1860 int access_remote_vm(struct mm_struct *mm, unsigned long addr,
1861                 void *buf, int len, int write)
1862 {
1863         return __access_remote_vm(NULL, mm, addr, buf, len, write);
1864 }
1865 
1866 /*
1867  * Access another process' address space.
1868  * - source/target buffer must be kernel space
1869  */
1870 int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
1871 {
1872         struct mm_struct *mm;
1873 
1874         if (addr + len < addr)
1875                 return 0;
1876 
1877         mm = get_task_mm(tsk);
1878         if (!mm)
1879                 return 0;
1880 
1881         len = __access_remote_vm(tsk, mm, addr, buf, len, write);
1882 
1883         mmput(mm);
1884         return len;
1885 }
1886 
1887 /**
1888  * nommu_shrink_inode_mappings - Shrink the shared mappings on an inode
1889  * @inode: The inode to check
1890  * @size: The current filesize of the inode
1891  * @newsize: The proposed filesize of the inode
1892  *
1893  * Check the shared mappings on an inode on behalf of a shrinking truncate to
1894  * make sure that that any outstanding VMAs aren't broken and then shrink the
1895  * vm_regions that extend that beyond so that do_mmap_pgoff() doesn't
1896  * automatically grant mappings that are too large.
1897  */
1898 int nommu_shrink_inode_mappings(struct inode *inode, size_t size,
1899                                 size_t newsize)
1900 {
1901         struct vm_area_struct *vma;
1902         struct vm_region *region;
1903         pgoff_t low, high;
1904         size_t r_size, r_top;
1905 
1906         low = newsize >> PAGE_SHIFT;
1907         high = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1908 
1909         down_write(&nommu_region_sem);
1910         i_mmap_lock_read(inode->i_mapping);
1911 
1912         /* search for VMAs that fall within the dead zone */
1913         vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, low, high) {
1914                 /* found one - only interested if it's shared out of the page
1915                  * cache */
1916                 if (vma->vm_flags & VM_SHARED) {
1917                         i_mmap_unlock_read(inode->i_mapping);
1918                         up_write(&nommu_region_sem);
1919                         return -ETXTBSY; /* not quite true, but near enough */
1920                 }
1921         }
1922 
1923         /* reduce any regions that overlap the dead zone - if in existence,
1924          * these will be pointed to by VMAs that don't overlap the dead zone
1925          *
1926          * we don't check for any regions that start beyond the EOF as there
1927          * shouldn't be any
1928          */
1929         vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, 0, ULONG_MAX) {
1930                 if (!(vma->vm_flags & VM_SHARED))
1931                         continue;
1932 
1933                 region = vma->vm_region;
1934                 r_size = region->vm_top - region->vm_start;
1935                 r_top = (region->vm_pgoff << PAGE_SHIFT) + r_size;
1936 
1937                 if (r_top > newsize) {
1938                         region->vm_top -= r_top - newsize;
1939                         if (region->vm_end > region->vm_top)
1940                                 region->vm_end = region->vm_top;
1941                 }
1942         }
1943 
1944         i_mmap_unlock_read(inode->i_mapping);
1945         up_write(&nommu_region_sem);
1946         return 0;
1947 }
1948 
1949 /*
1950  * Initialise sysctl_user_reserve_kbytes.
1951  *
1952  * This is intended to prevent a user from starting a single memory hogging
1953  * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
1954  * mode.
1955  *
1956  * The default value is min(3% of free memory, 128MB)
1957  * 128MB is enough to recover with sshd/login, bash, and top/kill.
1958  */
1959 static int __meminit init_user_reserve(void)
1960 {
1961         unsigned long free_kbytes;
1962 
1963         free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
1964 
1965         sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17);
1966         return 0;
1967 }
1968 subsys_initcall(init_user_reserve);
1969 
1970 /*
1971  * Initialise sysctl_admin_reserve_kbytes.
1972  *
1973  * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
1974  * to log in and kill a memory hogging process.
1975  *
1976  * Systems with more than 256MB will reserve 8MB, enough to recover
1977  * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
1978  * only reserve 3% of free pages by default.
1979  */
1980 static int __meminit init_admin_reserve(void)
1981 {
1982         unsigned long free_kbytes;
1983 
1984         free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
1985 
1986         sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13);
1987         return 0;
1988 }
1989 subsys_initcall(init_admin_reserve);
1990 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp