~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/hugetlb.c

Version: ~ [ linux-5.5 ] ~ [ linux-5.4.15 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.98 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.167 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.211 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.211 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.81 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Generic hugetlb support.
  3  * (C) William Irwin, April 2004
  4  */
  5 #include <linux/list.h>
  6 #include <linux/init.h>
  7 #include <linux/module.h>
  8 #include <linux/mm.h>
  9 #include <linux/seq_file.h>
 10 #include <linux/sysctl.h>
 11 #include <linux/highmem.h>
 12 #include <linux/mmu_notifier.h>
 13 #include <linux/nodemask.h>
 14 #include <linux/pagemap.h>
 15 #include <linux/mempolicy.h>
 16 #include <linux/cpuset.h>
 17 #include <linux/mutex.h>
 18 #include <linux/bootmem.h>
 19 #include <linux/sysfs.h>
 20 #include <linux/slab.h>
 21 #include <linux/rmap.h>
 22 #include <linux/swap.h>
 23 #include <linux/swapops.h>
 24 
 25 #include <asm/page.h>
 26 #include <asm/pgtable.h>
 27 #include <linux/io.h>
 28 
 29 #include <linux/hugetlb.h>
 30 #include <linux/node.h>
 31 #include "internal.h"
 32 
 33 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
 34 static gfp_t htlb_alloc_mask = GFP_HIGHUSER;
 35 unsigned long hugepages_treat_as_movable;
 36 
 37 static int max_hstate;
 38 unsigned int default_hstate_idx;
 39 struct hstate hstates[HUGE_MAX_HSTATE];
 40 
 41 __initdata LIST_HEAD(huge_boot_pages);
 42 
 43 /* for command line parsing */
 44 static struct hstate * __initdata parsed_hstate;
 45 static unsigned long __initdata default_hstate_max_huge_pages;
 46 static unsigned long __initdata default_hstate_size;
 47 
 48 #define for_each_hstate(h) \
 49         for ((h) = hstates; (h) < &hstates[max_hstate]; (h)++)
 50 
 51 /*
 52  * Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
 53  */
 54 static DEFINE_SPINLOCK(hugetlb_lock);
 55 
 56 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
 57 {
 58         bool free = (spool->count == 0) && (spool->used_hpages == 0);
 59 
 60         spin_unlock(&spool->lock);
 61 
 62         /* If no pages are used, and no other handles to the subpool
 63          * remain, free the subpool the subpool remain */
 64         if (free)
 65                 kfree(spool);
 66 }
 67 
 68 struct hugepage_subpool *hugepage_new_subpool(long nr_blocks)
 69 {
 70         struct hugepage_subpool *spool;
 71 
 72         spool = kmalloc(sizeof(*spool), GFP_KERNEL);
 73         if (!spool)
 74                 return NULL;
 75 
 76         spin_lock_init(&spool->lock);
 77         spool->count = 1;
 78         spool->max_hpages = nr_blocks;
 79         spool->used_hpages = 0;
 80 
 81         return spool;
 82 }
 83 
 84 void hugepage_put_subpool(struct hugepage_subpool *spool)
 85 {
 86         spin_lock(&spool->lock);
 87         BUG_ON(!spool->count);
 88         spool->count--;
 89         unlock_or_release_subpool(spool);
 90 }
 91 
 92 static int hugepage_subpool_get_pages(struct hugepage_subpool *spool,
 93                                       long delta)
 94 {
 95         int ret = 0;
 96 
 97         if (!spool)
 98                 return 0;
 99 
100         spin_lock(&spool->lock);
101         if ((spool->used_hpages + delta) <= spool->max_hpages) {
102                 spool->used_hpages += delta;
103         } else {
104                 ret = -ENOMEM;
105         }
106         spin_unlock(&spool->lock);
107 
108         return ret;
109 }
110 
111 static void hugepage_subpool_put_pages(struct hugepage_subpool *spool,
112                                        long delta)
113 {
114         if (!spool)
115                 return;
116 
117         spin_lock(&spool->lock);
118         spool->used_hpages -= delta;
119         /* If hugetlbfs_put_super couldn't free spool due to
120         * an outstanding quota reference, free it now. */
121         unlock_or_release_subpool(spool);
122 }
123 
124 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
125 {
126         return HUGETLBFS_SB(inode->i_sb)->spool;
127 }
128 
129 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
130 {
131         return subpool_inode(vma->vm_file->f_dentry->d_inode);
132 }
133 
134 /*
135  * Region tracking -- allows tracking of reservations and instantiated pages
136  *                    across the pages in a mapping.
137  *
138  * The region data structures are protected by a combination of the mmap_sem
139  * and the hugetlb_instantion_mutex.  To access or modify a region the caller
140  * must either hold the mmap_sem for write, or the mmap_sem for read and
141  * the hugetlb_instantiation mutex:
142  *
143  *      down_write(&mm->mmap_sem);
144  * or
145  *      down_read(&mm->mmap_sem);
146  *      mutex_lock(&hugetlb_instantiation_mutex);
147  */
148 struct file_region {
149         struct list_head link;
150         long from;
151         long to;
152 };
153 
154 static long region_add(struct list_head *head, long f, long t)
155 {
156         struct file_region *rg, *nrg, *trg;
157 
158         /* Locate the region we are either in or before. */
159         list_for_each_entry(rg, head, link)
160                 if (f <= rg->to)
161                         break;
162 
163         /* Round our left edge to the current segment if it encloses us. */
164         if (f > rg->from)
165                 f = rg->from;
166 
167         /* Check for and consume any regions we now overlap with. */
168         nrg = rg;
169         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
170                 if (&rg->link == head)
171                         break;
172                 if (rg->from > t)
173                         break;
174 
175                 /* If this area reaches higher then extend our area to
176                  * include it completely.  If this is not the first area
177                  * which we intend to reuse, free it. */
178                 if (rg->to > t)
179                         t = rg->to;
180                 if (rg != nrg) {
181                         list_del(&rg->link);
182                         kfree(rg);
183                 }
184         }
185         nrg->from = f;
186         nrg->to = t;
187         return 0;
188 }
189 
190 static long region_chg(struct list_head *head, long f, long t)
191 {
192         struct file_region *rg, *nrg;
193         long chg = 0;
194 
195         /* Locate the region we are before or in. */
196         list_for_each_entry(rg, head, link)
197                 if (f <= rg->to)
198                         break;
199 
200         /* If we are below the current region then a new region is required.
201          * Subtle, allocate a new region at the position but make it zero
202          * size such that we can guarantee to record the reservation. */
203         if (&rg->link == head || t < rg->from) {
204                 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
205                 if (!nrg)
206                         return -ENOMEM;
207                 nrg->from = f;
208                 nrg->to   = f;
209                 INIT_LIST_HEAD(&nrg->link);
210                 list_add(&nrg->link, rg->link.prev);
211 
212                 return t - f;
213         }
214 
215         /* Round our left edge to the current segment if it encloses us. */
216         if (f > rg->from)
217                 f = rg->from;
218         chg = t - f;
219 
220         /* Check for and consume any regions we now overlap with. */
221         list_for_each_entry(rg, rg->link.prev, link) {
222                 if (&rg->link == head)
223                         break;
224                 if (rg->from > t)
225                         return chg;
226 
227                 /* We overlap with this area, if it extends further than
228                  * us then we must extend ourselves.  Account for its
229                  * existing reservation. */
230                 if (rg->to > t) {
231                         chg += rg->to - t;
232                         t = rg->to;
233                 }
234                 chg -= rg->to - rg->from;
235         }
236         return chg;
237 }
238 
239 static long region_truncate(struct list_head *head, long end)
240 {
241         struct file_region *rg, *trg;
242         long chg = 0;
243 
244         /* Locate the region we are either in or before. */
245         list_for_each_entry(rg, head, link)
246                 if (end <= rg->to)
247                         break;
248         if (&rg->link == head)
249                 return 0;
250 
251         /* If we are in the middle of a region then adjust it. */
252         if (end > rg->from) {
253                 chg = rg->to - end;
254                 rg->to = end;
255                 rg = list_entry(rg->link.next, typeof(*rg), link);
256         }
257 
258         /* Drop any remaining regions. */
259         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
260                 if (&rg->link == head)
261                         break;
262                 chg += rg->to - rg->from;
263                 list_del(&rg->link);
264                 kfree(rg);
265         }
266         return chg;
267 }
268 
269 static long region_count(struct list_head *head, long f, long t)
270 {
271         struct file_region *rg;
272         long chg = 0;
273 
274         /* Locate each segment we overlap with, and count that overlap. */
275         list_for_each_entry(rg, head, link) {
276                 long seg_from;
277                 long seg_to;
278 
279                 if (rg->to <= f)
280                         continue;
281                 if (rg->from >= t)
282                         break;
283 
284                 seg_from = max(rg->from, f);
285                 seg_to = min(rg->to, t);
286 
287                 chg += seg_to - seg_from;
288         }
289 
290         return chg;
291 }
292 
293 /*
294  * Convert the address within this vma to the page offset within
295  * the mapping, in pagecache page units; huge pages here.
296  */
297 static pgoff_t vma_hugecache_offset(struct hstate *h,
298                         struct vm_area_struct *vma, unsigned long address)
299 {
300         return ((address - vma->vm_start) >> huge_page_shift(h)) +
301                         (vma->vm_pgoff >> huge_page_order(h));
302 }
303 
304 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
305                                      unsigned long address)
306 {
307         return vma_hugecache_offset(hstate_vma(vma), vma, address);
308 }
309 
310 /*
311  * Return the size of the pages allocated when backing a VMA. In the majority
312  * cases this will be same size as used by the page table entries.
313  */
314 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
315 {
316         struct hstate *hstate;
317 
318         if (!is_vm_hugetlb_page(vma))
319                 return PAGE_SIZE;
320 
321         hstate = hstate_vma(vma);
322 
323         return 1UL << (hstate->order + PAGE_SHIFT);
324 }
325 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
326 
327 /*
328  * Return the page size being used by the MMU to back a VMA. In the majority
329  * of cases, the page size used by the kernel matches the MMU size. On
330  * architectures where it differs, an architecture-specific version of this
331  * function is required.
332  */
333 #ifndef vma_mmu_pagesize
334 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
335 {
336         return vma_kernel_pagesize(vma);
337 }
338 #endif
339 
340 /*
341  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
342  * bits of the reservation map pointer, which are always clear due to
343  * alignment.
344  */
345 #define HPAGE_RESV_OWNER    (1UL << 0)
346 #define HPAGE_RESV_UNMAPPED (1UL << 1)
347 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
348 
349 /*
350  * These helpers are used to track how many pages are reserved for
351  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
352  * is guaranteed to have their future faults succeed.
353  *
354  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
355  * the reserve counters are updated with the hugetlb_lock held. It is safe
356  * to reset the VMA at fork() time as it is not in use yet and there is no
357  * chance of the global counters getting corrupted as a result of the values.
358  *
359  * The private mapping reservation is represented in a subtly different
360  * manner to a shared mapping.  A shared mapping has a region map associated
361  * with the underlying file, this region map represents the backing file
362  * pages which have ever had a reservation assigned which this persists even
363  * after the page is instantiated.  A private mapping has a region map
364  * associated with the original mmap which is attached to all VMAs which
365  * reference it, this region map represents those offsets which have consumed
366  * reservation ie. where pages have been instantiated.
367  */
368 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
369 {
370         return (unsigned long)vma->vm_private_data;
371 }
372 
373 static void set_vma_private_data(struct vm_area_struct *vma,
374                                                         unsigned long value)
375 {
376         vma->vm_private_data = (void *)value;
377 }
378 
379 struct resv_map {
380         struct kref refs;
381         struct list_head regions;
382 };
383 
384 static struct resv_map *resv_map_alloc(void)
385 {
386         struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
387         if (!resv_map)
388                 return NULL;
389 
390         kref_init(&resv_map->refs);
391         INIT_LIST_HEAD(&resv_map->regions);
392 
393         return resv_map;
394 }
395 
396 static void resv_map_release(struct kref *ref)
397 {
398         struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
399 
400         /* Clear out any active regions before we release the map. */
401         region_truncate(&resv_map->regions, 0);
402         kfree(resv_map);
403 }
404 
405 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
406 {
407         VM_BUG_ON(!is_vm_hugetlb_page(vma));
408         if (!(vma->vm_flags & VM_MAYSHARE))
409                 return (struct resv_map *)(get_vma_private_data(vma) &
410                                                         ~HPAGE_RESV_MASK);
411         return NULL;
412 }
413 
414 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
415 {
416         VM_BUG_ON(!is_vm_hugetlb_page(vma));
417         VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
418 
419         set_vma_private_data(vma, (get_vma_private_data(vma) &
420                                 HPAGE_RESV_MASK) | (unsigned long)map);
421 }
422 
423 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
424 {
425         VM_BUG_ON(!is_vm_hugetlb_page(vma));
426         VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
427 
428         set_vma_private_data(vma, get_vma_private_data(vma) | flags);
429 }
430 
431 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
432 {
433         VM_BUG_ON(!is_vm_hugetlb_page(vma));
434 
435         return (get_vma_private_data(vma) & flag) != 0;
436 }
437 
438 /* Decrement the reserved pages in the hugepage pool by one */
439 static void decrement_hugepage_resv_vma(struct hstate *h,
440                         struct vm_area_struct *vma)
441 {
442         if (vma->vm_flags & VM_NORESERVE)
443                 return;
444 
445         if (vma->vm_flags & VM_MAYSHARE) {
446                 /* Shared mappings always use reserves */
447                 h->resv_huge_pages--;
448         } else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
449                 /*
450                  * Only the process that called mmap() has reserves for
451                  * private mappings.
452                  */
453                 h->resv_huge_pages--;
454         }
455 }
456 
457 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
458 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
459 {
460         VM_BUG_ON(!is_vm_hugetlb_page(vma));
461         if (!(vma->vm_flags & VM_MAYSHARE))
462                 vma->vm_private_data = (void *)0;
463 }
464 
465 /* Returns true if the VMA has associated reserve pages */
466 static int vma_has_reserves(struct vm_area_struct *vma)
467 {
468         if (vma->vm_flags & VM_MAYSHARE)
469                 return 1;
470         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
471                 return 1;
472         return 0;
473 }
474 
475 static void copy_gigantic_page(struct page *dst, struct page *src)
476 {
477         int i;
478         struct hstate *h = page_hstate(src);
479         struct page *dst_base = dst;
480         struct page *src_base = src;
481 
482         for (i = 0; i < pages_per_huge_page(h); ) {
483                 cond_resched();
484                 copy_highpage(dst, src);
485 
486                 i++;
487                 dst = mem_map_next(dst, dst_base, i);
488                 src = mem_map_next(src, src_base, i);
489         }
490 }
491 
492 void copy_huge_page(struct page *dst, struct page *src)
493 {
494         int i;
495         struct hstate *h = page_hstate(src);
496 
497         if (unlikely(pages_per_huge_page(h) > MAX_ORDER_NR_PAGES)) {
498                 copy_gigantic_page(dst, src);
499                 return;
500         }
501 
502         might_sleep();
503         for (i = 0; i < pages_per_huge_page(h); i++) {
504                 cond_resched();
505                 copy_highpage(dst + i, src + i);
506         }
507 }
508 
509 static void enqueue_huge_page(struct hstate *h, struct page *page)
510 {
511         int nid = page_to_nid(page);
512         list_add(&page->lru, &h->hugepage_freelists[nid]);
513         h->free_huge_pages++;
514         h->free_huge_pages_node[nid]++;
515 }
516 
517 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
518 {
519         struct page *page;
520 
521         if (list_empty(&h->hugepage_freelists[nid]))
522                 return NULL;
523         page = list_entry(h->hugepage_freelists[nid].next, struct page, lru);
524         list_del(&page->lru);
525         set_page_refcounted(page);
526         h->free_huge_pages--;
527         h->free_huge_pages_node[nid]--;
528         return page;
529 }
530 
531 static struct page *dequeue_huge_page_vma(struct hstate *h,
532                                 struct vm_area_struct *vma,
533                                 unsigned long address, int avoid_reserve)
534 {
535         struct page *page = NULL;
536         struct mempolicy *mpol;
537         nodemask_t *nodemask;
538         struct zonelist *zonelist;
539         struct zone *zone;
540         struct zoneref *z;
541         unsigned int cpuset_mems_cookie;
542 
543 retry_cpuset:
544         cpuset_mems_cookie = get_mems_allowed();
545         zonelist = huge_zonelist(vma, address,
546                                         htlb_alloc_mask, &mpol, &nodemask);
547         /*
548          * A child process with MAP_PRIVATE mappings created by their parent
549          * have no page reserves. This check ensures that reservations are
550          * not "stolen". The child may still get SIGKILLed
551          */
552         if (!vma_has_reserves(vma) &&
553                         h->free_huge_pages - h->resv_huge_pages == 0)
554                 goto err;
555 
556         /* If reserves cannot be used, ensure enough pages are in the pool */
557         if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
558                 goto err;
559 
560         for_each_zone_zonelist_nodemask(zone, z, zonelist,
561                                                 MAX_NR_ZONES - 1, nodemask) {
562                 if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask)) {
563                         page = dequeue_huge_page_node(h, zone_to_nid(zone));
564                         if (page) {
565                                 if (!avoid_reserve)
566                                         decrement_hugepage_resv_vma(h, vma);
567                                 break;
568                         }
569                 }
570         }
571 
572         mpol_cond_put(mpol);
573         if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
574                 goto retry_cpuset;
575         return page;
576 
577 err:
578         mpol_cond_put(mpol);
579         return NULL;
580 }
581 
582 static void update_and_free_page(struct hstate *h, struct page *page)
583 {
584         int i;
585 
586         VM_BUG_ON(h->order >= MAX_ORDER);
587 
588         h->nr_huge_pages--;
589         h->nr_huge_pages_node[page_to_nid(page)]--;
590         for (i = 0; i < pages_per_huge_page(h); i++) {
591                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
592                                 1 << PG_referenced | 1 << PG_dirty |
593                                 1 << PG_active | 1 << PG_reserved |
594                                 1 << PG_private | 1 << PG_writeback);
595         }
596         set_compound_page_dtor(page, NULL);
597         set_page_refcounted(page);
598         arch_release_hugepage(page);
599         __free_pages(page, huge_page_order(h));
600 }
601 
602 struct hstate *size_to_hstate(unsigned long size)
603 {
604         struct hstate *h;
605 
606         for_each_hstate(h) {
607                 if (huge_page_size(h) == size)
608                         return h;
609         }
610         return NULL;
611 }
612 
613 static void free_huge_page(struct page *page)
614 {
615         /*
616          * Can't pass hstate in here because it is called from the
617          * compound page destructor.
618          */
619         struct hstate *h = page_hstate(page);
620         int nid = page_to_nid(page);
621         struct hugepage_subpool *spool =
622                 (struct hugepage_subpool *)page_private(page);
623 
624         set_page_private(page, 0);
625         page->mapping = NULL;
626         BUG_ON(page_count(page));
627         BUG_ON(page_mapcount(page));
628         INIT_LIST_HEAD(&page->lru);
629 
630         spin_lock(&hugetlb_lock);
631         if (h->surplus_huge_pages_node[nid] && huge_page_order(h) < MAX_ORDER) {
632                 update_and_free_page(h, page);
633                 h->surplus_huge_pages--;
634                 h->surplus_huge_pages_node[nid]--;
635         } else {
636                 enqueue_huge_page(h, page);
637         }
638         spin_unlock(&hugetlb_lock);
639         hugepage_subpool_put_pages(spool, 1);
640 }
641 
642 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
643 {
644         set_compound_page_dtor(page, free_huge_page);
645         spin_lock(&hugetlb_lock);
646         h->nr_huge_pages++;
647         h->nr_huge_pages_node[nid]++;
648         spin_unlock(&hugetlb_lock);
649         put_page(page); /* free it into the hugepage allocator */
650 }
651 
652 static void prep_compound_gigantic_page(struct page *page, unsigned long order)
653 {
654         int i;
655         int nr_pages = 1 << order;
656         struct page *p = page + 1;
657 
658         /* we rely on prep_new_huge_page to set the destructor */
659         set_compound_order(page, order);
660         __SetPageHead(page);
661         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
662                 __SetPageTail(p);
663                 set_page_count(p, 0);
664                 p->first_page = page;
665         }
666 }
667 
668 int PageHuge(struct page *page)
669 {
670         compound_page_dtor *dtor;
671 
672         if (!PageCompound(page))
673                 return 0;
674 
675         page = compound_head(page);
676         dtor = get_compound_page_dtor(page);
677 
678         return dtor == free_huge_page;
679 }
680 EXPORT_SYMBOL_GPL(PageHuge);
681 
682 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
683 {
684         struct page *page;
685 
686         if (h->order >= MAX_ORDER)
687                 return NULL;
688 
689         page = alloc_pages_exact_node(nid,
690                 htlb_alloc_mask|__GFP_COMP|__GFP_THISNODE|
691                                                 __GFP_REPEAT|__GFP_NOWARN,
692                 huge_page_order(h));
693         if (page) {
694                 if (arch_prepare_hugepage(page)) {
695                         __free_pages(page, huge_page_order(h));
696                         return NULL;
697                 }
698                 prep_new_huge_page(h, page, nid);
699         }
700 
701         return page;
702 }
703 
704 /*
705  * common helper functions for hstate_next_node_to_{alloc|free}.
706  * We may have allocated or freed a huge page based on a different
707  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
708  * be outside of *nodes_allowed.  Ensure that we use an allowed
709  * node for alloc or free.
710  */
711 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
712 {
713         nid = next_node(nid, *nodes_allowed);
714         if (nid == MAX_NUMNODES)
715                 nid = first_node(*nodes_allowed);
716         VM_BUG_ON(nid >= MAX_NUMNODES);
717 
718         return nid;
719 }
720 
721 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
722 {
723         if (!node_isset(nid, *nodes_allowed))
724                 nid = next_node_allowed(nid, nodes_allowed);
725         return nid;
726 }
727 
728 /*
729  * returns the previously saved node ["this node"] from which to
730  * allocate a persistent huge page for the pool and advance the
731  * next node from which to allocate, handling wrap at end of node
732  * mask.
733  */
734 static int hstate_next_node_to_alloc(struct hstate *h,
735                                         nodemask_t *nodes_allowed)
736 {
737         int nid;
738 
739         VM_BUG_ON(!nodes_allowed);
740 
741         nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
742         h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
743 
744         return nid;
745 }
746 
747 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
748 {
749         struct page *page;
750         int start_nid;
751         int next_nid;
752         int ret = 0;
753 
754         start_nid = hstate_next_node_to_alloc(h, nodes_allowed);
755         next_nid = start_nid;
756 
757         do {
758                 page = alloc_fresh_huge_page_node(h, next_nid);
759                 if (page) {
760                         ret = 1;
761                         break;
762                 }
763                 next_nid = hstate_next_node_to_alloc(h, nodes_allowed);
764         } while (next_nid != start_nid);
765 
766         if (ret)
767                 count_vm_event(HTLB_BUDDY_PGALLOC);
768         else
769                 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
770 
771         return ret;
772 }
773 
774 /*
775  * helper for free_pool_huge_page() - return the previously saved
776  * node ["this node"] from which to free a huge page.  Advance the
777  * next node id whether or not we find a free huge page to free so
778  * that the next attempt to free addresses the next node.
779  */
780 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
781 {
782         int nid;
783 
784         VM_BUG_ON(!nodes_allowed);
785 
786         nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
787         h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
788 
789         return nid;
790 }
791 
792 /*
793  * Free huge page from pool from next node to free.
794  * Attempt to keep persistent huge pages more or less
795  * balanced over allowed nodes.
796  * Called with hugetlb_lock locked.
797  */
798 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
799                                                          bool acct_surplus)
800 {
801         int start_nid;
802         int next_nid;
803         int ret = 0;
804 
805         start_nid = hstate_next_node_to_free(h, nodes_allowed);
806         next_nid = start_nid;
807 
808         do {
809                 /*
810                  * If we're returning unused surplus pages, only examine
811                  * nodes with surplus pages.
812                  */
813                 if ((!acct_surplus || h->surplus_huge_pages_node[next_nid]) &&
814                     !list_empty(&h->hugepage_freelists[next_nid])) {
815                         struct page *page =
816                                 list_entry(h->hugepage_freelists[next_nid].next,
817                                           struct page, lru);
818                         list_del(&page->lru);
819                         h->free_huge_pages--;
820                         h->free_huge_pages_node[next_nid]--;
821                         if (acct_surplus) {
822                                 h->surplus_huge_pages--;
823                                 h->surplus_huge_pages_node[next_nid]--;
824                         }
825                         update_and_free_page(h, page);
826                         ret = 1;
827                         break;
828                 }
829                 next_nid = hstate_next_node_to_free(h, nodes_allowed);
830         } while (next_nid != start_nid);
831 
832         return ret;
833 }
834 
835 static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)
836 {
837         struct page *page;
838         unsigned int r_nid;
839 
840         if (h->order >= MAX_ORDER)
841                 return NULL;
842 
843         /*
844          * Assume we will successfully allocate the surplus page to
845          * prevent racing processes from causing the surplus to exceed
846          * overcommit
847          *
848          * This however introduces a different race, where a process B
849          * tries to grow the static hugepage pool while alloc_pages() is
850          * called by process A. B will only examine the per-node
851          * counters in determining if surplus huge pages can be
852          * converted to normal huge pages in adjust_pool_surplus(). A
853          * won't be able to increment the per-node counter, until the
854          * lock is dropped by B, but B doesn't drop hugetlb_lock until
855          * no more huge pages can be converted from surplus to normal
856          * state (and doesn't try to convert again). Thus, we have a
857          * case where a surplus huge page exists, the pool is grown, and
858          * the surplus huge page still exists after, even though it
859          * should just have been converted to a normal huge page. This
860          * does not leak memory, though, as the hugepage will be freed
861          * once it is out of use. It also does not allow the counters to
862          * go out of whack in adjust_pool_surplus() as we don't modify
863          * the node values until we've gotten the hugepage and only the
864          * per-node value is checked there.
865          */
866         spin_lock(&hugetlb_lock);
867         if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
868                 spin_unlock(&hugetlb_lock);
869                 return NULL;
870         } else {
871                 h->nr_huge_pages++;
872                 h->surplus_huge_pages++;
873         }
874         spin_unlock(&hugetlb_lock);
875 
876         if (nid == NUMA_NO_NODE)
877                 page = alloc_pages(htlb_alloc_mask|__GFP_COMP|
878                                    __GFP_REPEAT|__GFP_NOWARN,
879                                    huge_page_order(h));
880         else
881                 page = alloc_pages_exact_node(nid,
882                         htlb_alloc_mask|__GFP_COMP|__GFP_THISNODE|
883                         __GFP_REPEAT|__GFP_NOWARN, huge_page_order(h));
884 
885         if (page && arch_prepare_hugepage(page)) {
886                 __free_pages(page, huge_page_order(h));
887                 page = NULL;
888         }
889 
890         spin_lock(&hugetlb_lock);
891         if (page) {
892                 r_nid = page_to_nid(page);
893                 set_compound_page_dtor(page, free_huge_page);
894                 /*
895                  * We incremented the global counters already
896                  */
897                 h->nr_huge_pages_node[r_nid]++;
898                 h->surplus_huge_pages_node[r_nid]++;
899                 __count_vm_event(HTLB_BUDDY_PGALLOC);
900         } else {
901                 h->nr_huge_pages--;
902                 h->surplus_huge_pages--;
903                 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
904         }
905         spin_unlock(&hugetlb_lock);
906 
907         return page;
908 }
909 
910 /*
911  * This allocation function is useful in the context where vma is irrelevant.
912  * E.g. soft-offlining uses this function because it only cares physical
913  * address of error page.
914  */
915 struct page *alloc_huge_page_node(struct hstate *h, int nid)
916 {
917         struct page *page;
918 
919         spin_lock(&hugetlb_lock);
920         page = dequeue_huge_page_node(h, nid);
921         spin_unlock(&hugetlb_lock);
922 
923         if (!page)
924                 page = alloc_buddy_huge_page(h, nid);
925 
926         return page;
927 }
928 
929 /*
930  * Increase the hugetlb pool such that it can accommodate a reservation
931  * of size 'delta'.
932  */
933 static int gather_surplus_pages(struct hstate *h, int delta)
934 {
935         struct list_head surplus_list;
936         struct page *page, *tmp;
937         int ret, i;
938         int needed, allocated;
939         bool alloc_ok = true;
940 
941         needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
942         if (needed <= 0) {
943                 h->resv_huge_pages += delta;
944                 return 0;
945         }
946 
947         allocated = 0;
948         INIT_LIST_HEAD(&surplus_list);
949 
950         ret = -ENOMEM;
951 retry:
952         spin_unlock(&hugetlb_lock);
953         for (i = 0; i < needed; i++) {
954                 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
955                 if (!page) {
956                         alloc_ok = false;
957                         break;
958                 }
959                 list_add(&page->lru, &surplus_list);
960         }
961         allocated += i;
962 
963         /*
964          * After retaking hugetlb_lock, we need to recalculate 'needed'
965          * because either resv_huge_pages or free_huge_pages may have changed.
966          */
967         spin_lock(&hugetlb_lock);
968         needed = (h->resv_huge_pages + delta) -
969                         (h->free_huge_pages + allocated);
970         if (needed > 0) {
971                 if (alloc_ok)
972                         goto retry;
973                 /*
974                  * We were not able to allocate enough pages to
975                  * satisfy the entire reservation so we free what
976                  * we've allocated so far.
977                  */
978                 goto free;
979         }
980         /*
981          * The surplus_list now contains _at_least_ the number of extra pages
982          * needed to accommodate the reservation.  Add the appropriate number
983          * of pages to the hugetlb pool and free the extras back to the buddy
984          * allocator.  Commit the entire reservation here to prevent another
985          * process from stealing the pages as they are added to the pool but
986          * before they are reserved.
987          */
988         needed += allocated;
989         h->resv_huge_pages += delta;
990         ret = 0;
991 
992         /* Free the needed pages to the hugetlb pool */
993         list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
994                 if ((--needed) < 0)
995                         break;
996                 list_del(&page->lru);
997                 /*
998                  * This page is now managed by the hugetlb allocator and has
999                  * no users -- drop the buddy allocator's reference.
1000                  */
1001                 put_page_testzero(page);
1002                 VM_BUG_ON(page_count(page));
1003                 enqueue_huge_page(h, page);
1004         }
1005 free:
1006         spin_unlock(&hugetlb_lock);
1007 
1008         /* Free unnecessary surplus pages to the buddy allocator */
1009         if (!list_empty(&surplus_list)) {
1010                 list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1011                         list_del(&page->lru);
1012                         put_page(page);
1013                 }
1014         }
1015         spin_lock(&hugetlb_lock);
1016 
1017         return ret;
1018 }
1019 
1020 /*
1021  * When releasing a hugetlb pool reservation, any surplus pages that were
1022  * allocated to satisfy the reservation must be explicitly freed if they were
1023  * never used.
1024  * Called with hugetlb_lock held.
1025  */
1026 static void return_unused_surplus_pages(struct hstate *h,
1027                                         unsigned long unused_resv_pages)
1028 {
1029         unsigned long nr_pages;
1030 
1031         /* Uncommit the reservation */
1032         h->resv_huge_pages -= unused_resv_pages;
1033 
1034         /* Cannot return gigantic pages currently */
1035         if (h->order >= MAX_ORDER)
1036                 return;
1037 
1038         nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1039 
1040         /*
1041          * We want to release as many surplus pages as possible, spread
1042          * evenly across all nodes with memory. Iterate across these nodes
1043          * until we can no longer free unreserved surplus pages. This occurs
1044          * when the nodes with surplus pages have no free pages.
1045          * free_pool_huge_page() will balance the the freed pages across the
1046          * on-line nodes with memory and will handle the hstate accounting.
1047          */
1048         while (nr_pages--) {
1049                 if (!free_pool_huge_page(h, &node_states[N_HIGH_MEMORY], 1))
1050                         break;
1051         }
1052 }
1053 
1054 /*
1055  * Determine if the huge page at addr within the vma has an associated
1056  * reservation.  Where it does not we will need to logically increase
1057  * reservation and actually increase subpool usage before an allocation
1058  * can occur.  Where any new reservation would be required the
1059  * reservation change is prepared, but not committed.  Once the page
1060  * has been allocated from the subpool and instantiated the change should
1061  * be committed via vma_commit_reservation.  No action is required on
1062  * failure.
1063  */
1064 static long vma_needs_reservation(struct hstate *h,
1065                         struct vm_area_struct *vma, unsigned long addr)
1066 {
1067         struct address_space *mapping = vma->vm_file->f_mapping;
1068         struct inode *inode = mapping->host;
1069 
1070         if (vma->vm_flags & VM_MAYSHARE) {
1071                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1072                 return region_chg(&inode->i_mapping->private_list,
1073                                                         idx, idx + 1);
1074 
1075         } else if (!is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1076                 return 1;
1077 
1078         } else  {
1079                 long err;
1080                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1081                 struct resv_map *reservations = vma_resv_map(vma);
1082 
1083                 err = region_chg(&reservations->regions, idx, idx + 1);
1084                 if (err < 0)
1085                         return err;
1086                 return 0;
1087         }
1088 }
1089 static void vma_commit_reservation(struct hstate *h,
1090                         struct vm_area_struct *vma, unsigned long addr)
1091 {
1092         struct address_space *mapping = vma->vm_file->f_mapping;
1093         struct inode *inode = mapping->host;
1094 
1095         if (vma->vm_flags & VM_MAYSHARE) {
1096                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1097                 region_add(&inode->i_mapping->private_list, idx, idx + 1);
1098 
1099         } else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1100                 pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1101                 struct resv_map *reservations = vma_resv_map(vma);
1102 
1103                 /* Mark this page used in the map. */
1104                 region_add(&reservations->regions, idx, idx + 1);
1105         }
1106 }
1107 
1108 static struct page *alloc_huge_page(struct vm_area_struct *vma,
1109                                     unsigned long addr, int avoid_reserve)
1110 {
1111         struct hugepage_subpool *spool = subpool_vma(vma);
1112         struct hstate *h = hstate_vma(vma);
1113         struct page *page;
1114         long chg;
1115 
1116         /*
1117          * Processes that did not create the mapping will have no
1118          * reserves and will not have accounted against subpool
1119          * limit. Check that the subpool limit can be made before
1120          * satisfying the allocation MAP_NORESERVE mappings may also
1121          * need pages and subpool limit allocated allocated if no reserve
1122          * mapping overlaps.
1123          */
1124         chg = vma_needs_reservation(h, vma, addr);
1125         if (chg < 0)
1126                 return ERR_PTR(-VM_FAULT_OOM);
1127         if (chg)
1128                 if (hugepage_subpool_get_pages(spool, chg))
1129                         return ERR_PTR(-VM_FAULT_SIGBUS);
1130 
1131         spin_lock(&hugetlb_lock);
1132         page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve);
1133         spin_unlock(&hugetlb_lock);
1134 
1135         if (!page) {
1136                 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1137                 if (!page) {
1138                         hugepage_subpool_put_pages(spool, chg);
1139                         return ERR_PTR(-VM_FAULT_SIGBUS);
1140                 }
1141         }
1142 
1143         set_page_private(page, (unsigned long)spool);
1144 
1145         vma_commit_reservation(h, vma, addr);
1146 
1147         return page;
1148 }
1149 
1150 int __weak alloc_bootmem_huge_page(struct hstate *h)
1151 {
1152         struct huge_bootmem_page *m;
1153         int nr_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
1154 
1155         while (nr_nodes) {
1156                 void *addr;
1157 
1158                 addr = __alloc_bootmem_node_nopanic(
1159                                 NODE_DATA(hstate_next_node_to_alloc(h,
1160                                                 &node_states[N_HIGH_MEMORY])),
1161                                 huge_page_size(h), huge_page_size(h), 0);
1162 
1163                 if (addr) {
1164                         /*
1165                          * Use the beginning of the huge page to store the
1166                          * huge_bootmem_page struct (until gather_bootmem
1167                          * puts them into the mem_map).
1168                          */
1169                         m = addr;
1170                         goto found;
1171                 }
1172                 nr_nodes--;
1173         }
1174         return 0;
1175 
1176 found:
1177         BUG_ON((unsigned long)virt_to_phys(m) & (huge_page_size(h) - 1));
1178         /* Put them into a private list first because mem_map is not up yet */
1179         list_add(&m->list, &huge_boot_pages);
1180         m->hstate = h;
1181         return 1;
1182 }
1183 
1184 static void prep_compound_huge_page(struct page *page, int order)
1185 {
1186         if (unlikely(order > (MAX_ORDER - 1)))
1187                 prep_compound_gigantic_page(page, order);
1188         else
1189                 prep_compound_page(page, order);
1190 }
1191 
1192 /* Put bootmem huge pages into the standard lists after mem_map is up */
1193 static void __init gather_bootmem_prealloc(void)
1194 {
1195         struct huge_bootmem_page *m;
1196 
1197         list_for_each_entry(m, &huge_boot_pages, list) {
1198                 struct hstate *h = m->hstate;
1199                 struct page *page;
1200 
1201 #ifdef CONFIG_HIGHMEM
1202                 page = pfn_to_page(m->phys >> PAGE_SHIFT);
1203                 free_bootmem_late((unsigned long)m,
1204                                   sizeof(struct huge_bootmem_page));
1205 #else
1206                 page = virt_to_page(m);
1207 #endif
1208                 __ClearPageReserved(page);
1209                 WARN_ON(page_count(page) != 1);
1210                 prep_compound_huge_page(page, h->order);
1211                 prep_new_huge_page(h, page, page_to_nid(page));
1212                 /*
1213                  * If we had gigantic hugepages allocated at boot time, we need
1214                  * to restore the 'stolen' pages to totalram_pages in order to
1215                  * fix confusing memory reports from free(1) and another
1216                  * side-effects, like CommitLimit going negative.
1217                  */
1218                 if (h->order > (MAX_ORDER - 1))
1219                         totalram_pages += 1 << h->order;
1220         }
1221 }
1222 
1223 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
1224 {
1225         unsigned long i;
1226 
1227         for (i = 0; i < h->max_huge_pages; ++i) {
1228                 if (h->order >= MAX_ORDER) {
1229                         if (!alloc_bootmem_huge_page(h))
1230                                 break;
1231                 } else if (!alloc_fresh_huge_page(h,
1232                                          &node_states[N_HIGH_MEMORY]))
1233                         break;
1234         }
1235         h->max_huge_pages = i;
1236 }
1237 
1238 static void __init hugetlb_init_hstates(void)
1239 {
1240         struct hstate *h;
1241 
1242         for_each_hstate(h) {
1243                 /* oversize hugepages were init'ed in early boot */
1244                 if (h->order < MAX_ORDER)
1245                         hugetlb_hstate_alloc_pages(h);
1246         }
1247 }
1248 
1249 static char * __init memfmt(char *buf, unsigned long n)
1250 {
1251         if (n >= (1UL << 30))
1252                 sprintf(buf, "%lu GB", n >> 30);
1253         else if (n >= (1UL << 20))
1254                 sprintf(buf, "%lu MB", n >> 20);
1255         else
1256                 sprintf(buf, "%lu KB", n >> 10);
1257         return buf;
1258 }
1259 
1260 static void __init report_hugepages(void)
1261 {
1262         struct hstate *h;
1263 
1264         for_each_hstate(h) {
1265                 char buf[32];
1266                 printk(KERN_INFO "HugeTLB registered %s page size, "
1267                                  "pre-allocated %ld pages\n",
1268                         memfmt(buf, huge_page_size(h)),
1269                         h->free_huge_pages);
1270         }
1271 }
1272 
1273 #ifdef CONFIG_HIGHMEM
1274 static void try_to_free_low(struct hstate *h, unsigned long count,
1275                                                 nodemask_t *nodes_allowed)
1276 {
1277         int i;
1278 
1279         if (h->order >= MAX_ORDER)
1280                 return;
1281 
1282         for_each_node_mask(i, *nodes_allowed) {
1283                 struct page *page, *next;
1284                 struct list_head *freel = &h->hugepage_freelists[i];
1285                 list_for_each_entry_safe(page, next, freel, lru) {
1286                         if (count >= h->nr_huge_pages)
1287                                 return;
1288                         if (PageHighMem(page))
1289                                 continue;
1290                         list_del(&page->lru);
1291                         update_and_free_page(h, page);
1292                         h->free_huge_pages--;
1293                         h->free_huge_pages_node[page_to_nid(page)]--;
1294                 }
1295         }
1296 }
1297 #else
1298 static inline void try_to_free_low(struct hstate *h, unsigned long count,
1299                                                 nodemask_t *nodes_allowed)
1300 {
1301 }
1302 #endif
1303 
1304 /*
1305  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
1306  * balanced by operating on them in a round-robin fashion.
1307  * Returns 1 if an adjustment was made.
1308  */
1309 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
1310                                 int delta)
1311 {
1312         int start_nid, next_nid;
1313         int ret = 0;
1314 
1315         VM_BUG_ON(delta != -1 && delta != 1);
1316 
1317         if (delta < 0)
1318                 start_nid = hstate_next_node_to_alloc(h, nodes_allowed);
1319         else
1320                 start_nid = hstate_next_node_to_free(h, nodes_allowed);
1321         next_nid = start_nid;
1322 
1323         do {
1324                 int nid = next_nid;
1325                 if (delta < 0)  {
1326                         /*
1327                          * To shrink on this node, there must be a surplus page
1328                          */
1329                         if (!h->surplus_huge_pages_node[nid]) {
1330                                 next_nid = hstate_next_node_to_alloc(h,
1331                                                                 nodes_allowed);
1332                                 continue;
1333                         }
1334                 }
1335                 if (delta > 0) {
1336                         /*
1337                          * Surplus cannot exceed the total number of pages
1338                          */
1339                         if (h->surplus_huge_pages_node[nid] >=
1340                                                 h->nr_huge_pages_node[nid]) {
1341                                 next_nid = hstate_next_node_to_free(h,
1342                                                                 nodes_allowed);
1343                                 continue;
1344                         }
1345                 }
1346 
1347                 h->surplus_huge_pages += delta;
1348                 h->surplus_huge_pages_node[nid] += delta;
1349                 ret = 1;
1350                 break;
1351         } while (next_nid != start_nid);
1352 
1353         return ret;
1354 }
1355 
1356 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
1357 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
1358                                                 nodemask_t *nodes_allowed)
1359 {
1360         unsigned long min_count, ret;
1361 
1362         if (h->order >= MAX_ORDER)
1363                 return h->max_huge_pages;
1364 
1365         /*
1366          * Increase the pool size
1367          * First take pages out of surplus state.  Then make up the
1368          * remaining difference by allocating fresh huge pages.
1369          *
1370          * We might race with alloc_buddy_huge_page() here and be unable
1371          * to convert a surplus huge page to a normal huge page. That is
1372          * not critical, though, it just means the overall size of the
1373          * pool might be one hugepage larger than it needs to be, but
1374          * within all the constraints specified by the sysctls.
1375          */
1376         spin_lock(&hugetlb_lock);
1377         while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
1378                 if (!adjust_pool_surplus(h, nodes_allowed, -1))
1379                         break;
1380         }
1381 
1382         while (count > persistent_huge_pages(h)) {
1383                 /*
1384                  * If this allocation races such that we no longer need the
1385                  * page, free_huge_page will handle it by freeing the page
1386                  * and reducing the surplus.
1387                  */
1388                 spin_unlock(&hugetlb_lock);
1389                 ret = alloc_fresh_huge_page(h, nodes_allowed);
1390                 spin_lock(&hugetlb_lock);
1391                 if (!ret)
1392                         goto out;
1393 
1394                 /* Bail for signals. Probably ctrl-c from user */
1395                 if (signal_pending(current))
1396                         goto out;
1397         }
1398 
1399         /*
1400          * Decrease the pool size
1401          * First return free pages to the buddy allocator (being careful
1402          * to keep enough around to satisfy reservations).  Then place
1403          * pages into surplus state as needed so the pool will shrink
1404          * to the desired size as pages become free.
1405          *
1406          * By placing pages into the surplus state independent of the
1407          * overcommit value, we are allowing the surplus pool size to
1408          * exceed overcommit. There are few sane options here. Since
1409          * alloc_buddy_huge_page() is checking the global counter,
1410          * though, we'll note that we're not allowed to exceed surplus
1411          * and won't grow the pool anywhere else. Not until one of the
1412          * sysctls are changed, or the surplus pages go out of use.
1413          */
1414         min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
1415         min_count = max(count, min_count);
1416         try_to_free_low(h, min_count, nodes_allowed);
1417         while (min_count < persistent_huge_pages(h)) {
1418                 if (!free_pool_huge_page(h, nodes_allowed, 0))
1419                         break;
1420         }
1421         while (count < persistent_huge_pages(h)) {
1422                 if (!adjust_pool_surplus(h, nodes_allowed, 1))
1423                         break;
1424         }
1425 out:
1426         ret = persistent_huge_pages(h);
1427         spin_unlock(&hugetlb_lock);
1428         return ret;
1429 }
1430 
1431 #define HSTATE_ATTR_RO(_name) \
1432         static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
1433 
1434 #define HSTATE_ATTR(_name) \
1435         static struct kobj_attribute _name##_attr = \
1436                 __ATTR(_name, 0644, _name##_show, _name##_store)
1437 
1438 static struct kobject *hugepages_kobj;
1439 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
1440 
1441 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
1442 
1443 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
1444 {
1445         int i;
1446 
1447         for (i = 0; i < HUGE_MAX_HSTATE; i++)
1448                 if (hstate_kobjs[i] == kobj) {
1449                         if (nidp)
1450                                 *nidp = NUMA_NO_NODE;
1451                         return &hstates[i];
1452                 }
1453 
1454         return kobj_to_node_hstate(kobj, nidp);
1455 }
1456 
1457 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
1458                                         struct kobj_attribute *attr, char *buf)
1459 {
1460         struct hstate *h;
1461         unsigned long nr_huge_pages;
1462         int nid;
1463 
1464         h = kobj_to_hstate(kobj, &nid);
1465         if (nid == NUMA_NO_NODE)
1466                 nr_huge_pages = h->nr_huge_pages;
1467         else
1468                 nr_huge_pages = h->nr_huge_pages_node[nid];
1469 
1470         return sprintf(buf, "%lu\n", nr_huge_pages);
1471 }
1472 
1473 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
1474                         struct kobject *kobj, struct kobj_attribute *attr,
1475                         const char *buf, size_t len)
1476 {
1477         int err;
1478         int nid;
1479         unsigned long count;
1480         struct hstate *h;
1481         NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
1482 
1483         err = strict_strtoul(buf, 10, &count);
1484         if (err)
1485                 goto out;
1486 
1487         h = kobj_to_hstate(kobj, &nid);
1488         if (h->order >= MAX_ORDER) {
1489                 err = -EINVAL;
1490                 goto out;
1491         }
1492 
1493         if (nid == NUMA_NO_NODE) {
1494                 /*
1495                  * global hstate attribute
1496                  */
1497                 if (!(obey_mempolicy &&
1498                                 init_nodemask_of_mempolicy(nodes_allowed))) {
1499                         NODEMASK_FREE(nodes_allowed);
1500                         nodes_allowed = &node_states[N_HIGH_MEMORY];
1501                 }
1502         } else if (nodes_allowed) {
1503                 /*
1504                  * per node hstate attribute: adjust count to global,
1505                  * but restrict alloc/free to the specified node.
1506                  */
1507                 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
1508                 init_nodemask_of_node(nodes_allowed, nid);
1509         } else
1510                 nodes_allowed = &node_states[N_HIGH_MEMORY];
1511 
1512         h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
1513 
1514         if (nodes_allowed != &node_states[N_HIGH_MEMORY])
1515                 NODEMASK_FREE(nodes_allowed);
1516 
1517         return len;
1518 out:
1519         NODEMASK_FREE(nodes_allowed);
1520         return err;
1521 }
1522 
1523 static ssize_t nr_hugepages_show(struct kobject *kobj,
1524                                        struct kobj_attribute *attr, char *buf)
1525 {
1526         return nr_hugepages_show_common(kobj, attr, buf);
1527 }
1528 
1529 static ssize_t nr_hugepages_store(struct kobject *kobj,
1530                struct kobj_attribute *attr, const char *buf, size_t len)
1531 {
1532         return nr_hugepages_store_common(false, kobj, attr, buf, len);
1533 }
1534 HSTATE_ATTR(nr_hugepages);
1535 
1536 #ifdef CONFIG_NUMA
1537 
1538 /*
1539  * hstate attribute for optionally mempolicy-based constraint on persistent
1540  * huge page alloc/free.
1541  */
1542 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
1543                                        struct kobj_attribute *attr, char *buf)
1544 {
1545         return nr_hugepages_show_common(kobj, attr, buf);
1546 }
1547 
1548 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
1549                struct kobj_attribute *attr, const char *buf, size_t len)
1550 {
1551         return nr_hugepages_store_common(true, kobj, attr, buf, len);
1552 }
1553 HSTATE_ATTR(nr_hugepages_mempolicy);
1554 #endif
1555 
1556 
1557 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
1558                                         struct kobj_attribute *attr, char *buf)
1559 {
1560         struct hstate *h = kobj_to_hstate(kobj, NULL);
1561         return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
1562 }
1563 
1564 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
1565                 struct kobj_attribute *attr, const char *buf, size_t count)
1566 {
1567         int err;
1568         unsigned long input;
1569         struct hstate *h = kobj_to_hstate(kobj, NULL);
1570 
1571         if (h->order >= MAX_ORDER)
1572                 return -EINVAL;
1573 
1574         err = strict_strtoul(buf, 10, &input);
1575         if (err)
1576                 return err;
1577 
1578         spin_lock(&hugetlb_lock);
1579         h->nr_overcommit_huge_pages = input;
1580         spin_unlock(&hugetlb_lock);
1581 
1582         return count;
1583 }
1584 HSTATE_ATTR(nr_overcommit_hugepages);
1585 
1586 static ssize_t free_hugepages_show(struct kobject *kobj,
1587                                         struct kobj_attribute *attr, char *buf)
1588 {
1589         struct hstate *h;
1590         unsigned long free_huge_pages;
1591         int nid;
1592 
1593         h = kobj_to_hstate(kobj, &nid);
1594         if (nid == NUMA_NO_NODE)
1595                 free_huge_pages = h->free_huge_pages;
1596         else
1597                 free_huge_pages = h->free_huge_pages_node[nid];
1598 
1599         return sprintf(buf, "%lu\n", free_huge_pages);
1600 }
1601 HSTATE_ATTR_RO(free_hugepages);
1602 
1603 static ssize_t resv_hugepages_show(struct kobject *kobj,
1604                                         struct kobj_attribute *attr, char *buf)
1605 {
1606         struct hstate *h = kobj_to_hstate(kobj, NULL);
1607         return sprintf(buf, "%lu\n", h->resv_huge_pages);
1608 }
1609 HSTATE_ATTR_RO(resv_hugepages);
1610 
1611 static ssize_t surplus_hugepages_show(struct kobject *kobj,
1612                                         struct kobj_attribute *attr, char *buf)
1613 {
1614         struct hstate *h;
1615         unsigned long surplus_huge_pages;
1616         int nid;
1617 
1618         h = kobj_to_hstate(kobj, &nid);
1619         if (nid == NUMA_NO_NODE)
1620                 surplus_huge_pages = h->surplus_huge_pages;
1621         else
1622                 surplus_huge_pages = h->surplus_huge_pages_node[nid];
1623 
1624         return sprintf(buf, "%lu\n", surplus_huge_pages);
1625 }
1626 HSTATE_ATTR_RO(surplus_hugepages);
1627 
1628 static struct attribute *hstate_attrs[] = {
1629         &nr_hugepages_attr.attr,
1630         &nr_overcommit_hugepages_attr.attr,
1631         &free_hugepages_attr.attr,
1632         &resv_hugepages_attr.attr,
1633         &surplus_hugepages_attr.attr,
1634 #ifdef CONFIG_NUMA
1635         &nr_hugepages_mempolicy_attr.attr,
1636 #endif
1637         NULL,
1638 };
1639 
1640 static struct attribute_group hstate_attr_group = {
1641         .attrs = hstate_attrs,
1642 };
1643 
1644 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
1645                                     struct kobject **hstate_kobjs,
1646                                     struct attribute_group *hstate_attr_group)
1647 {
1648         int retval;
1649         int hi = h - hstates;
1650 
1651         hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
1652         if (!hstate_kobjs[hi])
1653                 return -ENOMEM;
1654 
1655         retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
1656         if (retval)
1657                 kobject_put(hstate_kobjs[hi]);
1658 
1659         return retval;
1660 }
1661 
1662 static void __init hugetlb_sysfs_init(void)
1663 {
1664         struct hstate *h;
1665         int err;
1666 
1667         hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
1668         if (!hugepages_kobj)
1669                 return;
1670 
1671         for_each_hstate(h) {
1672                 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
1673                                          hstate_kobjs, &hstate_attr_group);
1674                 if (err)
1675                         printk(KERN_ERR "Hugetlb: Unable to add hstate %s",
1676                                                                 h->name);
1677         }
1678 }
1679 
1680 #ifdef CONFIG_NUMA
1681 
1682 /*
1683  * node_hstate/s - associate per node hstate attributes, via their kobjects,
1684  * with node devices in node_devices[] using a parallel array.  The array
1685  * index of a node device or _hstate == node id.
1686  * This is here to avoid any static dependency of the node device driver, in
1687  * the base kernel, on the hugetlb module.
1688  */
1689 struct node_hstate {
1690         struct kobject          *hugepages_kobj;
1691         struct kobject          *hstate_kobjs[HUGE_MAX_HSTATE];
1692 };
1693 struct node_hstate node_hstates[MAX_NUMNODES];
1694 
1695 /*
1696  * A subset of global hstate attributes for node devices
1697  */
1698 static struct attribute *per_node_hstate_attrs[] = {
1699         &nr_hugepages_attr.attr,
1700         &free_hugepages_attr.attr,
1701         &surplus_hugepages_attr.attr,
1702         NULL,
1703 };
1704 
1705 static struct attribute_group per_node_hstate_attr_group = {
1706         .attrs = per_node_hstate_attrs,
1707 };
1708 
1709 /*
1710  * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
1711  * Returns node id via non-NULL nidp.
1712  */
1713 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1714 {
1715         int nid;
1716 
1717         for (nid = 0; nid < nr_node_ids; nid++) {
1718                 struct node_hstate *nhs = &node_hstates[nid];
1719                 int i;
1720                 for (i = 0; i < HUGE_MAX_HSTATE; i++)
1721                         if (nhs->hstate_kobjs[i] == kobj) {
1722                                 if (nidp)
1723                                         *nidp = nid;
1724                                 return &hstates[i];
1725                         }
1726         }
1727 
1728         BUG();
1729         return NULL;
1730 }
1731 
1732 /*
1733  * Unregister hstate attributes from a single node device.
1734  * No-op if no hstate attributes attached.
1735  */
1736 void hugetlb_unregister_node(struct node *node)
1737 {
1738         struct hstate *h;
1739         struct node_hstate *nhs = &node_hstates[node->dev.id];
1740 
1741         if (!nhs->hugepages_kobj)
1742                 return;         /* no hstate attributes */
1743 
1744         for_each_hstate(h)
1745                 if (nhs->hstate_kobjs[h - hstates]) {
1746                         kobject_put(nhs->hstate_kobjs[h - hstates]);
1747                         nhs->hstate_kobjs[h - hstates] = NULL;
1748                 }
1749 
1750         kobject_put(nhs->hugepages_kobj);
1751         nhs->hugepages_kobj = NULL;
1752 }
1753 
1754 /*
1755  * hugetlb module exit:  unregister hstate attributes from node devices
1756  * that have them.
1757  */
1758 static void hugetlb_unregister_all_nodes(void)
1759 {
1760         int nid;
1761 
1762         /*
1763          * disable node device registrations.
1764          */
1765         register_hugetlbfs_with_node(NULL, NULL);
1766 
1767         /*
1768          * remove hstate attributes from any nodes that have them.
1769          */
1770         for (nid = 0; nid < nr_node_ids; nid++)
1771                 hugetlb_unregister_node(&node_devices[nid]);
1772 }
1773 
1774 /*
1775  * Register hstate attributes for a single node device.
1776  * No-op if attributes already registered.
1777  */
1778 void hugetlb_register_node(struct node *node)
1779 {
1780         struct hstate *h;
1781         struct node_hstate *nhs = &node_hstates[node->dev.id];
1782         int err;
1783 
1784         if (nhs->hugepages_kobj)
1785                 return;         /* already allocated */
1786 
1787         nhs->hugepages_kobj = kobject_create_and_add("hugepages",
1788                                                         &node->dev.kobj);
1789         if (!nhs->hugepages_kobj)
1790                 return;
1791 
1792         for_each_hstate(h) {
1793                 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
1794                                                 nhs->hstate_kobjs,
1795                                                 &per_node_hstate_attr_group);
1796                 if (err) {
1797                         printk(KERN_ERR "Hugetlb: Unable to add hstate %s"
1798                                         " for node %d\n",
1799                                                 h->name, node->dev.id);
1800                         hugetlb_unregister_node(node);
1801                         break;
1802                 }
1803         }
1804 }
1805 
1806 /*
1807  * hugetlb init time:  register hstate attributes for all registered node
1808  * devices of nodes that have memory.  All on-line nodes should have
1809  * registered their associated device by this time.
1810  */
1811 static void hugetlb_register_all_nodes(void)
1812 {
1813         int nid;
1814 
1815         for_each_node_state(nid, N_HIGH_MEMORY) {
1816                 struct node *node = &node_devices[nid];
1817                 if (node->dev.id == nid)
1818                         hugetlb_register_node(node);
1819         }
1820 
1821         /*
1822          * Let the node device driver know we're here so it can
1823          * [un]register hstate attributes on node hotplug.
1824          */
1825         register_hugetlbfs_with_node(hugetlb_register_node,
1826                                      hugetlb_unregister_node);
1827 }
1828 #else   /* !CONFIG_NUMA */
1829 
1830 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1831 {
1832         BUG();
1833         if (nidp)
1834                 *nidp = -1;
1835         return NULL;
1836 }
1837 
1838 static void hugetlb_unregister_all_nodes(void) { }
1839 
1840 static void hugetlb_register_all_nodes(void) { }
1841 
1842 #endif
1843 
1844 static void __exit hugetlb_exit(void)
1845 {
1846         struct hstate *h;
1847 
1848         hugetlb_unregister_all_nodes();
1849 
1850         for_each_hstate(h) {
1851                 kobject_put(hstate_kobjs[h - hstates]);
1852         }
1853 
1854         kobject_put(hugepages_kobj);
1855 }
1856 module_exit(hugetlb_exit);
1857 
1858 static int __init hugetlb_init(void)
1859 {
1860         /* Some platform decide whether they support huge pages at boot
1861          * time. On these, such as powerpc, HPAGE_SHIFT is set to 0 when
1862          * there is no such support
1863          */
1864         if (HPAGE_SHIFT == 0)
1865                 return 0;
1866 
1867         if (!size_to_hstate(default_hstate_size)) {
1868                 default_hstate_size = HPAGE_SIZE;
1869                 if (!size_to_hstate(default_hstate_size))
1870                         hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
1871         }
1872         default_hstate_idx = size_to_hstate(default_hstate_size) - hstates;
1873         if (default_hstate_max_huge_pages)
1874                 default_hstate.max_huge_pages = default_hstate_max_huge_pages;
1875 
1876         hugetlb_init_hstates();
1877 
1878         gather_bootmem_prealloc();
1879 
1880         report_hugepages();
1881 
1882         hugetlb_sysfs_init();
1883 
1884         hugetlb_register_all_nodes();
1885 
1886         return 0;
1887 }
1888 module_init(hugetlb_init);
1889 
1890 /* Should be called on processing a hugepagesz=... option */
1891 void __init hugetlb_add_hstate(unsigned order)
1892 {
1893         struct hstate *h;
1894         unsigned long i;
1895 
1896         if (size_to_hstate(PAGE_SIZE << order)) {
1897                 printk(KERN_WARNING "hugepagesz= specified twice, ignoring\n");
1898                 return;
1899         }
1900         BUG_ON(max_hstate >= HUGE_MAX_HSTATE);
1901         BUG_ON(order == 0);
1902         h = &hstates[max_hstate++];
1903         h->order = order;
1904         h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
1905         h->nr_huge_pages = 0;
1906         h->free_huge_pages = 0;
1907         for (i = 0; i < MAX_NUMNODES; ++i)
1908                 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
1909         h->next_nid_to_alloc = first_node(node_states[N_HIGH_MEMORY]);
1910         h->next_nid_to_free = first_node(node_states[N_HIGH_MEMORY]);
1911         snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
1912                                         huge_page_size(h)/1024);
1913 
1914         parsed_hstate = h;
1915 }
1916 
1917 static int __init hugetlb_nrpages_setup(char *s)
1918 {
1919         unsigned long *mhp;
1920         static unsigned long *last_mhp;
1921 
1922         /*
1923          * !max_hstate means we haven't parsed a hugepagesz= parameter yet,
1924          * so this hugepages= parameter goes to the "default hstate".
1925          */
1926         if (!max_hstate)
1927                 mhp = &default_hstate_max_huge_pages;
1928         else
1929                 mhp = &parsed_hstate->max_huge_pages;
1930 
1931         if (mhp == last_mhp) {
1932                 printk(KERN_WARNING "hugepages= specified twice without "
1933                         "interleaving hugepagesz=, ignoring\n");
1934                 return 1;
1935         }
1936 
1937         if (sscanf(s, "%lu", mhp) <= 0)
1938                 *mhp = 0;
1939 
1940         /*
1941          * Global state is always initialized later in hugetlb_init.
1942          * But we need to allocate >= MAX_ORDER hstates here early to still
1943          * use the bootmem allocator.
1944          */
1945         if (max_hstate && parsed_hstate->order >= MAX_ORDER)
1946                 hugetlb_hstate_alloc_pages(parsed_hstate);
1947 
1948         last_mhp = mhp;
1949 
1950         return 1;
1951 }
1952 __setup("hugepages=", hugetlb_nrpages_setup);
1953 
1954 static int __init hugetlb_default_setup(char *s)
1955 {
1956         default_hstate_size = memparse(s, &s);
1957         return 1;
1958 }
1959 __setup("default_hugepagesz=", hugetlb_default_setup);
1960 
1961 static unsigned int cpuset_mems_nr(unsigned int *array)
1962 {
1963         int node;
1964         unsigned int nr = 0;
1965 
1966         for_each_node_mask(node, cpuset_current_mems_allowed)
1967                 nr += array[node];
1968 
1969         return nr;
1970 }
1971 
1972 #ifdef CONFIG_SYSCTL
1973 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
1974                          struct ctl_table *table, int write,
1975                          void __user *buffer, size_t *length, loff_t *ppos)
1976 {
1977         struct hstate *h = &default_hstate;
1978         unsigned long tmp;
1979         int ret;
1980 
1981         tmp = h->max_huge_pages;
1982 
1983         if (write && h->order >= MAX_ORDER)
1984                 return -EINVAL;
1985 
1986         table->data = &tmp;
1987         table->maxlen = sizeof(unsigned long);
1988         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
1989         if (ret)
1990                 goto out;
1991 
1992         if (write) {
1993                 NODEMASK_ALLOC(nodemask_t, nodes_allowed,
1994                                                 GFP_KERNEL | __GFP_NORETRY);
1995                 if (!(obey_mempolicy &&
1996                                init_nodemask_of_mempolicy(nodes_allowed))) {
1997                         NODEMASK_FREE(nodes_allowed);
1998                         nodes_allowed = &node_states[N_HIGH_MEMORY];
1999                 }
2000                 h->max_huge_pages = set_max_huge_pages(h, tmp, nodes_allowed);
2001 
2002                 if (nodes_allowed != &node_states[N_HIGH_MEMORY])
2003                         NODEMASK_FREE(nodes_allowed);
2004         }
2005 out:
2006         return ret;
2007 }
2008 
2009 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2010                           void __user *buffer, size_t *length, loff_t *ppos)
2011 {
2012 
2013         return hugetlb_sysctl_handler_common(false, table, write,
2014                                                         buffer, length, ppos);
2015 }
2016 
2017 #ifdef CONFIG_NUMA
2018 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2019                           void __user *buffer, size_t *length, loff_t *ppos)
2020 {
2021         return hugetlb_sysctl_handler_common(true, table, write,
2022                                                         buffer, length, ppos);
2023 }
2024 #endif /* CONFIG_NUMA */
2025 
2026 int hugetlb_treat_movable_handler(struct ctl_table *table, int write,
2027                         void __user *buffer,
2028                         size_t *length, loff_t *ppos)
2029 {
2030         proc_dointvec(table, write, buffer, length, ppos);
2031         if (hugepages_treat_as_movable)
2032                 htlb_alloc_mask = GFP_HIGHUSER_MOVABLE;
2033         else
2034                 htlb_alloc_mask = GFP_HIGHUSER;
2035         return 0;
2036 }
2037 
2038 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
2039                         void __user *buffer,
2040                         size_t *length, loff_t *ppos)
2041 {
2042         struct hstate *h = &default_hstate;
2043         unsigned long tmp;
2044         int ret;
2045 
2046         tmp = h->nr_overcommit_huge_pages;
2047 
2048         if (write && h->order >= MAX_ORDER)
2049                 return -EINVAL;
2050 
2051         table->data = &tmp;
2052         table->maxlen = sizeof(unsigned long);
2053         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2054         if (ret)
2055                 goto out;
2056 
2057         if (write) {
2058                 spin_lock(&hugetlb_lock);
2059                 h->nr_overcommit_huge_pages = tmp;
2060                 spin_unlock(&hugetlb_lock);
2061         }
2062 out:
2063         return ret;
2064 }
2065 
2066 #endif /* CONFIG_SYSCTL */
2067 
2068 void hugetlb_report_meminfo(struct seq_file *m)
2069 {
2070         struct hstate *h = &default_hstate;
2071         seq_printf(m,
2072                         "HugePages_Total:   %5lu\n"
2073                         "HugePages_Free:    %5lu\n"
2074                         "HugePages_Rsvd:    %5lu\n"
2075                         "HugePages_Surp:    %5lu\n"
2076                         "Hugepagesize:   %8lu kB\n",
2077                         h->nr_huge_pages,
2078                         h->free_huge_pages,
2079                         h->resv_huge_pages,
2080                         h->surplus_huge_pages,
2081                         1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2082 }
2083 
2084 int hugetlb_report_node_meminfo(int nid, char *buf)
2085 {
2086         struct hstate *h = &default_hstate;
2087         return sprintf(buf,
2088                 "Node %d HugePages_Total: %5u\n"
2089                 "Node %d HugePages_Free:  %5u\n"
2090                 "Node %d HugePages_Surp:  %5u\n",
2091                 nid, h->nr_huge_pages_node[nid],
2092                 nid, h->free_huge_pages_node[nid],
2093                 nid, h->surplus_huge_pages_node[nid]);
2094 }
2095 
2096 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
2097 unsigned long hugetlb_total_pages(void)
2098 {
2099         struct hstate *h = &default_hstate;
2100         return h->nr_huge_pages * pages_per_huge_page(h);
2101 }
2102 
2103 static int hugetlb_acct_memory(struct hstate *h, long delta)
2104 {
2105         int ret = -ENOMEM;
2106 
2107         spin_lock(&hugetlb_lock);
2108         /*
2109          * When cpuset is configured, it breaks the strict hugetlb page
2110          * reservation as the accounting is done on a global variable. Such
2111          * reservation is completely rubbish in the presence of cpuset because
2112          * the reservation is not checked against page availability for the
2113          * current cpuset. Application can still potentially OOM'ed by kernel
2114          * with lack of free htlb page in cpuset that the task is in.
2115          * Attempt to enforce strict accounting with cpuset is almost
2116          * impossible (or too ugly) because cpuset is too fluid that
2117          * task or memory node can be dynamically moved between cpusets.
2118          *
2119          * The change of semantics for shared hugetlb mapping with cpuset is
2120          * undesirable. However, in order to preserve some of the semantics,
2121          * we fall back to check against current free page availability as
2122          * a best attempt and hopefully to minimize the impact of changing
2123          * semantics that cpuset has.
2124          */
2125         if (delta > 0) {
2126                 if (gather_surplus_pages(h, delta) < 0)
2127                         goto out;
2128 
2129                 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
2130                         return_unused_surplus_pages(h, delta);
2131                         goto out;
2132                 }
2133         }
2134 
2135         ret = 0;
2136         if (delta < 0)
2137                 return_unused_surplus_pages(h, (unsigned long) -delta);
2138 
2139 out:
2140         spin_unlock(&hugetlb_lock);
2141         return ret;
2142 }
2143 
2144 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
2145 {
2146         struct resv_map *reservations = vma_resv_map(vma);
2147 
2148         /*
2149          * This new VMA should share its siblings reservation map if present.
2150          * The VMA will only ever have a valid reservation map pointer where
2151          * it is being copied for another still existing VMA.  As that VMA
2152          * has a reference to the reservation map it cannot disappear until
2153          * after this open call completes.  It is therefore safe to take a
2154          * new reference here without additional locking.
2155          */
2156         if (reservations)
2157                 kref_get(&reservations->refs);
2158 }
2159 
2160 static void resv_map_put(struct vm_area_struct *vma)
2161 {
2162         struct resv_map *reservations = vma_resv_map(vma);
2163 
2164         if (!reservations)
2165                 return;
2166         kref_put(&reservations->refs, resv_map_release);
2167 }
2168 
2169 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
2170 {
2171         struct hstate *h = hstate_vma(vma);
2172         struct resv_map *reservations = vma_resv_map(vma);
2173         struct hugepage_subpool *spool = subpool_vma(vma);
2174         unsigned long reserve;
2175         unsigned long start;
2176         unsigned long end;
2177 
2178         if (reservations) {
2179                 start = vma_hugecache_offset(h, vma, vma->vm_start);
2180                 end = vma_hugecache_offset(h, vma, vma->vm_end);
2181 
2182                 reserve = (end - start) -
2183                         region_count(&reservations->regions, start, end);
2184 
2185                 resv_map_put(vma);
2186 
2187                 if (reserve) {
2188                         hugetlb_acct_memory(h, -reserve);
2189                         hugepage_subpool_put_pages(spool, reserve);
2190                 }
2191         }
2192 }
2193 
2194 /*
2195  * We cannot handle pagefaults against hugetlb pages at all.  They cause
2196  * handle_mm_fault() to try to instantiate regular-sized pages in the
2197  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
2198  * this far.
2199  */
2200 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2201 {
2202         BUG();
2203         return 0;
2204 }
2205 
2206 const struct vm_operations_struct hugetlb_vm_ops = {
2207         .fault = hugetlb_vm_op_fault,
2208         .open = hugetlb_vm_op_open,
2209         .close = hugetlb_vm_op_close,
2210 };
2211 
2212 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
2213                                 int writable)
2214 {
2215         pte_t entry;
2216 
2217         if (writable) {
2218                 entry =
2219                     pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
2220         } else {
2221                 entry = huge_pte_wrprotect(mk_pte(page, vma->vm_page_prot));
2222         }
2223         entry = pte_mkyoung(entry);
2224         entry = pte_mkhuge(entry);
2225         entry = arch_make_huge_pte(entry, vma, page, writable);
2226 
2227         return entry;
2228 }
2229 
2230 static void set_huge_ptep_writable(struct vm_area_struct *vma,
2231                                    unsigned long address, pte_t *ptep)
2232 {
2233         pte_t entry;
2234 
2235         entry = pte_mkwrite(pte_mkdirty(huge_ptep_get(ptep)));
2236         if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
2237                 update_mmu_cache(vma, address, ptep);
2238 }
2239 
2240 
2241 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
2242                             struct vm_area_struct *vma)
2243 {
2244         pte_t *src_pte, *dst_pte, entry;
2245         struct page *ptepage;
2246         unsigned long addr;
2247         int cow;
2248         struct hstate *h = hstate_vma(vma);
2249         unsigned long sz = huge_page_size(h);
2250 
2251         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
2252 
2253         for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
2254                 src_pte = huge_pte_offset(src, addr);
2255                 if (!src_pte)
2256                         continue;
2257                 dst_pte = huge_pte_alloc(dst, addr, sz);
2258                 if (!dst_pte)
2259                         goto nomem;
2260 
2261                 /* If the pagetables are shared don't copy or take references */
2262                 if (dst_pte == src_pte)
2263                         continue;
2264 
2265                 spin_lock(&dst->page_table_lock);
2266                 spin_lock_nested(&src->page_table_lock, SINGLE_DEPTH_NESTING);
2267                 if (!huge_pte_none(huge_ptep_get(src_pte))) {
2268                         if (cow)
2269                                 huge_ptep_set_wrprotect(src, addr, src_pte);
2270                         entry = huge_ptep_get(src_pte);
2271                         ptepage = pte_page(entry);
2272                         get_page(ptepage);
2273                         page_dup_rmap(ptepage);
2274                         set_huge_pte_at(dst, addr, dst_pte, entry);
2275                 }
2276                 spin_unlock(&src->page_table_lock);
2277                 spin_unlock(&dst->page_table_lock);
2278         }
2279         return 0;
2280 
2281 nomem:
2282         return -ENOMEM;
2283 }
2284 
2285 static int is_hugetlb_entry_migration(pte_t pte)
2286 {
2287         swp_entry_t swp;
2288 
2289         if (huge_pte_none(pte) || pte_present(pte))
2290                 return 0;
2291         swp = pte_to_swp_entry(pte);
2292         if (non_swap_entry(swp) && is_migration_entry(swp))
2293                 return 1;
2294         else
2295                 return 0;
2296 }
2297 
2298 static int is_hugetlb_entry_hwpoisoned(pte_t pte)
2299 {
2300         swp_entry_t swp;
2301 
2302         if (huge_pte_none(pte) || pte_present(pte))
2303                 return 0;
2304         swp = pte_to_swp_entry(pte);
2305         if (non_swap_entry(swp) && is_hwpoison_entry(swp))
2306                 return 1;
2307         else
2308                 return 0;
2309 }
2310 
2311 void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2312                             unsigned long end, struct page *ref_page)
2313 {
2314         struct mm_struct *mm = vma->vm_mm;
2315         unsigned long address;
2316         pte_t *ptep;
2317         pte_t pte;
2318         struct page *page;
2319         struct page *tmp;
2320         struct hstate *h = hstate_vma(vma);
2321         unsigned long sz = huge_page_size(h);
2322 
2323         /*
2324          * A page gathering list, protected by per file i_mmap_mutex. The
2325          * lock is used to avoid list corruption from multiple unmapping
2326          * of the same page since we are using page->lru.
2327          */
2328         LIST_HEAD(page_list);
2329 
2330         WARN_ON(!is_vm_hugetlb_page(vma));
2331         BUG_ON(start & ~huge_page_mask(h));
2332         BUG_ON(end & ~huge_page_mask(h));
2333 
2334         mmu_notifier_invalidate_range_start(mm, start, end);
2335         spin_lock(&mm->page_table_lock);
2336         for (address = start; address < end; address += sz) {
2337                 ptep = huge_pte_offset(mm, address);
2338                 if (!ptep)
2339                         continue;
2340 
2341                 if (huge_pmd_unshare(mm, &address, ptep))
2342                         continue;
2343 
2344                 pte = huge_ptep_get(ptep);
2345                 if (huge_pte_none(pte))
2346                         continue;
2347 
2348                 /*
2349                  * HWPoisoned hugepage is already unmapped and dropped reference
2350                  */
2351                 if (unlikely(is_hugetlb_entry_hwpoisoned(pte)))
2352                         continue;
2353 
2354                 page = pte_page(pte);
2355                 /*
2356                  * If a reference page is supplied, it is because a specific
2357                  * page is being unmapped, not a range. Ensure the page we
2358                  * are about to unmap is the actual page of interest.
2359                  */
2360                 if (ref_page) {
2361                         if (page != ref_page)
2362                                 continue;
2363 
2364                         /*
2365                          * Mark the VMA as having unmapped its page so that
2366                          * future faults in this VMA will fail rather than
2367                          * looking like data was lost
2368                          */
2369                         set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
2370                 }
2371 
2372                 pte = huge_ptep_get_and_clear(mm, address, ptep);
2373                 if (pte_dirty(pte))
2374                         set_page_dirty(page);
2375                 list_add(&page->lru, &page_list);
2376 
2377                 /* Bail out after unmapping reference page if supplied */
2378                 if (ref_page)
2379                         break;
2380         }
2381         flush_tlb_range(vma, start, end);
2382         spin_unlock(&mm->page_table_lock);
2383         mmu_notifier_invalidate_range_end(mm, start, end);
2384         list_for_each_entry_safe(page, tmp, &page_list, lru) {
2385                 page_remove_rmap(page);
2386                 list_del(&page->lru);
2387                 put_page(page);
2388         }
2389 }
2390 
2391 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2392                           unsigned long end, struct page *ref_page)
2393 {
2394         mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex);
2395         __unmap_hugepage_range(vma, start, end, ref_page);
2396         /*
2397          * Clear this flag so that x86's huge_pmd_share page_table_shareable
2398          * test will fail on a vma being torn down, and not grab a page table
2399          * on its way out.  We're lucky that the flag has such an appropriate
2400          * name, and can in fact be safely cleared here. We could clear it
2401          * before the __unmap_hugepage_range above, but all that's necessary
2402          * is to clear it before releasing the i_mmap_mutex below.
2403          *
2404          * This works because in the contexts this is called, the VMA is
2405          * going to be destroyed. It is not vunerable to madvise(DONTNEED)
2406          * because madvise is not supported on hugetlbfs. The same applies
2407          * for direct IO. unmap_hugepage_range() is only being called just
2408          * before free_pgtables() so clearing VM_MAYSHARE will not cause
2409          * surprises later.
2410          */
2411         vma->vm_flags &= ~VM_MAYSHARE;
2412         mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
2413 }
2414 
2415 /*
2416  * This is called when the original mapper is failing to COW a MAP_PRIVATE
2417  * mappping it owns the reserve page for. The intention is to unmap the page
2418  * from other VMAs and let the children be SIGKILLed if they are faulting the
2419  * same region.
2420  */
2421 static int unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
2422                                 struct page *page, unsigned long address)
2423 {
2424         struct hstate *h = hstate_vma(vma);
2425         struct vm_area_struct *iter_vma;
2426         struct address_space *mapping;
2427         struct prio_tree_iter iter;
2428         pgoff_t pgoff;
2429 
2430         /*
2431          * vm_pgoff is in PAGE_SIZE units, hence the different calculation
2432          * from page cache lookup which is in HPAGE_SIZE units.
2433          */
2434         address = address & huge_page_mask(h);
2435         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
2436                         vma->vm_pgoff;
2437         mapping = vma->vm_file->f_dentry->d_inode->i_mapping;
2438 
2439         /*
2440          * Take the mapping lock for the duration of the table walk. As
2441          * this mapping should be shared between all the VMAs,
2442          * __unmap_hugepage_range() is called as the lock is already held
2443          */
2444         mutex_lock(&mapping->i_mmap_mutex);
2445         vma_prio_tree_foreach(iter_vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
2446                 /* Do not unmap the current VMA */
2447                 if (iter_vma == vma)
2448                         continue;
2449 
2450                 /*
2451                  * Unmap the page from other VMAs without their own reserves.
2452                  * They get marked to be SIGKILLed if they fault in these
2453                  * areas. This is because a future no-page fault on this VMA
2454                  * could insert a zeroed page instead of the data existing
2455                  * from the time of fork. This would look like data corruption
2456                  */
2457                 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
2458                         __unmap_hugepage_range(iter_vma,
2459                                 address, address + huge_page_size(h),
2460                                 page);
2461         }
2462         mutex_unlock(&mapping->i_mmap_mutex);
2463 
2464         return 1;
2465 }
2466 
2467 /*
2468  * Hugetlb_cow() should be called with page lock of the original hugepage held.
2469  * Called with hugetlb_instantiation_mutex held and pte_page locked so we
2470  * cannot race with other handlers or page migration.
2471  * Keep the pte_same checks anyway to make transition from the mutex easier.
2472  */
2473 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
2474                         unsigned long address, pte_t *ptep, pte_t pte,
2475                         struct page *pagecache_page)
2476 {
2477         struct hstate *h = hstate_vma(vma);
2478         struct page *old_page, *new_page;
2479         int avoidcopy;
2480         int outside_reserve = 0;
2481 
2482         old_page = pte_page(pte);
2483 
2484 retry_avoidcopy:
2485         /* If no-one else is actually using this page, avoid the copy
2486          * and just make the page writable */
2487         avoidcopy = (page_mapcount(old_page) == 1);
2488         if (avoidcopy) {
2489                 if (PageAnon(old_page))
2490                         page_move_anon_rmap(old_page, vma, address);
2491                 set_huge_ptep_writable(vma, address, ptep);
2492                 return 0;
2493         }
2494 
2495         /*
2496          * If the process that created a MAP_PRIVATE mapping is about to
2497          * perform a COW due to a shared page count, attempt to satisfy
2498          * the allocation without using the existing reserves. The pagecache
2499          * page is used to determine if the reserve at this address was
2500          * consumed or not. If reserves were used, a partial faulted mapping
2501          * at the time of fork() could consume its reserves on COW instead
2502          * of the full address range.
2503          */
2504         if (!(vma->vm_flags & VM_MAYSHARE) &&
2505                         is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
2506                         old_page != pagecache_page)
2507                 outside_reserve = 1;
2508 
2509         page_cache_get(old_page);
2510 
2511         /* Drop page_table_lock as buddy allocator may be called */
2512         spin_unlock(&mm->page_table_lock);
2513         new_page = alloc_huge_page(vma, address, outside_reserve);
2514 
2515         if (IS_ERR(new_page)) {
2516                 page_cache_release(old_page);
2517 
2518                 /*
2519                  * If a process owning a MAP_PRIVATE mapping fails to COW,
2520                  * it is due to references held by a child and an insufficient
2521                  * huge page pool. To guarantee the original mappers
2522                  * reliability, unmap the page from child processes. The child
2523                  * may get SIGKILLed if it later faults.
2524                  */
2525                 if (outside_reserve) {
2526                         BUG_ON(huge_pte_none(pte));
2527                         if (unmap_ref_private(mm, vma, old_page, address)) {
2528                                 BUG_ON(huge_pte_none(pte));
2529                                 spin_lock(&mm->page_table_lock);
2530                                 ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2531                                 if (likely(pte_same(huge_ptep_get(ptep), pte)))
2532                                         goto retry_avoidcopy;
2533                                 /*
2534                                  * race occurs while re-acquiring page_table_lock, and
2535                                  * our job is done.
2536                                  */
2537                                 return 0;
2538                         }
2539                         WARN_ON_ONCE(1);
2540                 }
2541 
2542                 /* Caller expects lock to be held */
2543                 spin_lock(&mm->page_table_lock);
2544                 return -PTR_ERR(new_page);
2545         }
2546 
2547         /*
2548          * When the original hugepage is shared one, it does not have
2549          * anon_vma prepared.
2550          */
2551         if (unlikely(anon_vma_prepare(vma))) {
2552                 page_cache_release(new_page);
2553                 page_cache_release(old_page);
2554                 /* Caller expects lock to be held */
2555                 spin_lock(&mm->page_table_lock);
2556                 return VM_FAULT_OOM;
2557         }
2558 
2559         copy_user_huge_page(new_page, old_page, address, vma,
2560                             pages_per_huge_page(h));
2561         __SetPageUptodate(new_page);
2562 
2563         /*
2564          * Retake the page_table_lock to check for racing updates
2565          * before the page tables are altered
2566          */
2567         spin_lock(&mm->page_table_lock);
2568         ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2569         if (likely(pte_same(huge_ptep_get(ptep), pte))) {
2570                 /* Break COW */
2571                 mmu_notifier_invalidate_range_start(mm,
2572                         address & huge_page_mask(h),
2573                         (address & huge_page_mask(h)) + huge_page_size(h));
2574                 huge_ptep_clear_flush(vma, address, ptep);
2575                 set_huge_pte_at(mm, address, ptep,
2576                                 make_huge_pte(vma, new_page, 1));
2577                 page_remove_rmap(old_page);
2578                 hugepage_add_new_anon_rmap(new_page, vma, address);
2579                 /* Make the old page be freed below */
2580                 new_page = old_page;
2581                 mmu_notifier_invalidate_range_end(mm,
2582                         address & huge_page_mask(h),
2583                         (address & huge_page_mask(h)) + huge_page_size(h));
2584         }
2585         page_cache_release(new_page);
2586         page_cache_release(old_page);
2587         return 0;
2588 }
2589 
2590 /* Return the pagecache page at a given address within a VMA */
2591 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
2592                         struct vm_area_struct *vma, unsigned long address)
2593 {
2594         struct address_space *mapping;
2595         pgoff_t idx;
2596 
2597         mapping = vma->vm_file->f_mapping;
2598         idx = vma_hugecache_offset(h, vma, address);
2599 
2600         return find_lock_page(mapping, idx);
2601 }
2602 
2603 /*
2604  * Return whether there is a pagecache page to back given address within VMA.
2605  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
2606  */
2607 static bool hugetlbfs_pagecache_present(struct hstate *h,
2608                         struct vm_area_struct *vma, unsigned long address)
2609 {
2610         struct address_space *mapping;
2611         pgoff_t idx;
2612         struct page *page;
2613 
2614         mapping = vma->vm_file->f_mapping;
2615         idx = vma_hugecache_offset(h, vma, address);
2616 
2617         page = find_get_page(mapping, idx);
2618         if (page)
2619                 put_page(page);
2620         return page != NULL;
2621 }
2622 
2623 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
2624                         unsigned long address, pte_t *ptep, unsigned int flags)
2625 {
2626         struct hstate *h = hstate_vma(vma);
2627         int ret = VM_FAULT_SIGBUS;
2628         int anon_rmap = 0;
2629         pgoff_t idx;
2630         unsigned long size;
2631         struct page *page;
2632         struct address_space *mapping;
2633         pte_t new_pte;
2634 
2635         /*
2636          * Currently, we are forced to kill the process in the event the
2637          * original mapper has unmapped pages from the child due to a failed
2638          * COW. Warn that such a situation has occurred as it may not be obvious
2639          */
2640         if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
2641                 printk(KERN_WARNING
2642                         "PID %d killed due to inadequate hugepage pool\n",
2643                         current->pid);
2644                 return ret;
2645         }
2646 
2647         mapping = vma->vm_file->f_mapping;
2648         idx = vma_hugecache_offset(h, vma, address);
2649 
2650         /*
2651          * Use page lock to guard against racing truncation
2652          * before we get page_table_lock.
2653          */
2654 retry:
2655         page = find_lock_page(mapping, idx);
2656         if (!page) {
2657                 size = i_size_read(mapping->host) >> huge_page_shift(h);
2658                 if (idx >= size)
2659                         goto out;
2660                 page = alloc_huge_page(vma, address, 0);
2661                 if (IS_ERR(page)) {
2662                         ret = -PTR_ERR(page);
2663                         goto out;
2664                 }
2665                 clear_huge_page(page, address, pages_per_huge_page(h));
2666                 __SetPageUptodate(page);
2667 
2668                 if (vma->vm_flags & VM_MAYSHARE) {
2669                         int err;
2670                         struct inode *inode = mapping->host;
2671 
2672                         err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
2673                         if (err) {
2674                                 put_page(page);
2675                                 if (err == -EEXIST)
2676                                         goto retry;
2677                                 goto out;
2678                         }
2679 
2680                         spin_lock(&inode->i_lock);
2681                         inode->i_blocks += blocks_per_huge_page(h);
2682                         spin_unlock(&inode->i_lock);
2683                 } else {
2684                         lock_page(page);
2685                         if (unlikely(anon_vma_prepare(vma))) {
2686                                 ret = VM_FAULT_OOM;
2687                                 goto backout_unlocked;
2688                         }
2689                         anon_rmap = 1;
2690                 }
2691         } else {
2692                 /*
2693                  * If memory error occurs between mmap() and fault, some process
2694                  * don't have hwpoisoned swap entry for errored virtual address.
2695                  * So we need to block hugepage fault by PG_hwpoison bit check.
2696                  */
2697                 if (unlikely(PageHWPoison(page))) {
2698                         ret = VM_FAULT_HWPOISON |
2699                               VM_FAULT_SET_HINDEX(h - hstates);
2700                         goto backout_unlocked;
2701                 }
2702         }
2703 
2704         /*
2705          * If we are going to COW a private mapping later, we examine the
2706          * pending reservations for this page now. This will ensure that
2707          * any allocations necessary to record that reservation occur outside
2708          * the spinlock.
2709          */
2710         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
2711                 if (vma_needs_reservation(h, vma, address) < 0) {
2712                         ret = VM_FAULT_OOM;
2713                         goto backout_unlocked;
2714                 }
2715 
2716         spin_lock(&mm->page_table_lock);
2717         size = i_size_read(mapping->host) >> huge_page_shift(h);
2718         if (idx >= size)
2719                 goto backout;
2720 
2721         ret = 0;
2722         if (!huge_pte_none(huge_ptep_get(ptep)))
2723                 goto backout;
2724 
2725         if (anon_rmap)
2726                 hugepage_add_new_anon_rmap(page, vma, address);
2727         else
2728                 page_dup_rmap(page);
2729         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
2730                                 && (vma->vm_flags & VM_SHARED)));
2731         set_huge_pte_at(mm, address, ptep, new_pte);
2732 
2733         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
2734                 /* Optimization, do the COW without a second fault */
2735                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page);
2736         }
2737 
2738         spin_unlock(&mm->page_table_lock);
2739         unlock_page(page);
2740 out:
2741         return ret;
2742 
2743 backout:
2744         spin_unlock(&mm->page_table_lock);
2745 backout_unlocked:
2746         unlock_page(page);
2747         put_page(page);
2748         goto out;
2749 }
2750 
2751 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
2752                         unsigned long address, unsigned int flags)
2753 {
2754         pte_t *ptep;
2755         pte_t entry;
2756         int ret;
2757         struct page *page = NULL;
2758         struct page *pagecache_page = NULL;
2759         static DEFINE_MUTEX(hugetlb_instantiation_mutex);
2760         struct hstate *h = hstate_vma(vma);
2761 
2762         address &= huge_page_mask(h);
2763 
2764         ptep = huge_pte_offset(mm, address);
2765         if (ptep) {
2766                 entry = huge_ptep_get(ptep);
2767                 if (unlikely(is_hugetlb_entry_migration(entry))) {
2768                         migration_entry_wait(mm, (pmd_t *)ptep, address);
2769                         return 0;
2770                 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
2771                         return VM_FAULT_HWPOISON_LARGE |
2772                                VM_FAULT_SET_HINDEX(h - hstates);
2773         }
2774 
2775         ptep = huge_pte_alloc(mm, address, huge_page_size(h));
2776         if (!ptep)
2777                 return VM_FAULT_OOM;
2778 
2779         /*
2780          * Serialize hugepage allocation and instantiation, so that we don't
2781          * get spurious allocation failures if two CPUs race to instantiate
2782          * the same page in the page cache.
2783          */
2784         mutex_lock(&hugetlb_instantiation_mutex);
2785         entry = huge_ptep_get(ptep);
2786         if (huge_pte_none(entry)) {
2787                 ret = hugetlb_no_page(mm, vma, address, ptep, flags);
2788                 goto out_mutex;
2789         }
2790 
2791         ret = 0;
2792 
2793         /*
2794          * If we are going to COW the mapping later, we examine the pending
2795          * reservations for this page now. This will ensure that any
2796          * allocations necessary to record that reservation occur outside the
2797          * spinlock. For private mappings, we also lookup the pagecache
2798          * page now as it is used to determine if a reservation has been
2799          * consumed.
2800          */
2801         if ((flags & FAULT_FLAG_WRITE) && !pte_write(entry)) {
2802                 if (vma_needs_reservation(h, vma, address) < 0) {
2803                         ret = VM_FAULT_OOM;
2804                         goto out_mutex;
2805                 }
2806 
2807                 if (!(vma->vm_flags & VM_MAYSHARE))
2808                         pagecache_page = hugetlbfs_pagecache_page(h,
2809                                                                 vma, address);
2810         }
2811 
2812         /*
2813          * hugetlb_cow() requires page locks of pte_page(entry) and
2814          * pagecache_page, so here we need take the former one
2815          * when page != pagecache_page or !pagecache_page.
2816          * Note that locking order is always pagecache_page -> page,
2817          * so no worry about deadlock.
2818          */
2819         page = pte_page(entry);
2820         get_page(page);
2821         if (page != pagecache_page)
2822                 lock_page(page);
2823 
2824         spin_lock(&mm->page_table_lock);
2825         /* Check for a racing update before calling hugetlb_cow */
2826         if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
2827                 goto out_page_table_lock;
2828 
2829 
2830         if (flags & FAULT_FLAG_WRITE) {
2831                 if (!pte_write(entry)) {
2832                         ret = hugetlb_cow(mm, vma, address, ptep, entry,
2833                                                         pagecache_page);
2834                         goto out_page_table_lock;
2835                 }
2836                 entry = pte_mkdirty(entry);
2837         }
2838         entry = pte_mkyoung(entry);
2839         if (huge_ptep_set_access_flags(vma, address, ptep, entry,
2840                                                 flags & FAULT_FLAG_WRITE))
2841                 update_mmu_cache(vma, address, ptep);
2842 
2843 out_page_table_lock:
2844         spin_unlock(&mm->page_table_lock);
2845 
2846         if (pagecache_page) {
2847                 unlock_page(pagecache_page);
2848                 put_page(pagecache_page);
2849         }
2850         if (page != pagecache_page)
2851                 unlock_page(page);
2852         put_page(page);
2853 
2854 out_mutex:
2855         mutex_unlock(&hugetlb_instantiation_mutex);
2856 
2857         return ret;
2858 }
2859 
2860 /* Can be overriden by architectures */
2861 __attribute__((weak)) struct page *
2862 follow_huge_pud(struct mm_struct *mm, unsigned long address,
2863                pud_t *pud, int write)
2864 {
2865         BUG();
2866         return NULL;
2867 }
2868 
2869 int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
2870                         struct page **pages, struct vm_area_struct **vmas,
2871                         unsigned long *position, int *length, int i,
2872                         unsigned int flags)
2873 {
2874         unsigned long pfn_offset;
2875         unsigned long vaddr = *position;
2876         int remainder = *length;
2877         struct hstate *h = hstate_vma(vma);
2878 
2879         spin_lock(&mm->page_table_lock);
2880         while (vaddr < vma->vm_end && remainder) {
2881                 pte_t *pte;
2882                 int absent;
2883                 struct page *page;
2884 
2885                 /*
2886                  * Some archs (sparc64, sh*) have multiple pte_ts to
2887                  * each hugepage.  We have to make sure we get the
2888                  * first, for the page indexing below to work.
2889                  */
2890                 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
2891                 absent = !pte || huge_pte_none(huge_ptep_get(pte));
2892 
2893                 /*
2894                  * When coredumping, it suits get_dump_page if we just return
2895                  * an error where there's an empty slot with no huge pagecache
2896                  * to back it.  This way, we avoid allocating a hugepage, and
2897                  * the sparse dumpfile avoids allocating disk blocks, but its
2898                  * huge holes still show up with zeroes where they need to be.
2899                  */
2900                 if (absent && (flags & FOLL_DUMP) &&
2901                     !hugetlbfs_pagecache_present(h, vma, vaddr)) {
2902                         remainder = 0;
2903                         break;
2904                 }
2905 
2906                 if (absent ||
2907                     ((flags & FOLL_WRITE) && !pte_write(huge_ptep_get(pte)))) {
2908                         int ret;
2909 
2910                         spin_unlock(&mm->page_table_lock);
2911                         ret = hugetlb_fault(mm, vma, vaddr,
2912                                 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
2913                         spin_lock(&mm->page_table_lock);
2914                         if (!(ret & VM_FAULT_ERROR))
2915                                 continue;
2916 
2917                         remainder = 0;
2918                         break;
2919                 }
2920 
2921                 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
2922                 page = pte_page(huge_ptep_get(pte));
2923 same_page:
2924                 if (pages) {
2925                         pages[i] = mem_map_offset(page, pfn_offset);
2926                         get_page(pages[i]);
2927                 }
2928 
2929                 if (vmas)
2930                         vmas[i] = vma;
2931 
2932                 vaddr += PAGE_SIZE;
2933                 ++pfn_offset;
2934                 --remainder;
2935                 ++i;
2936                 if (vaddr < vma->vm_end && remainder &&
2937                                 pfn_offset < pages_per_huge_page(h)) {
2938                         /*
2939                          * We use pfn_offset to avoid touching the pageframes
2940                          * of this compound page.
2941                          */
2942                         goto same_page;
2943                 }
2944         }
2945         spin_unlock(&mm->page_table_lock);
2946         *length = remainder;
2947         *position = vaddr;
2948 
2949         return i ? i : -EFAULT;
2950 }
2951 
2952 void hugetlb_change_protection(struct vm_area_struct *vma,
2953                 unsigned long address, unsigned long end, pgprot_t newprot)
2954 {
2955         struct mm_struct *mm = vma->vm_mm;
2956         unsigned long start = address;
2957         pte_t *ptep;
2958         pte_t pte;
2959         struct hstate *h = hstate_vma(vma);
2960 
2961         BUG_ON(address >= end);
2962         flush_cache_range(vma, address, end);
2963 
2964         mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex);
2965         spin_lock(&mm->page_table_lock);
2966         for (; address < end; address += huge_page_size(h)) {
2967                 ptep = huge_pte_offset(mm, address);
2968                 if (!ptep)
2969                         continue;
2970                 if (huge_pmd_unshare(mm, &address, ptep))
2971                         continue;
2972                 if (!huge_pte_none(huge_ptep_get(ptep))) {
2973                         pte = huge_ptep_get_and_clear(mm, address, ptep);
2974                         pte = pte_mkhuge(pte_modify(pte, newprot));
2975                         set_huge_pte_at(mm, address, ptep, pte);
2976                 }
2977         }
2978         spin_unlock(&mm->page_table_lock);
2979         /*
2980          * Must flush TLB before releasing i_mmap_mutex: x86's huge_pmd_unshare
2981          * may have cleared our pud entry and done put_page on the page table:
2982          * once we release i_mmap_mutex, another task can do the final put_page
2983          * and that page table be reused and filled with junk.
2984          */
2985         flush_tlb_range(vma, start, end);
2986         mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
2987 }
2988 
2989 int hugetlb_reserve_pages(struct inode *inode,
2990                                         long from, long to,
2991                                         struct vm_area_struct *vma,
2992                                         vm_flags_t vm_flags)
2993 {
2994         long ret, chg;
2995         struct hstate *h = hstate_inode(inode);
2996         struct hugepage_subpool *spool = subpool_inode(inode);
2997 
2998         /*
2999          * Only apply hugepage reservation if asked. At fault time, an
3000          * attempt will be made for VM_NORESERVE to allocate a page
3001          * without using reserves
3002          */
3003         if (vm_flags & VM_NORESERVE)
3004                 return 0;
3005 
3006         /*
3007          * Shared mappings base their reservation on the number of pages that
3008          * are already allocated on behalf of the file. Private mappings need
3009          * to reserve the full area even if read-only as mprotect() may be
3010          * called to make the mapping read-write. Assume !vma is a shm mapping
3011          */
3012         if (!vma || vma->vm_flags & VM_MAYSHARE)
3013                 chg = region_chg(&inode->i_mapping->private_list, from, to);
3014         else {
3015                 struct resv_map *resv_map = resv_map_alloc();
3016                 if (!resv_map)
3017                         return -ENOMEM;
3018 
3019                 chg = to - from;
3020 
3021                 set_vma_resv_map(vma, resv_map);
3022                 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
3023         }
3024 
3025         if (chg < 0) {
3026                 ret = chg;
3027                 goto out_err;
3028         }
3029 
3030         /* There must be enough pages in the subpool for the mapping */
3031         if (hugepage_subpool_get_pages(spool, chg)) {
3032                 ret = -ENOSPC;
3033                 goto out_err;
3034         }
3035 
3036         /*
3037          * Check enough hugepages are available for the reservation.
3038          * Hand the pages back to the subpool if there are not
3039          */
3040         ret = hugetlb_acct_memory(h, chg);
3041         if (ret < 0) {
3042                 hugepage_subpool_put_pages(spool, chg);
3043                 goto out_err;
3044         }
3045 
3046         /*
3047          * Account for the reservations made. Shared mappings record regions
3048          * that have reservations as they are shared by multiple VMAs.
3049          * When the last VMA disappears, the region map says how much
3050          * the reservation was and the page cache tells how much of
3051          * the reservation was consumed. Private mappings are per-VMA and
3052          * only the consumed reservations are tracked. When the VMA
3053          * disappears, the original reservation is the VMA size and the
3054          * consumed reservations are stored in the map. Hence, nothing
3055          * else has to be done for private mappings here
3056          */
3057         if (!vma || vma->vm_flags & VM_MAYSHARE)
3058                 region_add(&inode->i_mapping->private_list, from, to);
3059         return 0;
3060 out_err:
3061         if (vma)
3062                 resv_map_put(vma);
3063         return ret;
3064 }
3065 
3066 void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
3067 {
3068         struct hstate *h = hstate_inode(inode);
3069         long chg = region_truncate(&inode->i_mapping->private_list, offset);
3070         struct hugepage_subpool *spool = subpool_inode(inode);
3071 
3072         spin_lock(&inode->i_lock);
3073         inode->i_blocks -= (blocks_per_huge_page(h) * freed);
3074         spin_unlock(&inode->i_lock);
3075 
3076         hugepage_subpool_put_pages(spool, (chg - freed));
3077         hugetlb_acct_memory(h, -(chg - freed));
3078 }
3079 
3080 #ifdef CONFIG_MEMORY_FAILURE
3081 
3082 /* Should be called in hugetlb_lock */
3083 static int is_hugepage_on_freelist(struct page *hpage)
3084 {
3085         struct page *page;
3086         struct page *tmp;
3087         struct hstate *h = page_hstate(hpage);
3088         int nid = page_to_nid(hpage);
3089 
3090         list_for_each_entry_safe(page, tmp, &h->hugepage_freelists[nid], lru)
3091                 if (page == hpage)
3092                         return 1;
3093         return 0;
3094 }
3095 
3096 /*
3097  * This function is called from memory failure code.
3098  * Assume the caller holds page lock of the head page.
3099  */
3100 int dequeue_hwpoisoned_huge_page(struct page *hpage)
3101 {
3102         struct hstate *h = page_hstate(hpage);
3103         int nid = page_to_nid(hpage);
3104         int ret = -EBUSY;
3105 
3106         spin_lock(&hugetlb_lock);
3107         if (is_hugepage_on_freelist(hpage)) {
3108                 list_del(&hpage->lru);
3109                 set_page_refcounted(hpage);
3110                 h->free_huge_pages--;
3111                 h->free_huge_pages_node[nid]--;
3112                 ret = 0;
3113         }
3114         spin_unlock(&hugetlb_lock);
3115         return ret;
3116 }
3117 #endif
3118 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp