~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/hugetlb.c

Version: ~ [ linux-5.2-rc1 ] ~ [ linux-5.1.2 ] ~ [ linux-5.0.16 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.43 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.119 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.176 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.179 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.139 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.67 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Generic hugetlb support.
  3  * (C) Nadia Yvette Chambers, April 2004
  4  */
  5 #include <linux/list.h>
  6 #include <linux/init.h>
  7 #include <linux/module.h>
  8 #include <linux/mm.h>
  9 #include <linux/seq_file.h>
 10 #include <linux/sysctl.h>
 11 #include <linux/highmem.h>
 12 #include <linux/mmu_notifier.h>
 13 #include <linux/nodemask.h>
 14 #include <linux/pagemap.h>
 15 #include <linux/mempolicy.h>
 16 #include <linux/compiler.h>
 17 #include <linux/cpuset.h>
 18 #include <linux/mutex.h>
 19 #include <linux/bootmem.h>
 20 #include <linux/sysfs.h>
 21 #include <linux/slab.h>
 22 #include <linux/rmap.h>
 23 #include <linux/swap.h>
 24 #include <linux/swapops.h>
 25 #include <linux/page-isolation.h>
 26 #include <linux/jhash.h>
 27 
 28 #include <asm/page.h>
 29 #include <asm/pgtable.h>
 30 #include <asm/tlb.h>
 31 
 32 #include <linux/io.h>
 33 #include <linux/hugetlb.h>
 34 #include <linux/hugetlb_cgroup.h>
 35 #include <linux/node.h>
 36 #include "internal.h"
 37 
 38 int hugepages_treat_as_movable;
 39 
 40 int hugetlb_max_hstate __read_mostly;
 41 unsigned int default_hstate_idx;
 42 struct hstate hstates[HUGE_MAX_HSTATE];
 43 /*
 44  * Minimum page order among possible hugepage sizes, set to a proper value
 45  * at boot time.
 46  */
 47 static unsigned int minimum_order __read_mostly = UINT_MAX;
 48 
 49 __initdata LIST_HEAD(huge_boot_pages);
 50 
 51 /* for command line parsing */
 52 static struct hstate * __initdata parsed_hstate;
 53 static unsigned long __initdata default_hstate_max_huge_pages;
 54 static unsigned long __initdata default_hstate_size;
 55 
 56 /*
 57  * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
 58  * free_huge_pages, and surplus_huge_pages.
 59  */
 60 DEFINE_SPINLOCK(hugetlb_lock);
 61 
 62 /*
 63  * Serializes faults on the same logical page.  This is used to
 64  * prevent spurious OOMs when the hugepage pool is fully utilized.
 65  */
 66 static int num_fault_mutexes;
 67 static struct mutex *htlb_fault_mutex_table ____cacheline_aligned_in_smp;
 68 
 69 /* Forward declaration */
 70 static int hugetlb_acct_memory(struct hstate *h, long delta);
 71 
 72 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
 73 {
 74         bool free = (spool->count == 0) && (spool->used_hpages == 0);
 75 
 76         spin_unlock(&spool->lock);
 77 
 78         /* If no pages are used, and no other handles to the subpool
 79          * remain, give up any reservations mased on minimum size and
 80          * free the subpool */
 81         if (free) {
 82                 if (spool->min_hpages != -1)
 83                         hugetlb_acct_memory(spool->hstate,
 84                                                 -spool->min_hpages);
 85                 kfree(spool);
 86         }
 87 }
 88 
 89 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
 90                                                 long min_hpages)
 91 {
 92         struct hugepage_subpool *spool;
 93 
 94         spool = kzalloc(sizeof(*spool), GFP_KERNEL);
 95         if (!spool)
 96                 return NULL;
 97 
 98         spin_lock_init(&spool->lock);
 99         spool->count = 1;
100         spool->max_hpages = max_hpages;
101         spool->hstate = h;
102         spool->min_hpages = min_hpages;
103 
104         if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
105                 kfree(spool);
106                 return NULL;
107         }
108         spool->rsv_hpages = min_hpages;
109 
110         return spool;
111 }
112 
113 void hugepage_put_subpool(struct hugepage_subpool *spool)
114 {
115         spin_lock(&spool->lock);
116         BUG_ON(!spool->count);
117         spool->count--;
118         unlock_or_release_subpool(spool);
119 }
120 
121 /*
122  * Subpool accounting for allocating and reserving pages.
123  * Return -ENOMEM if there are not enough resources to satisfy the
124  * the request.  Otherwise, return the number of pages by which the
125  * global pools must be adjusted (upward).  The returned value may
126  * only be different than the passed value (delta) in the case where
127  * a subpool minimum size must be manitained.
128  */
129 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
130                                       long delta)
131 {
132         long ret = delta;
133 
134         if (!spool)
135                 return ret;
136 
137         spin_lock(&spool->lock);
138 
139         if (spool->max_hpages != -1) {          /* maximum size accounting */
140                 if ((spool->used_hpages + delta) <= spool->max_hpages)
141                         spool->used_hpages += delta;
142                 else {
143                         ret = -ENOMEM;
144                         goto unlock_ret;
145                 }
146         }
147 
148         if (spool->min_hpages != -1) {          /* minimum size accounting */
149                 if (delta > spool->rsv_hpages) {
150                         /*
151                          * Asking for more reserves than those already taken on
152                          * behalf of subpool.  Return difference.
153                          */
154                         ret = delta - spool->rsv_hpages;
155                         spool->rsv_hpages = 0;
156                 } else {
157                         ret = 0;        /* reserves already accounted for */
158                         spool->rsv_hpages -= delta;
159                 }
160         }
161 
162 unlock_ret:
163         spin_unlock(&spool->lock);
164         return ret;
165 }
166 
167 /*
168  * Subpool accounting for freeing and unreserving pages.
169  * Return the number of global page reservations that must be dropped.
170  * The return value may only be different than the passed value (delta)
171  * in the case where a subpool minimum size must be maintained.
172  */
173 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
174                                        long delta)
175 {
176         long ret = delta;
177 
178         if (!spool)
179                 return delta;
180 
181         spin_lock(&spool->lock);
182 
183         if (spool->max_hpages != -1)            /* maximum size accounting */
184                 spool->used_hpages -= delta;
185 
186         if (spool->min_hpages != -1) {          /* minimum size accounting */
187                 if (spool->rsv_hpages + delta <= spool->min_hpages)
188                         ret = 0;
189                 else
190                         ret = spool->rsv_hpages + delta - spool->min_hpages;
191 
192                 spool->rsv_hpages += delta;
193                 if (spool->rsv_hpages > spool->min_hpages)
194                         spool->rsv_hpages = spool->min_hpages;
195         }
196 
197         /*
198          * If hugetlbfs_put_super couldn't free spool due to an outstanding
199          * quota reference, free it now.
200          */
201         unlock_or_release_subpool(spool);
202 
203         return ret;
204 }
205 
206 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
207 {
208         return HUGETLBFS_SB(inode->i_sb)->spool;
209 }
210 
211 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
212 {
213         return subpool_inode(file_inode(vma->vm_file));
214 }
215 
216 /*
217  * Region tracking -- allows tracking of reservations and instantiated pages
218  *                    across the pages in a mapping.
219  *
220  * The region data structures are embedded into a resv_map and
221  * protected by a resv_map's lock
222  */
223 struct file_region {
224         struct list_head link;
225         long from;
226         long to;
227 };
228 
229 static long region_add(struct resv_map *resv, long f, long t)
230 {
231         struct list_head *head = &resv->regions;
232         struct file_region *rg, *nrg, *trg;
233 
234         spin_lock(&resv->lock);
235         /* Locate the region we are either in or before. */
236         list_for_each_entry(rg, head, link)
237                 if (f <= rg->to)
238                         break;
239 
240         /* Round our left edge to the current segment if it encloses us. */
241         if (f > rg->from)
242                 f = rg->from;
243 
244         /* Check for and consume any regions we now overlap with. */
245         nrg = rg;
246         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
247                 if (&rg->link == head)
248                         break;
249                 if (rg->from > t)
250                         break;
251 
252                 /* If this area reaches higher then extend our area to
253                  * include it completely.  If this is not the first area
254                  * which we intend to reuse, free it. */
255                 if (rg->to > t)
256                         t = rg->to;
257                 if (rg != nrg) {
258                         list_del(&rg->link);
259                         kfree(rg);
260                 }
261         }
262         nrg->from = f;
263         nrg->to = t;
264         spin_unlock(&resv->lock);
265         return 0;
266 }
267 
268 static long region_chg(struct resv_map *resv, long f, long t)
269 {
270         struct list_head *head = &resv->regions;
271         struct file_region *rg, *nrg = NULL;
272         long chg = 0;
273 
274 retry:
275         spin_lock(&resv->lock);
276         /* Locate the region we are before or in. */
277         list_for_each_entry(rg, head, link)
278                 if (f <= rg->to)
279                         break;
280 
281         /* If we are below the current region then a new region is required.
282          * Subtle, allocate a new region at the position but make it zero
283          * size such that we can guarantee to record the reservation. */
284         if (&rg->link == head || t < rg->from) {
285                 if (!nrg) {
286                         spin_unlock(&resv->lock);
287                         nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
288                         if (!nrg)
289                                 return -ENOMEM;
290 
291                         nrg->from = f;
292                         nrg->to   = f;
293                         INIT_LIST_HEAD(&nrg->link);
294                         goto retry;
295                 }
296 
297                 list_add(&nrg->link, rg->link.prev);
298                 chg = t - f;
299                 goto out_nrg;
300         }
301 
302         /* Round our left edge to the current segment if it encloses us. */
303         if (f > rg->from)
304                 f = rg->from;
305         chg = t - f;
306 
307         /* Check for and consume any regions we now overlap with. */
308         list_for_each_entry(rg, rg->link.prev, link) {
309                 if (&rg->link == head)
310                         break;
311                 if (rg->from > t)
312                         goto out;
313 
314                 /* We overlap with this area, if it extends further than
315                  * us then we must extend ourselves.  Account for its
316                  * existing reservation. */
317                 if (rg->to > t) {
318                         chg += rg->to - t;
319                         t = rg->to;
320                 }
321                 chg -= rg->to - rg->from;
322         }
323 
324 out:
325         spin_unlock(&resv->lock);
326         /*  We already know we raced and no longer need the new region */
327         kfree(nrg);
328         return chg;
329 out_nrg:
330         spin_unlock(&resv->lock);
331         return chg;
332 }
333 
334 static long region_truncate(struct resv_map *resv, long end)
335 {
336         struct list_head *head = &resv->regions;
337         struct file_region *rg, *trg;
338         long chg = 0;
339 
340         spin_lock(&resv->lock);
341         /* Locate the region we are either in or before. */
342         list_for_each_entry(rg, head, link)
343                 if (end <= rg->to)
344                         break;
345         if (&rg->link == head)
346                 goto out;
347 
348         /* If we are in the middle of a region then adjust it. */
349         if (end > rg->from) {
350                 chg = rg->to - end;
351                 rg->to = end;
352                 rg = list_entry(rg->link.next, typeof(*rg), link);
353         }
354 
355         /* Drop any remaining regions. */
356         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
357                 if (&rg->link == head)
358                         break;
359                 chg += rg->to - rg->from;
360                 list_del(&rg->link);
361                 kfree(rg);
362         }
363 
364 out:
365         spin_unlock(&resv->lock);
366         return chg;
367 }
368 
369 static long region_count(struct resv_map *resv, long f, long t)
370 {
371         struct list_head *head = &resv->regions;
372         struct file_region *rg;
373         long chg = 0;
374 
375         spin_lock(&resv->lock);
376         /* Locate each segment we overlap with, and count that overlap. */
377         list_for_each_entry(rg, head, link) {
378                 long seg_from;
379                 long seg_to;
380 
381                 if (rg->to <= f)
382                         continue;
383                 if (rg->from >= t)
384                         break;
385 
386                 seg_from = max(rg->from, f);
387                 seg_to = min(rg->to, t);
388 
389                 chg += seg_to - seg_from;
390         }
391         spin_unlock(&resv->lock);
392 
393         return chg;
394 }
395 
396 /*
397  * Convert the address within this vma to the page offset within
398  * the mapping, in pagecache page units; huge pages here.
399  */
400 static pgoff_t vma_hugecache_offset(struct hstate *h,
401                         struct vm_area_struct *vma, unsigned long address)
402 {
403         return ((address - vma->vm_start) >> huge_page_shift(h)) +
404                         (vma->vm_pgoff >> huge_page_order(h));
405 }
406 
407 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
408                                      unsigned long address)
409 {
410         return vma_hugecache_offset(hstate_vma(vma), vma, address);
411 }
412 
413 /*
414  * Return the size of the pages allocated when backing a VMA. In the majority
415  * cases this will be same size as used by the page table entries.
416  */
417 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
418 {
419         struct hstate *hstate;
420 
421         if (!is_vm_hugetlb_page(vma))
422                 return PAGE_SIZE;
423 
424         hstate = hstate_vma(vma);
425 
426         return 1UL << huge_page_shift(hstate);
427 }
428 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
429 
430 /*
431  * Return the page size being used by the MMU to back a VMA. In the majority
432  * of cases, the page size used by the kernel matches the MMU size. On
433  * architectures where it differs, an architecture-specific version of this
434  * function is required.
435  */
436 #ifndef vma_mmu_pagesize
437 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
438 {
439         return vma_kernel_pagesize(vma);
440 }
441 #endif
442 
443 /*
444  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
445  * bits of the reservation map pointer, which are always clear due to
446  * alignment.
447  */
448 #define HPAGE_RESV_OWNER    (1UL << 0)
449 #define HPAGE_RESV_UNMAPPED (1UL << 1)
450 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
451 
452 /*
453  * These helpers are used to track how many pages are reserved for
454  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
455  * is guaranteed to have their future faults succeed.
456  *
457  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
458  * the reserve counters are updated with the hugetlb_lock held. It is safe
459  * to reset the VMA at fork() time as it is not in use yet and there is no
460  * chance of the global counters getting corrupted as a result of the values.
461  *
462  * The private mapping reservation is represented in a subtly different
463  * manner to a shared mapping.  A shared mapping has a region map associated
464  * with the underlying file, this region map represents the backing file
465  * pages which have ever had a reservation assigned which this persists even
466  * after the page is instantiated.  A private mapping has a region map
467  * associated with the original mmap which is attached to all VMAs which
468  * reference it, this region map represents those offsets which have consumed
469  * reservation ie. where pages have been instantiated.
470  */
471 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
472 {
473         return (unsigned long)vma->vm_private_data;
474 }
475 
476 static void set_vma_private_data(struct vm_area_struct *vma,
477                                                         unsigned long value)
478 {
479         vma->vm_private_data = (void *)value;
480 }
481 
482 struct resv_map *resv_map_alloc(void)
483 {
484         struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
485         if (!resv_map)
486                 return NULL;
487 
488         kref_init(&resv_map->refs);
489         spin_lock_init(&resv_map->lock);
490         INIT_LIST_HEAD(&resv_map->regions);
491 
492         return resv_map;
493 }
494 
495 void resv_map_release(struct kref *ref)
496 {
497         struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
498 
499         /* Clear out any active regions before we release the map. */
500         region_truncate(resv_map, 0);
501         kfree(resv_map);
502 }
503 
504 static inline struct resv_map *inode_resv_map(struct inode *inode)
505 {
506         return inode->i_mapping->private_data;
507 }
508 
509 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
510 {
511         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
512         if (vma->vm_flags & VM_MAYSHARE) {
513                 struct address_space *mapping = vma->vm_file->f_mapping;
514                 struct inode *inode = mapping->host;
515 
516                 return inode_resv_map(inode);
517 
518         } else {
519                 return (struct resv_map *)(get_vma_private_data(vma) &
520                                                         ~HPAGE_RESV_MASK);
521         }
522 }
523 
524 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
525 {
526         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
527         VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
528 
529         set_vma_private_data(vma, (get_vma_private_data(vma) &
530                                 HPAGE_RESV_MASK) | (unsigned long)map);
531 }
532 
533 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
534 {
535         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
536         VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
537 
538         set_vma_private_data(vma, get_vma_private_data(vma) | flags);
539 }
540 
541 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
542 {
543         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
544 
545         return (get_vma_private_data(vma) & flag) != 0;
546 }
547 
548 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
549 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
550 {
551         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
552         if (!(vma->vm_flags & VM_MAYSHARE))
553                 vma->vm_private_data = (void *)0;
554 }
555 
556 /* Returns true if the VMA has associated reserve pages */
557 static int vma_has_reserves(struct vm_area_struct *vma, long chg)
558 {
559         if (vma->vm_flags & VM_NORESERVE) {
560                 /*
561                  * This address is already reserved by other process(chg == 0),
562                  * so, we should decrement reserved count. Without decrementing,
563                  * reserve count remains after releasing inode, because this
564                  * allocated page will go into page cache and is regarded as
565                  * coming from reserved pool in releasing step.  Currently, we
566                  * don't have any other solution to deal with this situation
567                  * properly, so add work-around here.
568                  */
569                 if (vma->vm_flags & VM_MAYSHARE && chg == 0)
570                         return 1;
571                 else
572                         return 0;
573         }
574 
575         /* Shared mappings always use reserves */
576         if (vma->vm_flags & VM_MAYSHARE)
577                 return 1;
578 
579         /*
580          * Only the process that called mmap() has reserves for
581          * private mappings.
582          */
583         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
584                 return 1;
585 
586         return 0;
587 }
588 
589 static void enqueue_huge_page(struct hstate *h, struct page *page)
590 {
591         int nid = page_to_nid(page);
592         list_move(&page->lru, &h->hugepage_freelists[nid]);
593         h->free_huge_pages++;
594         h->free_huge_pages_node[nid]++;
595 }
596 
597 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
598 {
599         struct page *page;
600 
601         list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
602                 if (!is_migrate_isolate_page(page))
603                         break;
604         /*
605          * if 'non-isolated free hugepage' not found on the list,
606          * the allocation fails.
607          */
608         if (&h->hugepage_freelists[nid] == &page->lru)
609                 return NULL;
610         list_move(&page->lru, &h->hugepage_activelist);
611         set_page_refcounted(page);
612         h->free_huge_pages--;
613         h->free_huge_pages_node[nid]--;
614         return page;
615 }
616 
617 /* Movability of hugepages depends on migration support. */
618 static inline gfp_t htlb_alloc_mask(struct hstate *h)
619 {
620         if (hugepages_treat_as_movable || hugepage_migration_supported(h))
621                 return GFP_HIGHUSER_MOVABLE;
622         else
623                 return GFP_HIGHUSER;
624 }
625 
626 static struct page *dequeue_huge_page_vma(struct hstate *h,
627                                 struct vm_area_struct *vma,
628                                 unsigned long address, int avoid_reserve,
629                                 long chg)
630 {
631         struct page *page = NULL;
632         struct mempolicy *mpol;
633         nodemask_t *nodemask;
634         struct zonelist *zonelist;
635         struct zone *zone;
636         struct zoneref *z;
637         unsigned int cpuset_mems_cookie;
638 
639         /*
640          * A child process with MAP_PRIVATE mappings created by their parent
641          * have no page reserves. This check ensures that reservations are
642          * not "stolen". The child may still get SIGKILLed
643          */
644         if (!vma_has_reserves(vma, chg) &&
645                         h->free_huge_pages - h->resv_huge_pages == 0)
646                 goto err;
647 
648         /* If reserves cannot be used, ensure enough pages are in the pool */
649         if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
650                 goto err;
651 
652 retry_cpuset:
653         cpuset_mems_cookie = read_mems_allowed_begin();
654         zonelist = huge_zonelist(vma, address,
655                                         htlb_alloc_mask(h), &mpol, &nodemask);
656 
657         for_each_zone_zonelist_nodemask(zone, z, zonelist,
658                                                 MAX_NR_ZONES - 1, nodemask) {
659                 if (cpuset_zone_allowed(zone, htlb_alloc_mask(h))) {
660                         page = dequeue_huge_page_node(h, zone_to_nid(zone));
661                         if (page) {
662                                 if (avoid_reserve)
663                                         break;
664                                 if (!vma_has_reserves(vma, chg))
665                                         break;
666 
667                                 SetPagePrivate(page);
668                                 h->resv_huge_pages--;
669                                 break;
670                         }
671                 }
672         }
673 
674         mpol_cond_put(mpol);
675         if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
676                 goto retry_cpuset;
677         return page;
678 
679 err:
680         return NULL;
681 }
682 
683 /*
684  * common helper functions for hstate_next_node_to_{alloc|free}.
685  * We may have allocated or freed a huge page based on a different
686  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
687  * be outside of *nodes_allowed.  Ensure that we use an allowed
688  * node for alloc or free.
689  */
690 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
691 {
692         nid = next_node(nid, *nodes_allowed);
693         if (nid == MAX_NUMNODES)
694                 nid = first_node(*nodes_allowed);
695         VM_BUG_ON(nid >= MAX_NUMNODES);
696 
697         return nid;
698 }
699 
700 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
701 {
702         if (!node_isset(nid, *nodes_allowed))
703                 nid = next_node_allowed(nid, nodes_allowed);
704         return nid;
705 }
706 
707 /*
708  * returns the previously saved node ["this node"] from which to
709  * allocate a persistent huge page for the pool and advance the
710  * next node from which to allocate, handling wrap at end of node
711  * mask.
712  */
713 static int hstate_next_node_to_alloc(struct hstate *h,
714                                         nodemask_t *nodes_allowed)
715 {
716         int nid;
717 
718         VM_BUG_ON(!nodes_allowed);
719 
720         nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
721         h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
722 
723         return nid;
724 }
725 
726 /*
727  * helper for free_pool_huge_page() - return the previously saved
728  * node ["this node"] from which to free a huge page.  Advance the
729  * next node id whether or not we find a free huge page to free so
730  * that the next attempt to free addresses the next node.
731  */
732 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
733 {
734         int nid;
735 
736         VM_BUG_ON(!nodes_allowed);
737 
738         nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
739         h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
740 
741         return nid;
742 }
743 
744 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)           \
745         for (nr_nodes = nodes_weight(*mask);                            \
746                 nr_nodes > 0 &&                                         \
747                 ((node = hstate_next_node_to_alloc(hs, mask)) || 1);    \
748                 nr_nodes--)
749 
750 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask)            \
751         for (nr_nodes = nodes_weight(*mask);                            \
752                 nr_nodes > 0 &&                                         \
753                 ((node = hstate_next_node_to_free(hs, mask)) || 1);     \
754                 nr_nodes--)
755 
756 #if defined(CONFIG_CMA) && defined(CONFIG_X86_64)
757 static void destroy_compound_gigantic_page(struct page *page,
758                                         unsigned int order)
759 {
760         int i;
761         int nr_pages = 1 << order;
762         struct page *p = page + 1;
763 
764         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
765                 __ClearPageTail(p);
766                 set_page_refcounted(p);
767                 p->first_page = NULL;
768         }
769 
770         set_compound_order(page, 0);
771         __ClearPageHead(page);
772 }
773 
774 static void free_gigantic_page(struct page *page, unsigned int order)
775 {
776         free_contig_range(page_to_pfn(page), 1 << order);
777 }
778 
779 static int __alloc_gigantic_page(unsigned long start_pfn,
780                                 unsigned long nr_pages)
781 {
782         unsigned long end_pfn = start_pfn + nr_pages;
783         return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
784 }
785 
786 static bool pfn_range_valid_gigantic(unsigned long start_pfn,
787                                 unsigned long nr_pages)
788 {
789         unsigned long i, end_pfn = start_pfn + nr_pages;
790         struct page *page;
791 
792         for (i = start_pfn; i < end_pfn; i++) {
793                 if (!pfn_valid(i))
794                         return false;
795 
796                 page = pfn_to_page(i);
797 
798                 if (PageReserved(page))
799                         return false;
800 
801                 if (page_count(page) > 0)
802                         return false;
803 
804                 if (PageHuge(page))
805                         return false;
806         }
807 
808         return true;
809 }
810 
811 static bool zone_spans_last_pfn(const struct zone *zone,
812                         unsigned long start_pfn, unsigned long nr_pages)
813 {
814         unsigned long last_pfn = start_pfn + nr_pages - 1;
815         return zone_spans_pfn(zone, last_pfn);
816 }
817 
818 static struct page *alloc_gigantic_page(int nid, unsigned int order)
819 {
820         unsigned long nr_pages = 1 << order;
821         unsigned long ret, pfn, flags;
822         struct zone *z;
823 
824         z = NODE_DATA(nid)->node_zones;
825         for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) {
826                 spin_lock_irqsave(&z->lock, flags);
827 
828                 pfn = ALIGN(z->zone_start_pfn, nr_pages);
829                 while (zone_spans_last_pfn(z, pfn, nr_pages)) {
830                         if (pfn_range_valid_gigantic(pfn, nr_pages)) {
831                                 /*
832                                  * We release the zone lock here because
833                                  * alloc_contig_range() will also lock the zone
834                                  * at some point. If there's an allocation
835                                  * spinning on this lock, it may win the race
836                                  * and cause alloc_contig_range() to fail...
837                                  */
838                                 spin_unlock_irqrestore(&z->lock, flags);
839                                 ret = __alloc_gigantic_page(pfn, nr_pages);
840                                 if (!ret)
841                                         return pfn_to_page(pfn);
842                                 spin_lock_irqsave(&z->lock, flags);
843                         }
844                         pfn += nr_pages;
845                 }
846 
847                 spin_unlock_irqrestore(&z->lock, flags);
848         }
849 
850         return NULL;
851 }
852 
853 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
854 static void prep_compound_gigantic_page(struct page *page, unsigned int order);
855 
856 static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid)
857 {
858         struct page *page;
859 
860         page = alloc_gigantic_page(nid, huge_page_order(h));
861         if (page) {
862                 prep_compound_gigantic_page(page, huge_page_order(h));
863                 prep_new_huge_page(h, page, nid);
864         }
865 
866         return page;
867 }
868 
869 static int alloc_fresh_gigantic_page(struct hstate *h,
870                                 nodemask_t *nodes_allowed)
871 {
872         struct page *page = NULL;
873         int nr_nodes, node;
874 
875         for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
876                 page = alloc_fresh_gigantic_page_node(h, node);
877                 if (page)
878                         return 1;
879         }
880 
881         return 0;
882 }
883 
884 static inline bool gigantic_page_supported(void) { return true; }
885 #else
886 static inline bool gigantic_page_supported(void) { return false; }
887 static inline void free_gigantic_page(struct page *page, unsigned int order) { }
888 static inline void destroy_compound_gigantic_page(struct page *page,
889                                                 unsigned int order) { }
890 static inline int alloc_fresh_gigantic_page(struct hstate *h,
891                                         nodemask_t *nodes_allowed) { return 0; }
892 #endif
893 
894 static void update_and_free_page(struct hstate *h, struct page *page)
895 {
896         int i;
897 
898         if (hstate_is_gigantic(h) && !gigantic_page_supported())
899                 return;
900 
901         h->nr_huge_pages--;
902         h->nr_huge_pages_node[page_to_nid(page)]--;
903         for (i = 0; i < pages_per_huge_page(h); i++) {
904                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
905                                 1 << PG_referenced | 1 << PG_dirty |
906                                 1 << PG_active | 1 << PG_private |
907                                 1 << PG_writeback);
908         }
909         VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
910         set_compound_page_dtor(page, NULL);
911         set_page_refcounted(page);
912         if (hstate_is_gigantic(h)) {
913                 destroy_compound_gigantic_page(page, huge_page_order(h));
914                 free_gigantic_page(page, huge_page_order(h));
915         } else {
916                 arch_release_hugepage(page);
917                 __free_pages(page, huge_page_order(h));
918         }
919 }
920 
921 struct hstate *size_to_hstate(unsigned long size)
922 {
923         struct hstate *h;
924 
925         for_each_hstate(h) {
926                 if (huge_page_size(h) == size)
927                         return h;
928         }
929         return NULL;
930 }
931 
932 /*
933  * Test to determine whether the hugepage is "active/in-use" (i.e. being linked
934  * to hstate->hugepage_activelist.)
935  *
936  * This function can be called for tail pages, but never returns true for them.
937  */
938 bool page_huge_active(struct page *page)
939 {
940         VM_BUG_ON_PAGE(!PageHuge(page), page);
941         return PageHead(page) && PagePrivate(&page[1]);
942 }
943 
944 /* never called for tail page */
945 static void set_page_huge_active(struct page *page)
946 {
947         VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
948         SetPagePrivate(&page[1]);
949 }
950 
951 static void clear_page_huge_active(struct page *page)
952 {
953         VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
954         ClearPagePrivate(&page[1]);
955 }
956 
957 void free_huge_page(struct page *page)
958 {
959         /*
960          * Can't pass hstate in here because it is called from the
961          * compound page destructor.
962          */
963         struct hstate *h = page_hstate(page);
964         int nid = page_to_nid(page);
965         struct hugepage_subpool *spool =
966                 (struct hugepage_subpool *)page_private(page);
967         bool restore_reserve;
968 
969         set_page_private(page, 0);
970         page->mapping = NULL;
971         BUG_ON(page_count(page));
972         BUG_ON(page_mapcount(page));
973         restore_reserve = PagePrivate(page);
974         ClearPagePrivate(page);
975 
976         /*
977          * A return code of zero implies that the subpool will be under its
978          * minimum size if the reservation is not restored after page is free.
979          * Therefore, force restore_reserve operation.
980          */
981         if (hugepage_subpool_put_pages(spool, 1) == 0)
982                 restore_reserve = true;
983 
984         spin_lock(&hugetlb_lock);
985         clear_page_huge_active(page);
986         hugetlb_cgroup_uncharge_page(hstate_index(h),
987                                      pages_per_huge_page(h), page);
988         if (restore_reserve)
989                 h->resv_huge_pages++;
990 
991         if (h->surplus_huge_pages_node[nid]) {
992                 /* remove the page from active list */
993                 list_del(&page->lru);
994                 update_and_free_page(h, page);
995                 h->surplus_huge_pages--;
996                 h->surplus_huge_pages_node[nid]--;
997         } else {
998                 arch_clear_hugepage_flags(page);
999                 enqueue_huge_page(h, page);
1000         }
1001         spin_unlock(&hugetlb_lock);
1002 }
1003 
1004 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
1005 {
1006         INIT_LIST_HEAD(&page->lru);
1007         set_compound_page_dtor(page, free_huge_page);
1008         spin_lock(&hugetlb_lock);
1009         set_hugetlb_cgroup(page, NULL);
1010         h->nr_huge_pages++;
1011         h->nr_huge_pages_node[nid]++;
1012         spin_unlock(&hugetlb_lock);
1013         put_page(page); /* free it into the hugepage allocator */
1014 }
1015 
1016 static void prep_compound_gigantic_page(struct page *page, unsigned int order)
1017 {
1018         int i;
1019         int nr_pages = 1 << order;
1020         struct page *p = page + 1;
1021 
1022         /* we rely on prep_new_huge_page to set the destructor */
1023         set_compound_order(page, order);
1024         __SetPageHead(page);
1025         __ClearPageReserved(page);
1026         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1027                 /*
1028                  * For gigantic hugepages allocated through bootmem at
1029                  * boot, it's safer to be consistent with the not-gigantic
1030                  * hugepages and clear the PG_reserved bit from all tail pages
1031                  * too.  Otherwse drivers using get_user_pages() to access tail
1032                  * pages may get the reference counting wrong if they see
1033                  * PG_reserved set on a tail page (despite the head page not
1034                  * having PG_reserved set).  Enforcing this consistency between
1035                  * head and tail pages allows drivers to optimize away a check
1036                  * on the head page when they need know if put_page() is needed
1037                  * after get_user_pages().
1038                  */
1039                 __ClearPageReserved(p);
1040                 set_page_count(p, 0);
1041                 p->first_page = page;
1042                 /* Make sure p->first_page is always valid for PageTail() */
1043                 smp_wmb();
1044                 __SetPageTail(p);
1045         }
1046 }
1047 
1048 /*
1049  * PageHuge() only returns true for hugetlbfs pages, but not for normal or
1050  * transparent huge pages.  See the PageTransHuge() documentation for more
1051  * details.
1052  */
1053 int PageHuge(struct page *page)
1054 {
1055         if (!PageCompound(page))
1056                 return 0;
1057 
1058         page = compound_head(page);
1059         return get_compound_page_dtor(page) == free_huge_page;
1060 }
1061 EXPORT_SYMBOL_GPL(PageHuge);
1062 
1063 /*
1064  * PageHeadHuge() only returns true for hugetlbfs head page, but not for
1065  * normal or transparent huge pages.
1066  */
1067 int PageHeadHuge(struct page *page_head)
1068 {
1069         if (!PageHead(page_head))
1070                 return 0;
1071 
1072         return get_compound_page_dtor(page_head) == free_huge_page;
1073 }
1074 
1075 pgoff_t __basepage_index(struct page *page)
1076 {
1077         struct page *page_head = compound_head(page);
1078         pgoff_t index = page_index(page_head);
1079         unsigned long compound_idx;
1080 
1081         if (!PageHuge(page_head))
1082                 return page_index(page);
1083 
1084         if (compound_order(page_head) >= MAX_ORDER)
1085                 compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
1086         else
1087                 compound_idx = page - page_head;
1088 
1089         return (index << compound_order(page_head)) + compound_idx;
1090 }
1091 
1092 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
1093 {
1094         struct page *page;
1095 
1096         page = alloc_pages_exact_node(nid,
1097                 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
1098                                                 __GFP_REPEAT|__GFP_NOWARN,
1099                 huge_page_order(h));
1100         if (page) {
1101                 if (arch_prepare_hugepage(page)) {
1102                         __free_pages(page, huge_page_order(h));
1103                         return NULL;
1104                 }
1105                 prep_new_huge_page(h, page, nid);
1106         }
1107 
1108         return page;
1109 }
1110 
1111 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
1112 {
1113         struct page *page;
1114         int nr_nodes, node;
1115         int ret = 0;
1116 
1117         for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1118                 page = alloc_fresh_huge_page_node(h, node);
1119                 if (page) {
1120                         ret = 1;
1121                         break;
1122                 }
1123         }
1124 
1125         if (ret)
1126                 count_vm_event(HTLB_BUDDY_PGALLOC);
1127         else
1128                 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1129 
1130         return ret;
1131 }
1132 
1133 /*
1134  * Free huge page from pool from next node to free.
1135  * Attempt to keep persistent huge pages more or less
1136  * balanced over allowed nodes.
1137  * Called with hugetlb_lock locked.
1138  */
1139 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1140                                                          bool acct_surplus)
1141 {
1142         int nr_nodes, node;
1143         int ret = 0;
1144 
1145         for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1146                 /*
1147                  * If we're returning unused surplus pages, only examine
1148                  * nodes with surplus pages.
1149                  */
1150                 if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
1151                     !list_empty(&h->hugepage_freelists[node])) {
1152                         struct page *page =
1153                                 list_entry(h->hugepage_freelists[node].next,
1154                                           struct page, lru);
1155                         list_del(&page->lru);
1156                         h->free_huge_pages--;
1157                         h->free_huge_pages_node[node]--;
1158                         if (acct_surplus) {
1159                                 h->surplus_huge_pages--;
1160                                 h->surplus_huge_pages_node[node]--;
1161                         }
1162                         update_and_free_page(h, page);
1163                         ret = 1;
1164                         break;
1165                 }
1166         }
1167 
1168         return ret;
1169 }
1170 
1171 /*
1172  * Dissolve a given free hugepage into free buddy pages. This function does
1173  * nothing for in-use (including surplus) hugepages.
1174  */
1175 static void dissolve_free_huge_page(struct page *page)
1176 {
1177         spin_lock(&hugetlb_lock);
1178         if (PageHuge(page) && !page_count(page)) {
1179                 struct hstate *h = page_hstate(page);
1180                 int nid = page_to_nid(page);
1181                 list_del(&page->lru);
1182                 h->free_huge_pages--;
1183                 h->free_huge_pages_node[nid]--;
1184                 update_and_free_page(h, page);
1185         }
1186         spin_unlock(&hugetlb_lock);
1187 }
1188 
1189 /*
1190  * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
1191  * make specified memory blocks removable from the system.
1192  * Note that start_pfn should aligned with (minimum) hugepage size.
1193  */
1194 void dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
1195 {
1196         unsigned long pfn;
1197 
1198         if (!hugepages_supported())
1199                 return;
1200 
1201         VM_BUG_ON(!IS_ALIGNED(start_pfn, 1 << minimum_order));
1202         for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order)
1203                 dissolve_free_huge_page(pfn_to_page(pfn));
1204 }
1205 
1206 static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)
1207 {
1208         struct page *page;
1209         unsigned int r_nid;
1210 
1211         if (hstate_is_gigantic(h))
1212                 return NULL;
1213 
1214         /*
1215          * Assume we will successfully allocate the surplus page to
1216          * prevent racing processes from causing the surplus to exceed
1217          * overcommit
1218          *
1219          * This however introduces a different race, where a process B
1220          * tries to grow the static hugepage pool while alloc_pages() is
1221          * called by process A. B will only examine the per-node
1222          * counters in determining if surplus huge pages can be
1223          * converted to normal huge pages in adjust_pool_surplus(). A
1224          * won't be able to increment the per-node counter, until the
1225          * lock is dropped by B, but B doesn't drop hugetlb_lock until
1226          * no more huge pages can be converted from surplus to normal
1227          * state (and doesn't try to convert again). Thus, we have a
1228          * case where a surplus huge page exists, the pool is grown, and
1229          * the surplus huge page still exists after, even though it
1230          * should just have been converted to a normal huge page. This
1231          * does not leak memory, though, as the hugepage will be freed
1232          * once it is out of use. It also does not allow the counters to
1233          * go out of whack in adjust_pool_surplus() as we don't modify
1234          * the node values until we've gotten the hugepage and only the
1235          * per-node value is checked there.
1236          */
1237         spin_lock(&hugetlb_lock);
1238         if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
1239                 spin_unlock(&hugetlb_lock);
1240                 return NULL;
1241         } else {
1242                 h->nr_huge_pages++;
1243                 h->surplus_huge_pages++;
1244         }
1245         spin_unlock(&hugetlb_lock);
1246 
1247         if (nid == NUMA_NO_NODE)
1248                 page = alloc_pages(htlb_alloc_mask(h)|__GFP_COMP|
1249                                    __GFP_REPEAT|__GFP_NOWARN,
1250                                    huge_page_order(h));
1251         else
1252                 page = alloc_pages_exact_node(nid,
1253                         htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
1254                         __GFP_REPEAT|__GFP_NOWARN, huge_page_order(h));
1255 
1256         if (page && arch_prepare_hugepage(page)) {
1257                 __free_pages(page, huge_page_order(h));
1258                 page = NULL;
1259         }
1260 
1261         spin_lock(&hugetlb_lock);
1262         if (page) {
1263                 INIT_LIST_HEAD(&page->lru);
1264                 r_nid = page_to_nid(page);
1265                 set_compound_page_dtor(page, free_huge_page);
1266                 set_hugetlb_cgroup(page, NULL);
1267                 /*
1268                  * We incremented the global counters already
1269                  */
1270                 h->nr_huge_pages_node[r_nid]++;
1271                 h->surplus_huge_pages_node[r_nid]++;
1272                 __count_vm_event(HTLB_BUDDY_PGALLOC);
1273         } else {
1274                 h->nr_huge_pages--;
1275                 h->surplus_huge_pages--;
1276                 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1277         }
1278         spin_unlock(&hugetlb_lock);
1279 
1280         return page;
1281 }
1282 
1283 /*
1284  * This allocation function is useful in the context where vma is irrelevant.
1285  * E.g. soft-offlining uses this function because it only cares physical
1286  * address of error page.
1287  */
1288 struct page *alloc_huge_page_node(struct hstate *h, int nid)
1289 {
1290         struct page *page = NULL;
1291 
1292         spin_lock(&hugetlb_lock);
1293         if (h->free_huge_pages - h->resv_huge_pages > 0)
1294                 page = dequeue_huge_page_node(h, nid);
1295         spin_unlock(&hugetlb_lock);
1296 
1297         if (!page)
1298                 page = alloc_buddy_huge_page(h, nid);
1299 
1300         return page;
1301 }
1302 
1303 /*
1304  * Increase the hugetlb pool such that it can accommodate a reservation
1305  * of size 'delta'.
1306  */
1307 static int gather_surplus_pages(struct hstate *h, int delta)
1308 {
1309         struct list_head surplus_list;
1310         struct page *page, *tmp;
1311         int ret, i;
1312         int needed, allocated;
1313         bool alloc_ok = true;
1314 
1315         needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
1316         if (needed <= 0) {
1317                 h->resv_huge_pages += delta;
1318                 return 0;
1319         }
1320 
1321         allocated = 0;
1322         INIT_LIST_HEAD(&surplus_list);
1323 
1324         ret = -ENOMEM;
1325 retry:
1326         spin_unlock(&hugetlb_lock);
1327         for (i = 0; i < needed; i++) {
1328                 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1329                 if (!page) {
1330                         alloc_ok = false;
1331                         break;
1332                 }
1333                 list_add(&page->lru, &surplus_list);
1334         }
1335         allocated += i;
1336 
1337         /*
1338          * After retaking hugetlb_lock, we need to recalculate 'needed'
1339          * because either resv_huge_pages or free_huge_pages may have changed.
1340          */
1341         spin_lock(&hugetlb_lock);
1342         needed = (h->resv_huge_pages + delta) -
1343                         (h->free_huge_pages + allocated);
1344         if (needed > 0) {
1345                 if (alloc_ok)
1346                         goto retry;
1347                 /*
1348                  * We were not able to allocate enough pages to
1349                  * satisfy the entire reservation so we free what
1350                  * we've allocated so far.
1351                  */
1352                 goto free;
1353         }
1354         /*
1355          * The surplus_list now contains _at_least_ the number of extra pages
1356          * needed to accommodate the reservation.  Add the appropriate number
1357          * of pages to the hugetlb pool and free the extras back to the buddy
1358          * allocator.  Commit the entire reservation here to prevent another
1359          * process from stealing the pages as they are added to the pool but
1360          * before they are reserved.
1361          */
1362         needed += allocated;
1363         h->resv_huge_pages += delta;
1364         ret = 0;
1365 
1366         /* Free the needed pages to the hugetlb pool */
1367         list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1368                 if ((--needed) < 0)
1369                         break;
1370                 /*
1371                  * This page is now managed by the hugetlb allocator and has
1372                  * no users -- drop the buddy allocator's reference.
1373                  */
1374                 put_page_testzero(page);
1375                 VM_BUG_ON_PAGE(page_count(page), page);
1376                 enqueue_huge_page(h, page);
1377         }
1378 free:
1379         spin_unlock(&hugetlb_lock);
1380 
1381         /* Free unnecessary surplus pages to the buddy allocator */
1382         list_for_each_entry_safe(page, tmp, &surplus_list, lru)
1383                 put_page(page);
1384         spin_lock(&hugetlb_lock);
1385 
1386         return ret;
1387 }
1388 
1389 /*
1390  * This routine has two main purposes:
1391  * 1) Decrement the reservation count (resv_huge_pages) by the value passed
1392  *    in unused_resv_pages.  This corresponds to the prior adjustments made
1393  *    to the associated reservation map.
1394  * 2) Free any unused surplus pages that may have been allocated to satisfy
1395  *    the reservation.  As many as unused_resv_pages may be freed.
1396  *
1397  * Called with hugetlb_lock held.  However, the lock could be dropped (and
1398  * reacquired) during calls to cond_resched_lock.  Whenever dropping the lock,
1399  * we must make sure nobody else can claim pages we are in the process of
1400  * freeing.  Do this by ensuring resv_huge_page always is greater than the
1401  * number of huge pages we plan to free when dropping the lock.
1402  */
1403 static void return_unused_surplus_pages(struct hstate *h,
1404                                         unsigned long unused_resv_pages)
1405 {
1406         unsigned long nr_pages;
1407 
1408         /* Cannot return gigantic pages currently */
1409         if (hstate_is_gigantic(h))
1410                 goto out;
1411 
1412         /*
1413          * Part (or even all) of the reservation could have been backed
1414          * by pre-allocated pages. Only free surplus pages.
1415          */
1416         nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1417 
1418         /*
1419          * We want to release as many surplus pages as possible, spread
1420          * evenly across all nodes with memory. Iterate across these nodes
1421          * until we can no longer free unreserved surplus pages. This occurs
1422          * when the nodes with surplus pages have no free pages.
1423          * free_pool_huge_page() will balance the the freed pages across the
1424          * on-line nodes with memory and will handle the hstate accounting.
1425          *
1426          * Note that we decrement resv_huge_pages as we free the pages.  If
1427          * we drop the lock, resv_huge_pages will still be sufficiently large
1428          * to cover subsequent pages we may free.
1429          */
1430         while (nr_pages--) {
1431                 h->resv_huge_pages--;
1432                 unused_resv_pages--;
1433                 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))
1434                         goto out;
1435                 cond_resched_lock(&hugetlb_lock);
1436         }
1437 
1438 out:
1439         /* Fully uncommit the reservation */
1440         h->resv_huge_pages -= unused_resv_pages;
1441 }
1442 
1443 /*
1444  * Determine if the huge page at addr within the vma has an associated
1445  * reservation.  Where it does not we will need to logically increase
1446  * reservation and actually increase subpool usage before an allocation
1447  * can occur.  Where any new reservation would be required the
1448  * reservation change is prepared, but not committed.  Once the page
1449  * has been allocated from the subpool and instantiated the change should
1450  * be committed via vma_commit_reservation.  No action is required on
1451  * failure.
1452  */
1453 static long vma_needs_reservation(struct hstate *h,
1454                         struct vm_area_struct *vma, unsigned long addr)
1455 {
1456         struct resv_map *resv;
1457         pgoff_t idx;
1458         long chg;
1459 
1460         resv = vma_resv_map(vma);
1461         if (!resv)
1462                 return 1;
1463 
1464         idx = vma_hugecache_offset(h, vma, addr);
1465         chg = region_chg(resv, idx, idx + 1);
1466 
1467         if (vma->vm_flags & VM_MAYSHARE)
1468                 return chg;
1469         else
1470                 return chg < 0 ? chg : 0;
1471 }
1472 static void vma_commit_reservation(struct hstate *h,
1473                         struct vm_area_struct *vma, unsigned long addr)
1474 {
1475         struct resv_map *resv;
1476         pgoff_t idx;
1477 
1478         resv = vma_resv_map(vma);
1479         if (!resv)
1480                 return;
1481 
1482         idx = vma_hugecache_offset(h, vma, addr);
1483         region_add(resv, idx, idx + 1);
1484 }
1485 
1486 static struct page *alloc_huge_page(struct vm_area_struct *vma,
1487                                     unsigned long addr, int avoid_reserve)
1488 {
1489         struct hugepage_subpool *spool = subpool_vma(vma);
1490         struct hstate *h = hstate_vma(vma);
1491         struct page *page;
1492         long chg;
1493         int ret, idx;
1494         struct hugetlb_cgroup *h_cg;
1495 
1496         idx = hstate_index(h);
1497         /*
1498          * Processes that did not create the mapping will have no
1499          * reserves and will not have accounted against subpool
1500          * limit. Check that the subpool limit can be made before
1501          * satisfying the allocation MAP_NORESERVE mappings may also
1502          * need pages and subpool limit allocated allocated if no reserve
1503          * mapping overlaps.
1504          */
1505         chg = vma_needs_reservation(h, vma, addr);
1506         if (chg < 0)
1507                 return ERR_PTR(-ENOMEM);
1508         if (chg || avoid_reserve)
1509                 if (hugepage_subpool_get_pages(spool, 1) < 0)
1510                         return ERR_PTR(-ENOSPC);
1511 
1512         ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
1513         if (ret)
1514                 goto out_subpool_put;
1515 
1516         spin_lock(&hugetlb_lock);
1517         page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, chg);
1518         if (!page) {
1519                 spin_unlock(&hugetlb_lock);
1520                 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1521                 if (!page)
1522                         goto out_uncharge_cgroup;
1523 
1524                 spin_lock(&hugetlb_lock);
1525                 list_move(&page->lru, &h->hugepage_activelist);
1526                 /* Fall through */
1527         }
1528         hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
1529         spin_unlock(&hugetlb_lock);
1530 
1531         set_page_private(page, (unsigned long)spool);
1532 
1533         vma_commit_reservation(h, vma, addr);
1534         return page;
1535 
1536 out_uncharge_cgroup:
1537         hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
1538 out_subpool_put:
1539         if (chg || avoid_reserve)
1540                 hugepage_subpool_put_pages(spool, 1);
1541         return ERR_PTR(-ENOSPC);
1542 }
1543 
1544 /*
1545  * alloc_huge_page()'s wrapper which simply returns the page if allocation
1546  * succeeds, otherwise NULL. This function is called from new_vma_page(),
1547  * where no ERR_VALUE is expected to be returned.
1548  */
1549 struct page *alloc_huge_page_noerr(struct vm_area_struct *vma,
1550                                 unsigned long addr, int avoid_reserve)
1551 {
1552         struct page *page = alloc_huge_page(vma, addr, avoid_reserve);
1553         if (IS_ERR(page))
1554                 page = NULL;
1555         return page;
1556 }
1557 
1558 int __weak alloc_bootmem_huge_page(struct hstate *h)
1559 {
1560         struct huge_bootmem_page *m;
1561         int nr_nodes, node;
1562 
1563         for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
1564                 void *addr;
1565 
1566                 addr = memblock_virt_alloc_try_nid_nopanic(
1567                                 huge_page_size(h), huge_page_size(h),
1568                                 0, BOOTMEM_ALLOC_ACCESSIBLE, node);
1569                 if (addr) {
1570                         /*
1571                          * Use the beginning of the huge page to store the
1572                          * huge_bootmem_page struct (until gather_bootmem
1573                          * puts them into the mem_map).
1574                          */
1575                         m = addr;
1576                         goto found;
1577                 }
1578         }
1579         return 0;
1580 
1581 found:
1582         BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h)));
1583         /* Put them into a private list first because mem_map is not up yet */
1584         list_add(&m->list, &huge_boot_pages);
1585         m->hstate = h;
1586         return 1;
1587 }
1588 
1589 static void __init prep_compound_huge_page(struct page *page,
1590                 unsigned int order)
1591 {
1592         if (unlikely(order > (MAX_ORDER - 1)))
1593                 prep_compound_gigantic_page(page, order);
1594         else
1595                 prep_compound_page(page, order);
1596 }
1597 
1598 /* Put bootmem huge pages into the standard lists after mem_map is up */
1599 static void __init gather_bootmem_prealloc(void)
1600 {
1601         struct huge_bootmem_page *m;
1602 
1603         list_for_each_entry(m, &huge_boot_pages, list) {
1604                 struct hstate *h = m->hstate;
1605                 struct page *page;
1606 
1607 #ifdef CONFIG_HIGHMEM
1608                 page = pfn_to_page(m->phys >> PAGE_SHIFT);
1609                 memblock_free_late(__pa(m),
1610                                    sizeof(struct huge_bootmem_page));
1611 #else
1612                 page = virt_to_page(m);
1613 #endif
1614                 WARN_ON(page_count(page) != 1);
1615                 prep_compound_huge_page(page, h->order);
1616                 WARN_ON(PageReserved(page));
1617                 prep_new_huge_page(h, page, page_to_nid(page));
1618                 /*
1619                  * If we had gigantic hugepages allocated at boot time, we need
1620                  * to restore the 'stolen' pages to totalram_pages in order to
1621                  * fix confusing memory reports from free(1) and another
1622                  * side-effects, like CommitLimit going negative.
1623                  */
1624                 if (hstate_is_gigantic(h))
1625                         adjust_managed_page_count(page, 1 << h->order);
1626         }
1627 }
1628 
1629 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
1630 {
1631         unsigned long i;
1632 
1633         for (i = 0; i < h->max_huge_pages; ++i) {
1634                 if (hstate_is_gigantic(h)) {
1635                         if (!alloc_bootmem_huge_page(h))
1636                                 break;
1637                 } else if (!alloc_fresh_huge_page(h,
1638                                          &node_states[N_MEMORY]))
1639                         break;
1640         }
1641         h->max_huge_pages = i;
1642 }
1643 
1644 static void __init hugetlb_init_hstates(void)
1645 {
1646         struct hstate *h;
1647 
1648         for_each_hstate(h) {
1649                 if (minimum_order > huge_page_order(h))
1650                         minimum_order = huge_page_order(h);
1651 
1652                 /* oversize hugepages were init'ed in early boot */
1653                 if (!hstate_is_gigantic(h))
1654                         hugetlb_hstate_alloc_pages(h);
1655         }
1656         VM_BUG_ON(minimum_order == UINT_MAX);
1657 }
1658 
1659 static char * __init memfmt(char *buf, unsigned long n)
1660 {
1661         if (n >= (1UL << 30))
1662                 sprintf(buf, "%lu GB", n >> 30);
1663         else if (n >= (1UL << 20))
1664                 sprintf(buf, "%lu MB", n >> 20);
1665         else
1666                 sprintf(buf, "%lu KB", n >> 10);
1667         return buf;
1668 }
1669 
1670 static void __init report_hugepages(void)
1671 {
1672         struct hstate *h;
1673 
1674         for_each_hstate(h) {
1675                 char buf[32];
1676                 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
1677                         memfmt(buf, huge_page_size(h)),
1678                         h->free_huge_pages);
1679         }
1680 }
1681 
1682 #ifdef CONFIG_HIGHMEM
1683 static void try_to_free_low(struct hstate *h, unsigned long count,
1684                                                 nodemask_t *nodes_allowed)
1685 {
1686         int i;
1687 
1688         if (hstate_is_gigantic(h))
1689                 return;
1690 
1691         for_each_node_mask(i, *nodes_allowed) {
1692                 struct page *page, *next;
1693                 struct list_head *freel = &h->hugepage_freelists[i];
1694                 list_for_each_entry_safe(page, next, freel, lru) {
1695                         if (count >= h->nr_huge_pages)
1696                                 return;
1697                         if (PageHighMem(page))
1698                                 continue;
1699                         list_del(&page->lru);
1700                         update_and_free_page(h, page);
1701                         h->free_huge_pages--;
1702                         h->free_huge_pages_node[page_to_nid(page)]--;
1703                 }
1704         }
1705 }
1706 #else
1707 static inline void try_to_free_low(struct hstate *h, unsigned long count,
1708                                                 nodemask_t *nodes_allowed)
1709 {
1710 }
1711 #endif
1712 
1713 /*
1714  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
1715  * balanced by operating on them in a round-robin fashion.
1716  * Returns 1 if an adjustment was made.
1717  */
1718 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
1719                                 int delta)
1720 {
1721         int nr_nodes, node;
1722 
1723         VM_BUG_ON(delta != -1 && delta != 1);
1724 
1725         if (delta < 0) {
1726                 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1727                         if (h->surplus_huge_pages_node[node])
1728                                 goto found;
1729                 }
1730         } else {
1731                 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1732                         if (h->surplus_huge_pages_node[node] <
1733                                         h->nr_huge_pages_node[node])
1734                                 goto found;
1735                 }
1736         }
1737         return 0;
1738 
1739 found:
1740         h->surplus_huge_pages += delta;
1741         h->surplus_huge_pages_node[node] += delta;
1742         return 1;
1743 }
1744 
1745 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
1746 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
1747                                                 nodemask_t *nodes_allowed)
1748 {
1749         unsigned long min_count, ret;
1750 
1751         if (hstate_is_gigantic(h) && !gigantic_page_supported())
1752                 return h->max_huge_pages;
1753 
1754         /*
1755          * Increase the pool size
1756          * First take pages out of surplus state.  Then make up the
1757          * remaining difference by allocating fresh huge pages.
1758          *
1759          * We might race with alloc_buddy_huge_page() here and be unable
1760          * to convert a surplus huge page to a normal huge page. That is
1761          * not critical, though, it just means the overall size of the
1762          * pool might be one hugepage larger than it needs to be, but
1763          * within all the constraints specified by the sysctls.
1764          */
1765         spin_lock(&hugetlb_lock);
1766         while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
1767                 if (!adjust_pool_surplus(h, nodes_allowed, -1))
1768                         break;
1769         }
1770 
1771         while (count > persistent_huge_pages(h)) {
1772                 /*
1773                  * If this allocation races such that we no longer need the
1774                  * page, free_huge_page will handle it by freeing the page
1775                  * and reducing the surplus.
1776                  */
1777                 spin_unlock(&hugetlb_lock);
1778 
1779                 /* yield cpu to avoid soft lockup */
1780                 cond_resched();
1781 
1782                 if (hstate_is_gigantic(h))
1783                         ret = alloc_fresh_gigantic_page(h, nodes_allowed);
1784                 else
1785                         ret = alloc_fresh_huge_page(h, nodes_allowed);
1786                 spin_lock(&hugetlb_lock);
1787                 if (!ret)
1788                         goto out;
1789 
1790                 /* Bail for signals. Probably ctrl-c from user */
1791                 if (signal_pending(current))
1792                         goto out;
1793         }
1794 
1795         /*
1796          * Decrease the pool size
1797          * First return free pages to the buddy allocator (being careful
1798          * to keep enough around to satisfy reservations).  Then place
1799          * pages into surplus state as needed so the pool will shrink
1800          * to the desired size as pages become free.
1801          *
1802          * By placing pages into the surplus state independent of the
1803          * overcommit value, we are allowing the surplus pool size to
1804          * exceed overcommit. There are few sane options here. Since
1805          * alloc_buddy_huge_page() is checking the global counter,
1806          * though, we'll note that we're not allowed to exceed surplus
1807          * and won't grow the pool anywhere else. Not until one of the
1808          * sysctls are changed, or the surplus pages go out of use.
1809          */
1810         min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
1811         min_count = max(count, min_count);
1812         try_to_free_low(h, min_count, nodes_allowed);
1813         while (min_count < persistent_huge_pages(h)) {
1814                 if (!free_pool_huge_page(h, nodes_allowed, 0))
1815                         break;
1816                 cond_resched_lock(&hugetlb_lock);
1817         }
1818         while (count < persistent_huge_pages(h)) {
1819                 if (!adjust_pool_surplus(h, nodes_allowed, 1))
1820                         break;
1821         }
1822 out:
1823         ret = persistent_huge_pages(h);
1824         spin_unlock(&hugetlb_lock);
1825         return ret;
1826 }
1827 
1828 #define HSTATE_ATTR_RO(_name) \
1829         static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
1830 
1831 #define HSTATE_ATTR(_name) \
1832         static struct kobj_attribute _name##_attr = \
1833                 __ATTR(_name, 0644, _name##_show, _name##_store)
1834 
1835 static struct kobject *hugepages_kobj;
1836 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
1837 
1838 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
1839 
1840 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
1841 {
1842         int i;
1843 
1844         for (i = 0; i < HUGE_MAX_HSTATE; i++)
1845                 if (hstate_kobjs[i] == kobj) {
1846                         if (nidp)
1847                                 *nidp = NUMA_NO_NODE;
1848                         return &hstates[i];
1849                 }
1850 
1851         return kobj_to_node_hstate(kobj, nidp);
1852 }
1853 
1854 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
1855                                         struct kobj_attribute *attr, char *buf)
1856 {
1857         struct hstate *h;
1858         unsigned long nr_huge_pages;
1859         int nid;
1860 
1861         h = kobj_to_hstate(kobj, &nid);
1862         if (nid == NUMA_NO_NODE)
1863                 nr_huge_pages = h->nr_huge_pages;
1864         else
1865                 nr_huge_pages = h->nr_huge_pages_node[nid];
1866 
1867         return sprintf(buf, "%lu\n", nr_huge_pages);
1868 }
1869 
1870 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
1871                                            struct hstate *h, int nid,
1872                                            unsigned long count, size_t len)
1873 {
1874         int err;
1875         NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
1876 
1877         if (hstate_is_gigantic(h) && !gigantic_page_supported()) {
1878                 err = -EINVAL;
1879                 goto out;
1880         }
1881 
1882         if (nid == NUMA_NO_NODE) {
1883                 /*
1884                  * global hstate attribute
1885                  */
1886                 if (!(obey_mempolicy &&
1887                                 init_nodemask_of_mempolicy(nodes_allowed))) {
1888                         NODEMASK_FREE(nodes_allowed);
1889                         nodes_allowed = &node_states[N_MEMORY];
1890                 }
1891         } else if (nodes_allowed) {
1892                 /*
1893                  * per node hstate attribute: adjust count to global,
1894                  * but restrict alloc/free to the specified node.
1895                  */
1896                 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
1897                 init_nodemask_of_node(nodes_allowed, nid);
1898         } else
1899                 nodes_allowed = &node_states[N_MEMORY];
1900 
1901         h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
1902 
1903         if (nodes_allowed != &node_states[N_MEMORY])
1904                 NODEMASK_FREE(nodes_allowed);
1905 
1906         return len;
1907 out:
1908         NODEMASK_FREE(nodes_allowed);
1909         return err;
1910 }
1911 
1912 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
1913                                          struct kobject *kobj, const char *buf,
1914                                          size_t len)
1915 {
1916         struct hstate *h;
1917         unsigned long count;
1918         int nid;
1919         int err;
1920 
1921         err = kstrtoul(buf, 10, &count);
1922         if (err)
1923                 return err;
1924 
1925         h = kobj_to_hstate(kobj, &nid);
1926         return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
1927 }
1928 
1929 static ssize_t nr_hugepages_show(struct kobject *kobj,
1930                                        struct kobj_attribute *attr, char *buf)
1931 {
1932         return nr_hugepages_show_common(kobj, attr, buf);
1933 }
1934 
1935 static ssize_t nr_hugepages_store(struct kobject *kobj,
1936                struct kobj_attribute *attr, const char *buf, size_t len)
1937 {
1938         return nr_hugepages_store_common(false, kobj, buf, len);
1939 }
1940 HSTATE_ATTR(nr_hugepages);
1941 
1942 #ifdef CONFIG_NUMA
1943 
1944 /*
1945  * hstate attribute for optionally mempolicy-based constraint on persistent
1946  * huge page alloc/free.
1947  */
1948 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
1949                                        struct kobj_attribute *attr, char *buf)
1950 {
1951         return nr_hugepages_show_common(kobj, attr, buf);
1952 }
1953 
1954 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
1955                struct kobj_attribute *attr, const char *buf, size_t len)
1956 {
1957         return nr_hugepages_store_common(true, kobj, buf, len);
1958 }
1959 HSTATE_ATTR(nr_hugepages_mempolicy);
1960 #endif
1961 
1962 
1963 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
1964                                         struct kobj_attribute *attr, char *buf)
1965 {
1966         struct hstate *h = kobj_to_hstate(kobj, NULL);
1967         return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
1968 }
1969 
1970 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
1971                 struct kobj_attribute *attr, const char *buf, size_t count)
1972 {
1973         int err;
1974         unsigned long input;
1975         struct hstate *h = kobj_to_hstate(kobj, NULL);
1976 
1977         if (hstate_is_gigantic(h))
1978                 return -EINVAL;
1979 
1980         err = kstrtoul(buf, 10, &input);
1981         if (err)
1982                 return err;
1983 
1984         spin_lock(&hugetlb_lock);
1985         h->nr_overcommit_huge_pages = input;
1986         spin_unlock(&hugetlb_lock);
1987 
1988         return count;
1989 }
1990 HSTATE_ATTR(nr_overcommit_hugepages);
1991 
1992 static ssize_t free_hugepages_show(struct kobject *kobj,
1993                                         struct kobj_attribute *attr, char *buf)
1994 {
1995         struct hstate *h;
1996         unsigned long free_huge_pages;
1997         int nid;
1998 
1999         h = kobj_to_hstate(kobj, &nid);
2000         if (nid == NUMA_NO_NODE)
2001                 free_huge_pages = h->free_huge_pages;
2002         else
2003                 free_huge_pages = h->free_huge_pages_node[nid];
2004 
2005         return sprintf(buf, "%lu\n", free_huge_pages);
2006 }
2007 HSTATE_ATTR_RO(free_hugepages);
2008 
2009 static ssize_t resv_hugepages_show(struct kobject *kobj,
2010                                         struct kobj_attribute *attr, char *buf)
2011 {
2012         struct hstate *h = kobj_to_hstate(kobj, NULL);
2013         return sprintf(buf, "%lu\n", h->resv_huge_pages);
2014 }
2015 HSTATE_ATTR_RO(resv_hugepages);
2016 
2017 static ssize_t surplus_hugepages_show(struct kobject *kobj,
2018                                         struct kobj_attribute *attr, char *buf)
2019 {
2020         struct hstate *h;
2021         unsigned long surplus_huge_pages;
2022         int nid;
2023 
2024         h = kobj_to_hstate(kobj, &nid);
2025         if (nid == NUMA_NO_NODE)
2026                 surplus_huge_pages = h->surplus_huge_pages;
2027         else
2028                 surplus_huge_pages = h->surplus_huge_pages_node[nid];
2029 
2030         return sprintf(buf, "%lu\n", surplus_huge_pages);
2031 }
2032 HSTATE_ATTR_RO(surplus_hugepages);
2033 
2034 static struct attribute *hstate_attrs[] = {
2035         &nr_hugepages_attr.attr,
2036         &nr_overcommit_hugepages_attr.attr,
2037         &free_hugepages_attr.attr,
2038         &resv_hugepages_attr.attr,
2039         &surplus_hugepages_attr.attr,
2040 #ifdef CONFIG_NUMA
2041         &nr_hugepages_mempolicy_attr.attr,
2042 #endif
2043         NULL,
2044 };
2045 
2046 static struct attribute_group hstate_attr_group = {
2047         .attrs = hstate_attrs,
2048 };
2049 
2050 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
2051                                     struct kobject **hstate_kobjs,
2052                                     struct attribute_group *hstate_attr_group)
2053 {
2054         int retval;
2055         int hi = hstate_index(h);
2056 
2057         hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
2058         if (!hstate_kobjs[hi])
2059                 return -ENOMEM;
2060 
2061         retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
2062         if (retval)
2063                 kobject_put(hstate_kobjs[hi]);
2064 
2065         return retval;
2066 }
2067 
2068 static void __init hugetlb_sysfs_init(void)
2069 {
2070         struct hstate *h;
2071         int err;
2072 
2073         hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
2074         if (!hugepages_kobj)
2075                 return;
2076 
2077         for_each_hstate(h) {
2078                 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
2079                                          hstate_kobjs, &hstate_attr_group);
2080                 if (err)
2081                         pr_err("Hugetlb: Unable to add hstate %s", h->name);
2082         }
2083 }
2084 
2085 #ifdef CONFIG_NUMA
2086 
2087 /*
2088  * node_hstate/s - associate per node hstate attributes, via their kobjects,
2089  * with node devices in node_devices[] using a parallel array.  The array
2090  * index of a node device or _hstate == node id.
2091  * This is here to avoid any static dependency of the node device driver, in
2092  * the base kernel, on the hugetlb module.
2093  */
2094 struct node_hstate {
2095         struct kobject          *hugepages_kobj;
2096         struct kobject          *hstate_kobjs[HUGE_MAX_HSTATE];
2097 };
2098 struct node_hstate node_hstates[MAX_NUMNODES];
2099 
2100 /*
2101  * A subset of global hstate attributes for node devices
2102  */
2103 static struct attribute *per_node_hstate_attrs[] = {
2104         &nr_hugepages_attr.attr,
2105         &free_hugepages_attr.attr,
2106         &surplus_hugepages_attr.attr,
2107         NULL,
2108 };
2109 
2110 static struct attribute_group per_node_hstate_attr_group = {
2111         .attrs = per_node_hstate_attrs,
2112 };
2113 
2114 /*
2115  * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
2116  * Returns node id via non-NULL nidp.
2117  */
2118 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2119 {
2120         int nid;
2121 
2122         for (nid = 0; nid < nr_node_ids; nid++) {
2123                 struct node_hstate *nhs = &node_hstates[nid];
2124                 int i;
2125                 for (i = 0; i < HUGE_MAX_HSTATE; i++)
2126                         if (nhs->hstate_kobjs[i] == kobj) {
2127                                 if (nidp)
2128                                         *nidp = nid;
2129                                 return &hstates[i];
2130                         }
2131         }
2132 
2133         BUG();
2134         return NULL;
2135 }
2136 
2137 /*
2138  * Unregister hstate attributes from a single node device.
2139  * No-op if no hstate attributes attached.
2140  */
2141 static void hugetlb_unregister_node(struct node *node)
2142 {
2143         struct hstate *h;
2144         struct node_hstate *nhs = &node_hstates[node->dev.id];
2145 
2146         if (!nhs->hugepages_kobj)
2147                 return;         /* no hstate attributes */
2148 
2149         for_each_hstate(h) {
2150                 int idx = hstate_index(h);
2151                 if (nhs->hstate_kobjs[idx]) {
2152                         kobject_put(nhs->hstate_kobjs[idx]);
2153                         nhs->hstate_kobjs[idx] = NULL;
2154                 }
2155         }
2156 
2157         kobject_put(nhs->hugepages_kobj);
2158         nhs->hugepages_kobj = NULL;
2159 }
2160 
2161 /*
2162  * hugetlb module exit:  unregister hstate attributes from node devices
2163  * that have them.
2164  */
2165 static void hugetlb_unregister_all_nodes(void)
2166 {
2167         int nid;
2168 
2169         /*
2170          * disable node device registrations.
2171          */
2172         register_hugetlbfs_with_node(NULL, NULL);
2173 
2174         /*
2175          * remove hstate attributes from any nodes that have them.
2176          */
2177         for (nid = 0; nid < nr_node_ids; nid++)
2178                 hugetlb_unregister_node(node_devices[nid]);
2179 }
2180 
2181 /*
2182  * Register hstate attributes for a single node device.
2183  * No-op if attributes already registered.
2184  */
2185 static void hugetlb_register_node(struct node *node)
2186 {
2187         struct hstate *h;
2188         struct node_hstate *nhs = &node_hstates[node->dev.id];
2189         int err;
2190 
2191         if (nhs->hugepages_kobj)
2192                 return;         /* already allocated */
2193 
2194         nhs->hugepages_kobj = kobject_create_and_add("hugepages",
2195                                                         &node->dev.kobj);
2196         if (!nhs->hugepages_kobj)
2197                 return;
2198 
2199         for_each_hstate(h) {
2200                 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
2201                                                 nhs->hstate_kobjs,
2202                                                 &per_node_hstate_attr_group);
2203                 if (err) {
2204                         pr_err("Hugetlb: Unable to add hstate %s for node %d\n",
2205                                 h->name, node->dev.id);
2206                         hugetlb_unregister_node(node);
2207                         break;
2208                 }
2209         }
2210 }
2211 
2212 /*
2213  * hugetlb init time:  register hstate attributes for all registered node
2214  * devices of nodes that have memory.  All on-line nodes should have
2215  * registered their associated device by this time.
2216  */
2217 static void __init hugetlb_register_all_nodes(void)
2218 {
2219         int nid;
2220 
2221         for_each_node_state(nid, N_MEMORY) {
2222                 struct node *node = node_devices[nid];
2223                 if (node->dev.id == nid)
2224                         hugetlb_register_node(node);
2225         }
2226 
2227         /*
2228          * Let the node device driver know we're here so it can
2229          * [un]register hstate attributes on node hotplug.
2230          */
2231         register_hugetlbfs_with_node(hugetlb_register_node,
2232                                      hugetlb_unregister_node);
2233 }
2234 #else   /* !CONFIG_NUMA */
2235 
2236 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2237 {
2238         BUG();
2239         if (nidp)
2240                 *nidp = -1;
2241         return NULL;
2242 }
2243 
2244 static void hugetlb_unregister_all_nodes(void) { }
2245 
2246 static void hugetlb_register_all_nodes(void) { }
2247 
2248 #endif
2249 
2250 static void __exit hugetlb_exit(void)
2251 {
2252         struct hstate *h;
2253 
2254         hugetlb_unregister_all_nodes();
2255 
2256         for_each_hstate(h) {
2257                 kobject_put(hstate_kobjs[hstate_index(h)]);
2258         }
2259 
2260         kobject_put(hugepages_kobj);
2261         kfree(htlb_fault_mutex_table);
2262 }
2263 module_exit(hugetlb_exit);
2264 
2265 static int __init hugetlb_init(void)
2266 {
2267         int i;
2268 
2269         if (!hugepages_supported())
2270                 return 0;
2271 
2272         if (!size_to_hstate(default_hstate_size)) {
2273                 default_hstate_size = HPAGE_SIZE;
2274                 if (!size_to_hstate(default_hstate_size))
2275                         hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
2276         }
2277         default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size));
2278         if (default_hstate_max_huge_pages)
2279                 default_hstate.max_huge_pages = default_hstate_max_huge_pages;
2280 
2281         hugetlb_init_hstates();
2282         gather_bootmem_prealloc();
2283         report_hugepages();
2284 
2285         hugetlb_sysfs_init();
2286         hugetlb_register_all_nodes();
2287         hugetlb_cgroup_file_init();
2288 
2289 #ifdef CONFIG_SMP
2290         num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
2291 #else
2292         num_fault_mutexes = 1;
2293 #endif
2294         htlb_fault_mutex_table =
2295                 kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL);
2296         BUG_ON(!htlb_fault_mutex_table);
2297 
2298         for (i = 0; i < num_fault_mutexes; i++)
2299                 mutex_init(&htlb_fault_mutex_table[i]);
2300         return 0;
2301 }
2302 module_init(hugetlb_init);
2303 
2304 /* Should be called on processing a hugepagesz=... option */
2305 void __init hugetlb_add_hstate(unsigned int order)
2306 {
2307         struct hstate *h;
2308         unsigned long i;
2309 
2310         if (size_to_hstate(PAGE_SIZE << order)) {
2311                 pr_warning("hugepagesz= specified twice, ignoring\n");
2312                 return;
2313         }
2314         BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
2315         BUG_ON(order == 0);
2316         h = &hstates[hugetlb_max_hstate++];
2317         h->order = order;
2318         h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
2319         h->nr_huge_pages = 0;
2320         h->free_huge_pages = 0;
2321         for (i = 0; i < MAX_NUMNODES; ++i)
2322                 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
2323         INIT_LIST_HEAD(&h->hugepage_activelist);
2324         h->next_nid_to_alloc = first_node(node_states[N_MEMORY]);
2325         h->next_nid_to_free = first_node(node_states[N_MEMORY]);
2326         snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
2327                                         huge_page_size(h)/1024);
2328 
2329         parsed_hstate = h;
2330 }
2331 
2332 static int __init hugetlb_nrpages_setup(char *s)
2333 {
2334         unsigned long *mhp;
2335         static unsigned long *last_mhp;
2336 
2337         /*
2338          * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet,
2339          * so this hugepages= parameter goes to the "default hstate".
2340          */
2341         if (!hugetlb_max_hstate)
2342                 mhp = &default_hstate_max_huge_pages;
2343         else
2344                 mhp = &parsed_hstate->max_huge_pages;
2345 
2346         if (mhp == last_mhp) {
2347                 pr_warning("hugepages= specified twice without "
2348                            "interleaving hugepagesz=, ignoring\n");
2349                 return 1;
2350         }
2351 
2352         if (sscanf(s, "%lu", mhp) <= 0)
2353                 *mhp = 0;
2354 
2355         /*
2356          * Global state is always initialized later in hugetlb_init.
2357          * But we need to allocate >= MAX_ORDER hstates here early to still
2358          * use the bootmem allocator.
2359          */
2360         if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER)
2361                 hugetlb_hstate_alloc_pages(parsed_hstate);
2362 
2363         last_mhp = mhp;
2364 
2365         return 1;
2366 }
2367 __setup("hugepages=", hugetlb_nrpages_setup);
2368 
2369 static int __init hugetlb_default_setup(char *s)
2370 {
2371         default_hstate_size = memparse(s, &s);
2372         return 1;
2373 }
2374 __setup("default_hugepagesz=", hugetlb_default_setup);
2375 
2376 static unsigned int cpuset_mems_nr(unsigned int *array)
2377 {
2378         int node;
2379         unsigned int nr = 0;
2380 
2381         for_each_node_mask(node, cpuset_current_mems_allowed)
2382                 nr += array[node];
2383 
2384         return nr;
2385 }
2386 
2387 #ifdef CONFIG_SYSCTL
2388 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
2389                          struct ctl_table *table, int write,
2390                          void __user *buffer, size_t *length, loff_t *ppos)
2391 {
2392         struct hstate *h = &default_hstate;
2393         unsigned long tmp = h->max_huge_pages;
2394         int ret;
2395 
2396         if (!hugepages_supported())
2397                 return -ENOTSUPP;
2398 
2399         table->data = &tmp;
2400         table->maxlen = sizeof(unsigned long);
2401         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2402         if (ret)
2403                 goto out;
2404 
2405         if (write)
2406                 ret = __nr_hugepages_store_common(obey_mempolicy, h,
2407                                                   NUMA_NO_NODE, tmp, *length);
2408 out:
2409         return ret;
2410 }
2411 
2412 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2413                           void __user *buffer, size_t *length, loff_t *ppos)
2414 {
2415 
2416         return hugetlb_sysctl_handler_common(false, table, write,
2417                                                         buffer, length, ppos);
2418 }
2419 
2420 #ifdef CONFIG_NUMA
2421 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2422                           void __user *buffer, size_t *length, loff_t *ppos)
2423 {
2424         return hugetlb_sysctl_handler_common(true, table, write,
2425                                                         buffer, length, ppos);
2426 }
2427 #endif /* CONFIG_NUMA */
2428 
2429 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
2430                         void __user *buffer,
2431                         size_t *length, loff_t *ppos)
2432 {
2433         struct hstate *h = &default_hstate;
2434         unsigned long tmp;
2435         int ret;
2436 
2437         if (!hugepages_supported())
2438                 return -ENOTSUPP;
2439 
2440         tmp = h->nr_overcommit_huge_pages;
2441 
2442         if (write && hstate_is_gigantic(h))
2443                 return -EINVAL;
2444 
2445         table->data = &tmp;
2446         table->maxlen = sizeof(unsigned long);
2447         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2448         if (ret)
2449                 goto out;
2450 
2451         if (write) {
2452                 spin_lock(&hugetlb_lock);
2453                 h->nr_overcommit_huge_pages = tmp;
2454                 spin_unlock(&hugetlb_lock);
2455         }
2456 out:
2457         return ret;
2458 }
2459 
2460 #endif /* CONFIG_SYSCTL */
2461 
2462 void hugetlb_report_meminfo(struct seq_file *m)
2463 {
2464         struct hstate *h = &default_hstate;
2465         if (!hugepages_supported())
2466                 return;
2467         seq_printf(m,
2468                         "HugePages_Total:   %5lu\n"
2469                         "HugePages_Free:    %5lu\n"
2470                         "HugePages_Rsvd:    %5lu\n"
2471                         "HugePages_Surp:    %5lu\n"
2472                         "Hugepagesize:   %8lu kB\n",
2473                         h->nr_huge_pages,
2474                         h->free_huge_pages,
2475                         h->resv_huge_pages,
2476                         h->surplus_huge_pages,
2477                         1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2478 }
2479 
2480 int hugetlb_report_node_meminfo(int nid, char *buf)
2481 {
2482         struct hstate *h = &default_hstate;
2483         if (!hugepages_supported())
2484                 return 0;
2485         return sprintf(buf,
2486                 "Node %d HugePages_Total: %5u\n"
2487                 "Node %d HugePages_Free:  %5u\n"
2488                 "Node %d HugePages_Surp:  %5u\n",
2489                 nid, h->nr_huge_pages_node[nid],
2490                 nid, h->free_huge_pages_node[nid],
2491                 nid, h->surplus_huge_pages_node[nid]);
2492 }
2493 
2494 void hugetlb_show_meminfo(void)
2495 {
2496         struct hstate *h;
2497         int nid;
2498 
2499         if (!hugepages_supported())
2500                 return;
2501 
2502         for_each_node_state(nid, N_MEMORY)
2503                 for_each_hstate(h)
2504                         pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
2505                                 nid,
2506                                 h->nr_huge_pages_node[nid],
2507                                 h->free_huge_pages_node[nid],
2508                                 h->surplus_huge_pages_node[nid],
2509                                 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2510 }
2511 
2512 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
2513 unsigned long hugetlb_total_pages(void)
2514 {
2515         struct hstate *h;
2516         unsigned long nr_total_pages = 0;
2517 
2518         for_each_hstate(h)
2519                 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
2520         return nr_total_pages;
2521 }
2522 
2523 static int hugetlb_acct_memory(struct hstate *h, long delta)
2524 {
2525         int ret = -ENOMEM;
2526 
2527         spin_lock(&hugetlb_lock);
2528         /*
2529          * When cpuset is configured, it breaks the strict hugetlb page
2530          * reservation as the accounting is done on a global variable. Such
2531          * reservation is completely rubbish in the presence of cpuset because
2532          * the reservation is not checked against page availability for the
2533          * current cpuset. Application can still potentially OOM'ed by kernel
2534          * with lack of free htlb page in cpuset that the task is in.
2535          * Attempt to enforce strict accounting with cpuset is almost
2536          * impossible (or too ugly) because cpuset is too fluid that
2537          * task or memory node can be dynamically moved between cpusets.
2538          *
2539          * The change of semantics for shared hugetlb mapping with cpuset is
2540          * undesirable. However, in order to preserve some of the semantics,
2541          * we fall back to check against current free page availability as
2542          * a best attempt and hopefully to minimize the impact of changing
2543          * semantics that cpuset has.
2544          */
2545         if (delta > 0) {
2546                 if (gather_surplus_pages(h, delta) < 0)
2547                         goto out;
2548 
2549                 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
2550                         return_unused_surplus_pages(h, delta);
2551                         goto out;
2552                 }
2553         }
2554 
2555         ret = 0;
2556         if (delta < 0)
2557                 return_unused_surplus_pages(h, (unsigned long) -delta);
2558 
2559 out:
2560         spin_unlock(&hugetlb_lock);
2561         return ret;
2562 }
2563 
2564 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
2565 {
2566         struct resv_map *resv = vma_resv_map(vma);
2567 
2568         /*
2569          * This new VMA should share its siblings reservation map if present.
2570          * The VMA will only ever have a valid reservation map pointer where
2571          * it is being copied for another still existing VMA.  As that VMA
2572          * has a reference to the reservation map it cannot disappear until
2573          * after this open call completes.  It is therefore safe to take a
2574          * new reference here without additional locking.
2575          */
2576         if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
2577                 kref_get(&resv->refs);
2578 }
2579 
2580 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
2581 {
2582         struct hstate *h = hstate_vma(vma);
2583         struct resv_map *resv = vma_resv_map(vma);
2584         struct hugepage_subpool *spool = subpool_vma(vma);
2585         unsigned long reserve, start, end;
2586         long gbl_reserve;
2587 
2588         if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
2589                 return;
2590 
2591         start = vma_hugecache_offset(h, vma, vma->vm_start);
2592         end = vma_hugecache_offset(h, vma, vma->vm_end);
2593 
2594         reserve = (end - start) - region_count(resv, start, end);
2595 
2596         kref_put(&resv->refs, resv_map_release);
2597 
2598         if (reserve) {
2599                 /*
2600                  * Decrement reserve counts.  The global reserve count may be
2601                  * adjusted if the subpool has a minimum size.
2602                  */
2603                 gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
2604                 hugetlb_acct_memory(h, -gbl_reserve);
2605         }
2606 }
2607 
2608 /*
2609  * We cannot handle pagefaults against hugetlb pages at all.  They cause
2610  * handle_mm_fault() to try to instantiate regular-sized pages in the
2611  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
2612  * this far.
2613  */
2614 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2615 {
2616         BUG();
2617         return 0;
2618 }
2619 
2620 const struct vm_operations_struct hugetlb_vm_ops = {
2621         .fault = hugetlb_vm_op_fault,
2622         .open = hugetlb_vm_op_open,
2623         .close = hugetlb_vm_op_close,
2624 };
2625 
2626 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
2627                                 int writable)
2628 {
2629         pte_t entry;
2630 
2631         if (writable) {
2632                 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
2633                                          vma->vm_page_prot)));
2634         } else {
2635                 entry = huge_pte_wrprotect(mk_huge_pte(page,
2636                                            vma->vm_page_prot));
2637         }
2638         entry = pte_mkyoung(entry);
2639         entry = pte_mkhuge(entry);
2640         entry = arch_make_huge_pte(entry, vma, page, writable);
2641 
2642         return entry;
2643 }
2644 
2645 static void set_huge_ptep_writable(struct vm_area_struct *vma,
2646                                    unsigned long address, pte_t *ptep)
2647 {
2648         pte_t entry;
2649 
2650         entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
2651         if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
2652                 update_mmu_cache(vma, address, ptep);
2653 }
2654 
2655 static int is_hugetlb_entry_migration(pte_t pte)
2656 {
2657         swp_entry_t swp;
2658 
2659         if (huge_pte_none(pte) || pte_present(pte))
2660                 return 0;
2661         swp = pte_to_swp_entry(pte);
2662         if (non_swap_entry(swp) && is_migration_entry(swp))
2663                 return 1;
2664         else
2665                 return 0;
2666 }
2667 
2668 static int is_hugetlb_entry_hwpoisoned(pte_t pte)
2669 {
2670         swp_entry_t swp;
2671 
2672         if (huge_pte_none(pte) || pte_present(pte))
2673                 return 0;
2674         swp = pte_to_swp_entry(pte);
2675         if (non_swap_entry(swp) && is_hwpoison_entry(swp))
2676                 return 1;
2677         else
2678                 return 0;
2679 }
2680 
2681 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
2682                             struct vm_area_struct *vma)
2683 {
2684         pte_t *src_pte, *dst_pte, entry;
2685         struct page *ptepage;
2686         unsigned long addr;
2687         int cow;
2688         struct hstate *h = hstate_vma(vma);
2689         unsigned long sz = huge_page_size(h);
2690         unsigned long mmun_start;       /* For mmu_notifiers */
2691         unsigned long mmun_end;         /* For mmu_notifiers */
2692         int ret = 0;
2693 
2694         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
2695 
2696         mmun_start = vma->vm_start;
2697         mmun_end = vma->vm_end;
2698         if (cow)
2699                 mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end);
2700 
2701         for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
2702                 spinlock_t *src_ptl, *dst_ptl;
2703                 src_pte = huge_pte_offset(src, addr);
2704                 if (!src_pte)
2705                         continue;
2706                 dst_pte = huge_pte_alloc(dst, addr, sz);
2707                 if (!dst_pte) {
2708                         ret = -ENOMEM;
2709                         break;
2710                 }
2711 
2712                 /* If the pagetables are shared don't copy or take references */
2713                 if (dst_pte == src_pte)
2714                         continue;
2715 
2716                 dst_ptl = huge_pte_lock(h, dst, dst_pte);
2717                 src_ptl = huge_pte_lockptr(h, src, src_pte);
2718                 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
2719                 entry = huge_ptep_get(src_pte);
2720                 if (huge_pte_none(entry)) { /* skip none entry */
2721                         ;
2722                 } else if (unlikely(is_hugetlb_entry_migration(entry) ||
2723                                     is_hugetlb_entry_hwpoisoned(entry))) {
2724                         swp_entry_t swp_entry = pte_to_swp_entry(entry);
2725 
2726                         if (is_write_migration_entry(swp_entry) && cow) {
2727                                 /*
2728                                  * COW mappings require pages in both
2729                                  * parent and child to be set to read.
2730                                  */
2731                                 make_migration_entry_read(&swp_entry);
2732                                 entry = swp_entry_to_pte(swp_entry);
2733                                 set_huge_pte_at(src, addr, src_pte, entry);
2734                         }
2735                         set_huge_pte_at(dst, addr, dst_pte, entry);
2736                 } else {
2737                         if (cow) {
2738                                 huge_ptep_set_wrprotect(src, addr, src_pte);
2739                                 mmu_notifier_invalidate_range(src, mmun_start,
2740                                                                    mmun_end);
2741                         }
2742                         entry = huge_ptep_get(src_pte);
2743                         ptepage = pte_page(entry);
2744                         get_page(ptepage);
2745                         page_dup_rmap(ptepage);
2746                         set_huge_pte_at(dst, addr, dst_pte, entry);
2747                 }
2748                 spin_unlock(src_ptl);
2749                 spin_unlock(dst_ptl);
2750         }
2751 
2752         if (cow)
2753                 mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end);
2754 
2755         return ret;
2756 }
2757 
2758 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
2759                             unsigned long start, unsigned long end,
2760                             struct page *ref_page)
2761 {
2762         int force_flush = 0;
2763         struct mm_struct *mm = vma->vm_mm;
2764         unsigned long address;
2765         pte_t *ptep;
2766         pte_t pte;
2767         spinlock_t *ptl;
2768         struct page *page;
2769         struct hstate *h = hstate_vma(vma);
2770         unsigned long sz = huge_page_size(h);
2771         const unsigned long mmun_start = start; /* For mmu_notifiers */
2772         const unsigned long mmun_end   = end;   /* For mmu_notifiers */
2773 
2774         WARN_ON(!is_vm_hugetlb_page(vma));
2775         BUG_ON(start & ~huge_page_mask(h));
2776         BUG_ON(end & ~huge_page_mask(h));
2777 
2778         tlb_start_vma(tlb, vma);
2779         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
2780         address = start;
2781 again:
2782         for (; address < end; address += sz) {
2783                 ptep = huge_pte_offset(mm, address);
2784                 if (!ptep)
2785                         continue;
2786 
2787                 ptl = huge_pte_lock(h, mm, ptep);
2788                 if (huge_pmd_unshare(mm, &address, ptep))
2789                         goto unlock;
2790 
2791                 pte = huge_ptep_get(ptep);
2792                 if (huge_pte_none(pte))
2793                         goto unlock;
2794 
2795                 /*
2796                  * Migrating hugepage or HWPoisoned hugepage is already
2797                  * unmapped and its refcount is dropped, so just clear pte here.
2798                  */
2799                 if (unlikely(!pte_present(pte))) {
2800                         huge_pte_clear(mm, address, ptep);
2801                         goto unlock;
2802                 }
2803 
2804                 page = pte_page(pte);
2805                 /*
2806                  * If a reference page is supplied, it is because a specific
2807                  * page is being unmapped, not a range. Ensure the page we
2808                  * are about to unmap is the actual page of interest.
2809                  */
2810                 if (ref_page) {
2811                         if (page != ref_page)
2812                                 goto unlock;
2813 
2814                         /*
2815                          * Mark the VMA as having unmapped its page so that
2816                          * future faults in this VMA will fail rather than
2817                          * looking like data was lost
2818                          */
2819                         set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
2820                 }
2821 
2822                 pte = huge_ptep_get_and_clear(mm, address, ptep);
2823                 tlb_remove_tlb_entry(tlb, ptep, address);
2824                 if (huge_pte_dirty(pte))
2825                         set_page_dirty(page);
2826 
2827                 page_remove_rmap(page);
2828                 force_flush = !__tlb_remove_page(tlb, page);
2829                 if (force_flush) {
2830                         address += sz;
2831                         spin_unlock(ptl);
2832                         break;
2833                 }
2834                 /* Bail out after unmapping reference page if supplied */
2835                 if (ref_page) {
2836                         spin_unlock(ptl);
2837                         break;
2838                 }
2839 unlock:
2840                 spin_unlock(ptl);
2841         }
2842         /*
2843          * mmu_gather ran out of room to batch pages, we break out of
2844          * the PTE lock to avoid doing the potential expensive TLB invalidate
2845          * and page-free while holding it.
2846          */
2847         if (force_flush) {
2848                 force_flush = 0;
2849                 tlb_flush_mmu(tlb);
2850                 if (address < end && !ref_page)
2851                         goto again;
2852         }
2853         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2854         tlb_end_vma(tlb, vma);
2855 }
2856 
2857 void __unmap_hugepage_range_final(struct mmu_gather *tlb,
2858                           struct vm_area_struct *vma, unsigned long start,
2859                           unsigned long end, struct page *ref_page)
2860 {
2861         __unmap_hugepage_range(tlb, vma, start, end, ref_page);
2862 
2863         /*
2864          * Clear this flag so that x86's huge_pmd_share page_table_shareable
2865          * test will fail on a vma being torn down, and not grab a page table
2866          * on its way out.  We're lucky that the flag has such an appropriate
2867          * name, and can in fact be safely cleared here. We could clear it
2868          * before the __unmap_hugepage_range above, but all that's necessary
2869          * is to clear it before releasing the i_mmap_rwsem. This works
2870          * because in the context this is called, the VMA is about to be
2871          * destroyed and the i_mmap_rwsem is held.
2872          */
2873         vma->vm_flags &= ~VM_MAYSHARE;
2874 }
2875 
2876 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2877                           unsigned long end, struct page *ref_page)
2878 {
2879         struct mm_struct *mm;
2880         struct mmu_gather tlb;
2881 
2882         mm = vma->vm_mm;
2883 
2884         tlb_gather_mmu(&tlb, mm, start, end);
2885         __unmap_hugepage_range(&tlb, vma, start, end, ref_page);
2886         tlb_finish_mmu(&tlb, start, end);
2887 }
2888 
2889 /*
2890  * This is called when the original mapper is failing to COW a MAP_PRIVATE
2891  * mappping it owns the reserve page for. The intention is to unmap the page
2892  * from other VMAs and let the children be SIGKILLed if they are faulting the
2893  * same region.
2894  */
2895 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
2896                               struct page *page, unsigned long address)
2897 {
2898         struct hstate *h = hstate_vma(vma);
2899         struct vm_area_struct *iter_vma;
2900         struct address_space *mapping;
2901         pgoff_t pgoff;
2902 
2903         /*
2904          * vm_pgoff is in PAGE_SIZE units, hence the different calculation
2905          * from page cache lookup which is in HPAGE_SIZE units.
2906          */
2907         address = address & huge_page_mask(h);
2908         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
2909                         vma->vm_pgoff;
2910         mapping = file_inode(vma->vm_file)->i_mapping;
2911 
2912         /*
2913          * Take the mapping lock for the duration of the table walk. As
2914          * this mapping should be shared between all the VMAs,
2915          * __unmap_hugepage_range() is called as the lock is already held
2916          */
2917         i_mmap_lock_write(mapping);
2918         vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
2919                 /* Do not unmap the current VMA */
2920                 if (iter_vma == vma)
2921                         continue;
2922 
2923                 /*
2924                  * Shared VMAs have their own reserves and do not affect
2925                  * MAP_PRIVATE accounting but it is possible that a shared
2926                  * VMA is using the same page so check and skip such VMAs.
2927                  */
2928                 if (iter_vma->vm_flags & VM_MAYSHARE)
2929                         continue;
2930 
2931                 /*
2932                  * Unmap the page from other VMAs without their own reserves.
2933                  * They get marked to be SIGKILLed if they fault in these
2934                  * areas. This is because a future no-page fault on this VMA
2935                  * could insert a zeroed page instead of the data existing
2936                  * from the time of fork. This would look like data corruption
2937                  */
2938                 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
2939                         unmap_hugepage_range(iter_vma, address,
2940                                              address + huge_page_size(h), page);
2941         }
2942         i_mmap_unlock_write(mapping);
2943 }
2944 
2945 /*
2946  * Hugetlb_cow() should be called with page lock of the original hugepage held.
2947  * Called with hugetlb_instantiation_mutex held and pte_page locked so we
2948  * cannot race with other handlers or page migration.
2949  * Keep the pte_same checks anyway to make transition from the mutex easier.
2950  */
2951 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
2952                         unsigned long address, pte_t *ptep, pte_t pte,
2953                         struct page *pagecache_page, spinlock_t *ptl)
2954 {
2955         struct hstate *h = hstate_vma(vma);
2956         struct page *old_page, *new_page;
2957         int ret = 0, outside_reserve = 0;
2958         unsigned long mmun_start;       /* For mmu_notifiers */
2959         unsigned long mmun_end;         /* For mmu_notifiers */
2960 
2961         old_page = pte_page(pte);
2962 
2963 retry_avoidcopy:
2964         /* If no-one else is actually using this page, avoid the copy
2965          * and just make the page writable */
2966         if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
2967                 page_move_anon_rmap(old_page, vma, address);
2968                 set_huge_ptep_writable(vma, address, ptep);
2969                 return 0;
2970         }
2971 
2972         /*
2973          * If the process that created a MAP_PRIVATE mapping is about to
2974          * perform a COW due to a shared page count, attempt to satisfy
2975          * the allocation without using the existing reserves. The pagecache
2976          * page is used to determine if the reserve at this address was
2977          * consumed or not. If reserves were used, a partial faulted mapping
2978          * at the time of fork() could consume its reserves on COW instead
2979          * of the full address range.
2980          */
2981         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
2982                         old_page != pagecache_page)
2983                 outside_reserve = 1;
2984 
2985         page_cache_get(old_page);
2986 
2987         /*
2988          * Drop page table lock as buddy allocator may be called. It will
2989          * be acquired again before returning to the caller, as expected.
2990          */
2991         spin_unlock(ptl);
2992         new_page = alloc_huge_page(vma, address, outside_reserve);
2993 
2994         if (IS_ERR(new_page)) {
2995                 /*
2996                  * If a process owning a MAP_PRIVATE mapping fails to COW,
2997                  * it is due to references held by a child and an insufficient
2998                  * huge page pool. To guarantee the original mappers
2999                  * reliability, unmap the page from child processes. The child
3000                  * may get SIGKILLed if it later faults.
3001                  */
3002                 if (outside_reserve) {
3003                         page_cache_release(old_page);
3004                         BUG_ON(huge_pte_none(pte));
3005                         unmap_ref_private(mm, vma, old_page, address);
3006                         BUG_ON(huge_pte_none(pte));
3007                         spin_lock(ptl);
3008                         ptep = huge_pte_offset(mm, address & huge_page_mask(h));
3009                         if (likely(ptep &&
3010                                    pte_same(huge_ptep_get(ptep), pte)))
3011                                 goto retry_avoidcopy;
3012                         /*
3013                          * race occurs while re-acquiring page table
3014                          * lock, and our job is done.
3015                          */
3016                         return 0;
3017                 }
3018 
3019                 ret = (PTR_ERR(new_page) == -ENOMEM) ?
3020                         VM_FAULT_OOM : VM_FAULT_SIGBUS;
3021                 goto out_release_old;
3022         }
3023 
3024         /*
3025          * When the original hugepage is shared one, it does not have
3026          * anon_vma prepared.
3027          */
3028         if (unlikely(anon_vma_prepare(vma))) {
3029                 ret = VM_FAULT_OOM;
3030                 goto out_release_all;
3031         }
3032 
3033         copy_user_huge_page(new_page, old_page, address, vma,
3034                             pages_per_huge_page(h));
3035         __SetPageUptodate(new_page);
3036         set_page_huge_active(new_page);
3037 
3038         mmun_start = address & huge_page_mask(h);
3039         mmun_end = mmun_start + huge_page_size(h);
3040         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
3041 
3042         /*
3043          * Retake the page table lock to check for racing updates
3044          * before the page tables are altered
3045          */
3046         spin_lock(ptl);
3047         ptep = huge_pte_offset(mm, address & huge_page_mask(h));
3048         if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
3049                 ClearPagePrivate(new_page);
3050 
3051                 /* Break COW */
3052                 huge_ptep_clear_flush(vma, address, ptep);
3053                 mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
3054                 set_huge_pte_at(mm, address, ptep,
3055                                 make_huge_pte(vma, new_page, 1));
3056                 page_remove_rmap(old_page);
3057                 hugepage_add_new_anon_rmap(new_page, vma, address);
3058                 /* Make the old page be freed below */
3059                 new_page = old_page;
3060         }
3061         spin_unlock(ptl);
3062         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
3063 out_release_all:
3064         page_cache_release(new_page);
3065 out_release_old:
3066         page_cache_release(old_page);
3067 
3068         spin_lock(ptl); /* Caller expects lock to be held */
3069         return ret;
3070 }
3071 
3072 /* Return the pagecache page at a given address within a VMA */
3073 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
3074                         struct vm_area_struct *vma, unsigned long address)
3075 {
3076         struct address_space *mapping;
3077         pgoff_t idx;
3078 
3079         mapping = vma->vm_file->f_mapping;
3080         idx = vma_hugecache_offset(h, vma, address);
3081 
3082         return find_lock_page(mapping, idx);
3083 }
3084 
3085 /*
3086  * Return whether there is a pagecache page to back given address within VMA.
3087  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
3088  */
3089 static bool hugetlbfs_pagecache_present(struct hstate *h,
3090                         struct vm_area_struct *vma, unsigned long address)
3091 {
3092         struct address_space *mapping;
3093         pgoff_t idx;
3094         struct page *page;
3095 
3096         mapping = vma->vm_file->f_mapping;
3097         idx = vma_hugecache_offset(h, vma, address);
3098 
3099         page = find_get_page(mapping, idx);
3100         if (page)
3101                 put_page(page);
3102         return page != NULL;
3103 }
3104 
3105 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
3106                            struct address_space *mapping, pgoff_t idx,
3107                            unsigned long address, pte_t *ptep, unsigned int flags)
3108 {
3109         struct hstate *h = hstate_vma(vma);
3110         int ret = VM_FAULT_SIGBUS;
3111         int anon_rmap = 0;
3112         unsigned long size;
3113         struct page *page;
3114         pte_t new_pte;
3115         spinlock_t *ptl;
3116 
3117         /*
3118          * Currently, we are forced to kill the process in the event the
3119          * original mapper has unmapped pages from the child due to a failed
3120          * COW. Warn that such a situation has occurred as it may not be obvious
3121          */
3122         if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
3123                 pr_warning("PID %d killed due to inadequate hugepage pool\n",
3124                            current->pid);
3125                 return ret;
3126         }
3127 
3128         /*
3129          * Use page lock to guard against racing truncation
3130          * before we get page_table_lock.
3131          */
3132 retry:
3133         page = find_lock_page(mapping, idx);
3134         if (!page) {
3135                 size = i_size_read(mapping->host) >> huge_page_shift(h);
3136                 if (idx >= size)
3137                         goto out;
3138                 page = alloc_huge_page(vma, address, 0);
3139                 if (IS_ERR(page)) {
3140                         ret = PTR_ERR(page);
3141                         if (ret == -ENOMEM)
3142                                 ret = VM_FAULT_OOM;
3143                         else
3144                                 ret = VM_FAULT_SIGBUS;
3145                         goto out;
3146                 }
3147                 clear_huge_page(page, address, pages_per_huge_page(h));
3148                 __SetPageUptodate(page);
3149                 set_page_huge_active(page);
3150 
3151                 if (vma->vm_flags & VM_MAYSHARE) {
3152                         int err;
3153                         struct inode *inode = mapping->host;
3154 
3155                         err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
3156                         if (err) {
3157                                 put_page(page);
3158                                 if (err == -EEXIST)
3159                                         goto retry;
3160                                 goto out;
3161                         }
3162                         ClearPagePrivate(page);
3163 
3164                         spin_lock(&inode->i_lock);
3165                         inode->i_blocks += blocks_per_huge_page(h);
3166                         spin_unlock(&inode->i_lock);
3167                 } else {
3168                         lock_page(page);
3169                         if (unlikely(anon_vma_prepare(vma))) {
3170                                 ret = VM_FAULT_OOM;
3171                                 goto backout_unlocked;
3172                         }
3173                         anon_rmap = 1;
3174                 }
3175         } else {
3176                 /*
3177                  * If memory error occurs between mmap() and fault, some process
3178                  * don't have hwpoisoned swap entry for errored virtual address.
3179                  * So we need to block hugepage fault by PG_hwpoison bit check.
3180                  */
3181                 if (unlikely(PageHWPoison(page))) {
3182                         ret = VM_FAULT_HWPOISON |
3183                                 VM_FAULT_SET_HINDEX(hstate_index(h));
3184                         goto backout_unlocked;
3185                 }
3186         }
3187 
3188         /*
3189          * If we are going to COW a private mapping later, we examine the
3190          * pending reservations for this page now. This will ensure that
3191          * any allocations necessary to record that reservation occur outside
3192          * the spinlock.
3193          */
3194         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
3195                 if (vma_needs_reservation(h, vma, address) < 0) {
3196                         ret = VM_FAULT_OOM;
3197                         goto backout_unlocked;
3198                 }
3199 
3200         ptl = huge_pte_lockptr(h, mm, ptep);
3201         spin_lock(ptl);
3202         size = i_size_read(mapping->host) >> huge_page_shift(h);
3203         if (idx >= size)
3204                 goto backout;
3205 
3206         ret = 0;
3207         if (!huge_pte_none(huge_ptep_get(ptep)))
3208                 goto backout;
3209 
3210         if (anon_rmap) {
3211                 ClearPagePrivate(page);
3212                 hugepage_add_new_anon_rmap(page, vma, address);
3213         } else
3214                 page_dup_rmap(page);
3215         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
3216                                 && (vma->vm_flags & VM_SHARED)));
3217         set_huge_pte_at(mm, address, ptep, new_pte);
3218 
3219         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3220                 /* Optimization, do the COW without a second fault */
3221                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page, ptl);
3222         }
3223 
3224         spin_unlock(ptl);
3225         unlock_page(page);
3226 out:
3227         return ret;
3228 
3229 backout:
3230         spin_unlock(ptl);
3231 backout_unlocked:
3232         unlock_page(page);
3233         put_page(page);
3234         goto out;
3235 }
3236 
3237 #ifdef CONFIG_SMP
3238 static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3239                             struct vm_area_struct *vma,
3240                             struct address_space *mapping,
3241                             pgoff_t idx, unsigned long address)
3242 {
3243         unsigned long key[2];
3244         u32 hash;
3245 
3246         if (vma->vm_flags & VM_SHARED) {
3247                 key[0] = (unsigned long) mapping;
3248                 key[1] = idx;
3249         } else {
3250                 key[0] = (unsigned long) mm;
3251                 key[1] = address >> huge_page_shift(h);
3252         }
3253 
3254         hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0);
3255 
3256         return hash & (num_fault_mutexes - 1);
3257 }
3258 #else
3259 /*
3260  * For uniprocesor systems we always use a single mutex, so just
3261  * return 0 and avoid the hashing overhead.
3262  */
3263 static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3264                             struct vm_area_struct *vma,
3265                             struct address_space *mapping,
3266                             pgoff_t idx, unsigned long address)
3267 {
3268         return 0;
3269 }
3270 #endif
3271 
3272 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
3273                         unsigned long address, unsigned int flags)
3274 {
3275         pte_t *ptep, entry;
3276         spinlock_t *ptl;
3277         int ret;
3278         u32 hash;
3279         pgoff_t idx;
3280         struct page *page = NULL;
3281         struct page *pagecache_page = NULL;
3282         struct hstate *h = hstate_vma(vma);
3283         struct address_space *mapping;
3284         int need_wait_lock = 0;
3285 
3286         address &= huge_page_mask(h);
3287 
3288         ptep = huge_pte_offset(mm, address);
3289         if (ptep) {
3290                 entry = huge_ptep_get(ptep);
3291                 if (unlikely(is_hugetlb_entry_migration(entry))) {
3292                         migration_entry_wait_huge(vma, mm, ptep);
3293                         return 0;
3294                 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
3295                         return VM_FAULT_HWPOISON_LARGE |
3296                                 VM_FAULT_SET_HINDEX(hstate_index(h));
3297         }
3298 
3299         ptep = huge_pte_alloc(mm, address, huge_page_size(h));
3300         if (!ptep)
3301                 return VM_FAULT_OOM;
3302 
3303         mapping = vma->vm_file->f_mapping;
3304         idx = vma_hugecache_offset(h, vma, address);
3305 
3306         /*
3307          * Serialize hugepage allocation and instantiation, so that we don't
3308          * get spurious allocation failures if two CPUs race to instantiate
3309          * the same page in the page cache.
3310          */
3311         hash = fault_mutex_hash(h, mm, vma, mapping, idx, address);
3312         mutex_lock(&htlb_fault_mutex_table[hash]);
3313 
3314         entry = huge_ptep_get(ptep);
3315         if (huge_pte_none(entry)) {
3316                 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
3317                 goto out_mutex;
3318         }
3319 
3320         ret = 0;
3321 
3322         /*
3323          * entry could be a migration/hwpoison entry at this point, so this
3324          * check prevents the kernel from going below assuming that we have
3325          * a active hugepage in pagecache. This goto expects the 2nd page fault,
3326          * and is_hugetlb_entry_(migration|hwpoisoned) check will properly
3327          * handle it.
3328          */
3329         if (!pte_present(entry))
3330                 goto out_mutex;
3331 
3332         /*
3333          * If we are going to COW the mapping later, we examine the pending
3334          * reservations for this page now. This will ensure that any
3335          * allocations necessary to record that reservation occur outside the
3336          * spinlock. For private mappings, we also lookup the pagecache
3337          * page now as it is used to determine if a reservation has been
3338          * consumed.
3339          */
3340         if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
3341                 if (vma_needs_reservation(h, vma, address) < 0) {
3342                         ret = VM_FAULT_OOM;
3343                         goto out_mutex;
3344                 }
3345 
3346                 if (!(vma->vm_flags & VM_MAYSHARE))
3347                         pagecache_page = hugetlbfs_pagecache_page(h,
3348                                                                 vma, address);
3349         }
3350 
3351         ptl = huge_pte_lock(h, mm, ptep);
3352 
3353         /* Check for a racing update before calling hugetlb_cow */
3354         if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
3355                 goto out_ptl;
3356 
3357         /*
3358          * hugetlb_cow() requires page locks of pte_page(entry) and
3359          * pagecache_page, so here we need take the former one
3360          * when page != pagecache_page or !pagecache_page.
3361          */
3362         page = pte_page(entry);
3363         if (page != pagecache_page)
3364                 if (!trylock_page(page)) {
3365                         need_wait_lock = 1;
3366                         goto out_ptl;
3367                 }
3368 
3369         get_page(page);
3370 
3371         if (flags & FAULT_FLAG_WRITE) {
3372                 if (!huge_pte_write(entry)) {
3373                         ret = hugetlb_cow(mm, vma, address, ptep, entry,
3374                                         pagecache_page, ptl);
3375                         goto out_put_page;
3376                 }
3377                 entry = huge_pte_mkdirty(entry);
3378         }
3379         entry = pte_mkyoung(entry);
3380         if (huge_ptep_set_access_flags(vma, address, ptep, entry,
3381                                                 flags & FAULT_FLAG_WRITE))
3382                 update_mmu_cache(vma, address, ptep);
3383 out_put_page:
3384         if (page != pagecache_page)
3385                 unlock_page(page);
3386         put_page(page);
3387 out_ptl:
3388         spin_unlock(ptl);
3389 
3390         if (pagecache_page) {
3391                 unlock_page(pagecache_page);
3392                 put_page(pagecache_page);
3393         }
3394 out_mutex:
3395         mutex_unlock(&htlb_fault_mutex_table[hash]);
3396         /*
3397          * Generally it's safe to hold refcount during waiting page lock. But
3398          * here we just wait to defer the next page fault to avoid busy loop and
3399          * the page is not used after unlocked before returning from the current
3400          * page fault. So we are safe from accessing freed page, even if we wait
3401          * here without taking refcount.
3402          */
3403         if (need_wait_lock)
3404                 wait_on_page_locked(page);
3405         return ret;
3406 }
3407 
3408 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
3409                          struct page **pages, struct vm_area_struct **vmas,
3410                          unsigned long *position, unsigned long *nr_pages,
3411                          long i, unsigned int flags)
3412 {
3413         unsigned long pfn_offset;
3414         unsigned long vaddr = *position;
3415         unsigned long remainder = *nr_pages;
3416         struct hstate *h = hstate_vma(vma);
3417 
3418         while (vaddr < vma->vm_end && remainder) {
3419                 pte_t *pte;
3420                 spinlock_t *ptl = NULL;
3421                 int absent;
3422                 struct page *page;
3423 
3424                 /*
3425                  * If we have a pending SIGKILL, don't keep faulting pages and
3426                  * potentially allocating memory.
3427                  */
3428                 if (unlikely(fatal_signal_pending(current))) {
3429                         remainder = 0;
3430                         break;
3431                 }
3432 
3433                 /*
3434                  * Some archs (sparc64, sh*) have multiple pte_ts to
3435                  * each hugepage.  We have to make sure we get the
3436                  * first, for the page indexing below to work.
3437                  *
3438                  * Note that page table lock is not held when pte is null.
3439                  */
3440                 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
3441                 if (pte)
3442                         ptl = huge_pte_lock(h, mm, pte);
3443                 absent = !pte || huge_pte_none(huge_ptep_get(pte));
3444 
3445                 /*
3446                  * When coredumping, it suits get_dump_page if we just return
3447                  * an error where there's an empty slot with no huge pagecache
3448                  * to back it.  This way, we avoid allocating a hugepage, and
3449                  * the sparse dumpfile avoids allocating disk blocks, but its
3450                  * huge holes still show up with zeroes where they need to be.
3451                  */
3452                 if (absent && (flags & FOLL_DUMP) &&
3453                     !hugetlbfs_pagecache_present(h, vma, vaddr)) {
3454                         if (pte)
3455                                 spin_unlock(ptl);
3456                         remainder = 0;
3457                         break;
3458                 }
3459 
3460                 /*
3461                  * We need call hugetlb_fault for both hugepages under migration
3462                  * (in which case hugetlb_fault waits for the migration,) and
3463                  * hwpoisoned hugepages (in which case we need to prevent the
3464                  * caller from accessing to them.) In order to do this, we use
3465                  * here is_swap_pte instead of is_hugetlb_entry_migration and
3466                  * is_hugetlb_entry_hwpoisoned. This is because it simply covers
3467                  * both cases, and because we can't follow correct pages
3468                  * directly from any kind of swap entries.
3469                  */
3470                 if (absent || is_swap_pte(huge_ptep_get(pte)) ||
3471                     ((flags & FOLL_WRITE) &&
3472                       !huge_pte_write(huge_ptep_get(pte)))) {
3473                         int ret;
3474 
3475                         if (pte)
3476                                 spin_unlock(ptl);
3477                         ret = hugetlb_fault(mm, vma, vaddr,
3478                                 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
3479                         if (!(ret & VM_FAULT_ERROR))
3480                                 continue;
3481 
3482                         remainder = 0;
3483                         break;
3484                 }
3485 
3486                 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
3487                 page = pte_page(huge_ptep_get(pte));
3488 same_page:
3489                 if (pages) {
3490                         pages[i] = mem_map_offset(page, pfn_offset);
3491                         get_page_foll(pages[i]);
3492                 }
3493 
3494                 if (vmas)
3495                         vmas[i] = vma;
3496 
3497                 vaddr += PAGE_SIZE;
3498                 ++pfn_offset;
3499                 --remainder;
3500                 ++i;
3501                 if (vaddr < vma->vm_end && remainder &&
3502                                 pfn_offset < pages_per_huge_page(h)) {
3503                         /*
3504                          * We use pfn_offset to avoid touching the pageframes
3505                          * of this compound page.
3506                          */
3507                         goto same_page;
3508                 }
3509                 spin_unlock(ptl);
3510         }
3511         *nr_pages = remainder;
3512         *position = vaddr;
3513 
3514         return i ? i : -EFAULT;
3515 }
3516 
3517 unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
3518                 unsigned long address, unsigned long end, pgprot_t newprot)
3519 {
3520         struct mm_struct *mm = vma->vm_mm;
3521         unsigned long start = address;
3522         pte_t *ptep;
3523         pte_t pte;
3524         struct hstate *h = hstate_vma(vma);
3525         unsigned long pages = 0;
3526 
3527         BUG_ON(address >= end);
3528         flush_cache_range(vma, address, end);
3529 
3530         mmu_notifier_invalidate_range_start(mm, start, end);
3531         i_mmap_lock_write(vma->vm_file->f_mapping);
3532         for (; address < end; address += huge_page_size(h)) {
3533                 spinlock_t *ptl;
3534                 ptep = huge_pte_offset(mm, address);
3535                 if (!ptep)
3536                         continue;
3537                 ptl = huge_pte_lock(h, mm, ptep);
3538                 if (huge_pmd_unshare(mm, &address, ptep)) {
3539                         pages++;
3540                         spin_unlock(ptl);
3541                         continue;
3542                 }
3543                 pte = huge_ptep_get(ptep);
3544                 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
3545                         spin_unlock(ptl);
3546                         continue;
3547                 }
3548                 if (unlikely(is_hugetlb_entry_migration(pte))) {
3549                         swp_entry_t entry = pte_to_swp_entry(pte);
3550 
3551                         if (is_write_migration_entry(entry)) {
3552                                 pte_t newpte;
3553 
3554                                 make_migration_entry_read(&entry);
3555                                 newpte = swp_entry_to_pte(entry);
3556                                 set_huge_pte_at(mm, address, ptep, newpte);
3557                                 pages++;
3558                         }
3559                         spin_unlock(ptl);
3560                         continue;
3561                 }
3562                 if (!huge_pte_none(pte)) {
3563                         pte = huge_ptep_get_and_clear(mm, address, ptep);
3564                         pte = pte_mkhuge(huge_pte_modify(pte, newprot));
3565                         pte = arch_make_huge_pte(pte, vma, NULL, 0);
3566                         set_huge_pte_at(mm, address, ptep, pte);
3567                         pages++;
3568                 }
3569                 spin_unlock(ptl);
3570         }
3571         /*
3572          * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
3573          * may have cleared our pud entry and done put_page on the page table:
3574          * once we release i_mmap_rwsem, another task can do the final put_page
3575          * and that page table be reused and filled with junk.
3576          */
3577         flush_tlb_range(vma, start, end);
3578         mmu_notifier_invalidate_range(mm, start, end);
3579         i_mmap_unlock_write(vma->vm_file->f_mapping);
3580         mmu_notifier_invalidate_range_end(mm, start, end);
3581 
3582         return pages << h->order;
3583 }
3584 
3585 int hugetlb_reserve_pages(struct inode *inode,
3586                                         long from, long to,
3587                                         struct vm_area_struct *vma,
3588                                         vm_flags_t vm_flags)
3589 {
3590         long ret, chg;
3591         struct hstate *h = hstate_inode(inode);
3592         struct hugepage_subpool *spool = subpool_inode(inode);
3593         struct resv_map *resv_map;
3594         long gbl_reserve;
3595 
3596         /*
3597          * Only apply hugepage reservation if asked. At fault time, an
3598          * attempt will be made for VM_NORESERVE to allocate a page
3599          * without using reserves
3600          */
3601         if (vm_flags & VM_NORESERVE)
3602                 return 0;
3603 
3604         /*
3605          * Shared mappings base their reservation on the number of pages that
3606          * are already allocated on behalf of the file. Private mappings need
3607          * to reserve the full area even if read-only as mprotect() may be
3608          * called to make the mapping read-write. Assume !vma is a shm mapping
3609          */
3610         if (!vma || vma->vm_flags & VM_MAYSHARE) {
3611                 resv_map = inode_resv_map(inode);
3612 
3613                 chg = region_chg(resv_map, from, to);
3614 
3615         } else {
3616                 resv_map = resv_map_alloc();
3617                 if (!resv_map)
3618                         return -ENOMEM;
3619 
3620                 chg = to - from;
3621 
3622                 set_vma_resv_map(vma, resv_map);
3623                 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
3624         }
3625 
3626         if (chg < 0) {
3627                 ret = chg;
3628                 goto out_err;
3629         }
3630 
3631         /*
3632          * There must be enough pages in the subpool for the mapping. If
3633          * the subpool has a minimum size, there may be some global
3634          * reservations already in place (gbl_reserve).
3635          */
3636         gbl_reserve = hugepage_subpool_get_pages(spool, chg);
3637         if (gbl_reserve < 0) {
3638                 ret = -ENOSPC;
3639                 goto out_err;
3640         }
3641 
3642         /*
3643          * Check enough hugepages are available for the reservation.
3644          * Hand the pages back to the subpool if there are not
3645          */
3646         ret = hugetlb_acct_memory(h, gbl_reserve);
3647         if (ret < 0) {
3648                 /* put back original number of pages, chg */
3649                 (void)hugepage_subpool_put_pages(spool, chg);
3650                 goto out_err;
3651         }
3652 
3653         /*
3654          * Account for the reservations made. Shared mappings record regions
3655          * that have reservations as they are shared by multiple VMAs.
3656          * When the last VMA disappears, the region map says how much
3657          * the reservation was and the page cache tells how much of
3658          * the reservation was consumed. Private mappings are per-VMA and
3659          * only the consumed reservations are tracked. When the VMA
3660          * disappears, the original reservation is the VMA size and the
3661          * consumed reservations are stored in the map. Hence, nothing
3662          * else has to be done for private mappings here
3663          */
3664         if (!vma || vma->vm_flags & VM_MAYSHARE)
3665                 region_add(resv_map, from, to);
3666         return 0;
3667 out_err:
3668         if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3669                 kref_put(&resv_map->refs, resv_map_release);
3670         return ret;
3671 }
3672 
3673 void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
3674 {
3675         struct hstate *h = hstate_inode(inode);
3676         struct resv_map *resv_map = inode_resv_map(inode);
3677         long chg = 0;
3678         struct hugepage_subpool *spool = subpool_inode(inode);
3679         long gbl_reserve;
3680 
3681         if (resv_map)
3682                 chg = region_truncate(resv_map, offset);
3683         spin_lock(&inode->i_lock);
3684         inode->i_blocks -= (blocks_per_huge_page(h) * freed);
3685         spin_unlock(&inode->i_lock);
3686 
3687         /*
3688          * If the subpool has a minimum size, the number of global
3689          * reservations to be released may be adjusted.
3690          */
3691         gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
3692         hugetlb_acct_memory(h, -gbl_reserve);
3693 }
3694 
3695 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
3696 static unsigned long page_table_shareable(struct vm_area_struct *svma,
3697                                 struct vm_area_struct *vma,
3698                                 unsigned long addr, pgoff_t idx)
3699 {
3700         unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
3701                                 svma->vm_start;
3702         unsigned long sbase = saddr & PUD_MASK;
3703         unsigned long s_end = sbase + PUD_SIZE;
3704 
3705         /* Allow segments to share if only one is marked locked */
3706         unsigned long vm_flags = vma->vm_flags & ~VM_LOCKED;
3707         unsigned long svm_flags = svma->vm_flags & ~VM_LOCKED;
3708 
3709         /*
3710          * match the virtual addresses, permission and the alignment of the
3711          * page table page.
3712          */
3713         if (pmd_index(addr) != pmd_index(saddr) ||
3714             vm_flags != svm_flags ||
3715             sbase < svma->vm_start || svma->vm_end < s_end)
3716                 return 0;
3717 
3718         return saddr;
3719 }
3720 
3721 static int vma_shareable(struct vm_area_struct *vma, unsigned long addr)
3722 {
3723         unsigned long base = addr & PUD_MASK;
3724         unsigned long end = base + PUD_SIZE;
3725 
3726         /*
3727          * check on proper vm_flags and page table alignment
3728          */
3729         if (vma->vm_flags & VM_MAYSHARE &&
3730             vma->vm_start <= base && end <= vma->vm_end)
3731                 return 1;
3732         return 0;
3733 }
3734 
3735 /*
3736  * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
3737  * and returns the corresponding pte. While this is not necessary for the
3738  * !shared pmd case because we can allocate the pmd later as well, it makes the
3739  * code much cleaner. pmd allocation is essential for the shared case because
3740  * pud has to be populated inside the same i_mmap_rwsem section - otherwise
3741  * racing tasks could either miss the sharing (see huge_pte_offset) or select a
3742  * bad pmd for sharing.
3743  */
3744 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
3745 {
3746         struct vm_area_struct *vma = find_vma(mm, addr);
3747         struct address_space *mapping = vma->vm_file->f_mapping;
3748         pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
3749                         vma->vm_pgoff;
3750         struct vm_area_struct *svma;
3751         unsigned long saddr;
3752         pte_t *spte = NULL;
3753         pte_t *pte;
3754         spinlock_t *ptl;
3755 
3756         if (!vma_shareable(vma, addr))
3757                 return (pte_t *)pmd_alloc(mm, pud, addr);
3758 
3759         i_mmap_lock_write(mapping);
3760         vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
3761                 if (svma == vma)
3762                         continue;
3763 
3764                 saddr = page_table_shareable(svma, vma, addr, idx);
3765                 if (saddr) {
3766                         spte = huge_pte_offset(svma->vm_mm, saddr);
3767                         if (spte) {
3768                                 mm_inc_nr_pmds(mm);
3769                                 get_page(virt_to_page(spte));
3770                                 break;
3771                         }
3772                 }
3773         }
3774 
3775         if (!spte)
3776                 goto out;
3777 
3778         ptl = huge_pte_lockptr(hstate_vma(vma), mm, spte);
3779         spin_lock(ptl);
3780         if (pud_none(*pud)) {
3781                 pud_populate(mm, pud,
3782                                 (pmd_t *)((unsigned long)spte & PAGE_MASK));
3783         } else {
3784                 put_page(virt_to_page(spte));
3785                 mm_inc_nr_pmds(mm);
3786         }
3787         spin_unlock(ptl);
3788 out:
3789         pte = (pte_t *)pmd_alloc(mm, pud, addr);
3790         i_mmap_unlock_write(mapping);
3791         return pte;
3792 }
3793 
3794 /*
3795  * unmap huge page backed by shared pte.
3796  *
3797  * Hugetlb pte page is ref counted at the time of mapping.  If pte is shared
3798  * indicated by page_count > 1, unmap is achieved by clearing pud and
3799  * decrementing the ref count. If count == 1, the pte page is not shared.
3800  *
3801  * called with page table lock held.
3802  *
3803  * returns: 1 successfully unmapped a shared pte page
3804  *          0 the underlying pte page is not shared, or it is the last user
3805  */
3806 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
3807 {
3808         pgd_t *pgd = pgd_offset(mm, *addr);
3809         pud_t *pud = pud_offset(pgd, *addr);
3810 
3811         BUG_ON(page_count(virt_to_page(ptep)) == 0);
3812         if (page_count(virt_to_page(ptep)) == 1)
3813                 return 0;
3814 
3815         pud_clear(pud);
3816         put_page(virt_to_page(ptep));
3817         mm_dec_nr_pmds(mm);
3818         *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
3819         return 1;
3820 }
3821 #define want_pmd_share()        (1)
3822 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
3823 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
3824 {
3825         return NULL;
3826 }
3827 #define want_pmd_share()        (0)
3828 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
3829 
3830 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
3831 pte_t *huge_pte_alloc(struct mm_struct *mm,
3832                         unsigned long addr, unsigned long sz)
3833 {
3834         pgd_t *pgd;
3835         pud_t *pud;
3836         pte_t *pte = NULL;
3837 
3838         pgd = pgd_offset(mm, addr);
3839         pud = pud_alloc(mm, pgd, addr);
3840         if (pud) {
3841                 if (sz == PUD_SIZE) {
3842                         pte = (pte_t *)pud;
3843                 } else {
3844                         BUG_ON(sz != PMD_SIZE);
3845                         if (want_pmd_share() && pud_none(*pud))
3846                                 pte = huge_pmd_share(mm, addr, pud);
3847                         else
3848                                 pte = (pte_t *)pmd_alloc(mm, pud, addr);
3849                 }
3850         }
3851         BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte));
3852 
3853         return pte;
3854 }
3855 
3856 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
3857 {
3858         pgd_t *pgd;
3859         pud_t *pud;
3860         pmd_t *pmd = NULL;
3861 
3862         pgd = pgd_offset(mm, addr);
3863         if (pgd_present(*pgd)) {
3864                 pud = pud_offset(pgd, addr);
3865                 if (pud_present(*pud)) {
3866                         if (pud_huge(*pud))
3867                                 return (pte_t *)pud;
3868                         pmd = pmd_offset(pud, addr);
3869                 }
3870         }
3871         return (pte_t *) pmd;
3872 }
3873 
3874 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
3875 
3876 /*
3877  * These functions are overwritable if your architecture needs its own
3878  * behavior.
3879  */
3880 struct page * __weak
3881 follow_huge_addr(struct mm_struct *mm, unsigned long address,
3882                               int write)
3883 {
3884         return ERR_PTR(-EINVAL);
3885 }
3886 
3887 struct page * __weak
3888 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
3889                 pmd_t *pmd, int flags)
3890 {
3891         struct page *page = NULL;
3892         spinlock_t *ptl;
3893         pte_t pte;
3894 retry:
3895         ptl = pmd_lockptr(mm, pmd);
3896         spin_lock(ptl);
3897         /*
3898          * make sure that the address range covered by this pmd is not
3899          * unmapped from other threads.
3900          */
3901         if (!pmd_huge(*pmd))
3902                 goto out;
3903         pte = huge_ptep_get((pte_t *)pmd);
3904         if (pte_present(pte)) {
3905                 page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT);
3906                 if (flags & FOLL_GET)
3907                         get_page(page);
3908         } else {
3909                 if (is_hugetlb_entry_migration(pte)) {
3910                         spin_unlock(ptl);
3911                         __migration_entry_wait(mm, (pte_t *)pmd, ptl);
3912                         goto retry;
3913                 }
3914                 /*
3915                  * hwpoisoned entry is treated as no_page_table in
3916                  * follow_page_mask().
3917                  */
3918         }
3919 out:
3920         spin_unlock(ptl);
3921         return page;
3922 }
3923 
3924 struct page * __weak
3925 follow_huge_pud(struct mm_struct *mm, unsigned long address,
3926                 pud_t *pud, int flags)
3927 {
3928         if (flags & FOLL_GET)
3929                 return NULL;
3930 
3931         return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
3932 }
3933 
3934 #ifdef CONFIG_MEMORY_FAILURE
3935 
3936 /*
3937  * This function is called from memory failure code.
3938  * Assume the caller holds page lock of the head page.
3939  */
3940 int dequeue_hwpoisoned_huge_page(struct page *hpage)
3941 {
3942         struct hstate *h = page_hstate(hpage);
3943         int nid = page_to_nid(hpage);
3944         int ret = -EBUSY;
3945 
3946         spin_lock(&hugetlb_lock);
3947         /*
3948          * Just checking !page_huge_active is not enough, because that could be
3949          * an isolated/hwpoisoned hugepage (which have >0 refcount).
3950          */
3951         if (!page_huge_active(hpage) && !page_count(hpage)) {
3952                 /*
3953                  * Hwpoisoned hugepage isn't linked to activelist or freelist,
3954                  * but dangling hpage->lru can trigger list-debug warnings
3955                  * (this happens when we call unpoison_memory() on it),
3956                  * so let it point to itself with list_del_init().
3957                  */
3958                 list_del_init(&hpage->lru);
3959                 set_page_refcounted(hpage);
3960                 h->free_huge_pages--;
3961                 h->free_huge_pages_node[nid]--;
3962                 ret = 0;
3963         }
3964         spin_unlock(&hugetlb_lock);
3965         return ret;
3966 }
3967 #endif
3968 
3969 bool isolate_huge_page(struct page *page, struct list_head *list)
3970 {
3971         bool ret = true;
3972 
3973         VM_BUG_ON_PAGE(!PageHead(page), page);
3974         spin_lock(&hugetlb_lock);
3975         if (!page_huge_active(page) || !get_page_unless_zero(page)) {
3976                 ret = false;
3977                 goto unlock;
3978         }
3979         clear_page_huge_active(page);
3980         list_move_tail(&page->lru, list);
3981 unlock:
3982         spin_unlock(&hugetlb_lock);
3983         return ret;
3984 }
3985 
3986 void putback_active_hugepage(struct page *page)
3987 {
3988         VM_BUG_ON_PAGE(!PageHead(page), page);
3989         spin_lock(&hugetlb_lock);
3990         set_page_huge_active(page);
3991         list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
3992         spin_unlock(&hugetlb_lock);
3993         put_page(page);
3994 }
3995 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp