~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/hugetlb.c

Version: ~ [ linux-5.18-rc6 ] ~ [ linux-5.17.6 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.38 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.114 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.192 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.241 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.277 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.312 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Generic hugetlb support.
  3  * (C) Nadia Yvette Chambers, April 2004
  4  */
  5 #include <linux/list.h>
  6 #include <linux/init.h>
  7 #include <linux/mm.h>
  8 #include <linux/seq_file.h>
  9 #include <linux/sysctl.h>
 10 #include <linux/highmem.h>
 11 #include <linux/mmu_notifier.h>
 12 #include <linux/nodemask.h>
 13 #include <linux/pagemap.h>
 14 #include <linux/mempolicy.h>
 15 #include <linux/compiler.h>
 16 #include <linux/cpuset.h>
 17 #include <linux/mutex.h>
 18 #include <linux/bootmem.h>
 19 #include <linux/sysfs.h>
 20 #include <linux/slab.h>
 21 #include <linux/rmap.h>
 22 #include <linux/swap.h>
 23 #include <linux/swapops.h>
 24 #include <linux/page-isolation.h>
 25 #include <linux/jhash.h>
 26 
 27 #include <asm/page.h>
 28 #include <asm/pgtable.h>
 29 #include <asm/tlb.h>
 30 
 31 #include <linux/io.h>
 32 #include <linux/hugetlb.h>
 33 #include <linux/hugetlb_cgroup.h>
 34 #include <linux/node.h>
 35 #include "internal.h"
 36 
 37 int hugepages_treat_as_movable;
 38 
 39 int hugetlb_max_hstate __read_mostly;
 40 unsigned int default_hstate_idx;
 41 struct hstate hstates[HUGE_MAX_HSTATE];
 42 /*
 43  * Minimum page order among possible hugepage sizes, set to a proper value
 44  * at boot time.
 45  */
 46 static unsigned int minimum_order __read_mostly = UINT_MAX;
 47 
 48 __initdata LIST_HEAD(huge_boot_pages);
 49 
 50 /* for command line parsing */
 51 static struct hstate * __initdata parsed_hstate;
 52 static unsigned long __initdata default_hstate_max_huge_pages;
 53 static unsigned long __initdata default_hstate_size;
 54 static bool __initdata parsed_valid_hugepagesz = true;
 55 
 56 /*
 57  * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
 58  * free_huge_pages, and surplus_huge_pages.
 59  */
 60 DEFINE_SPINLOCK(hugetlb_lock);
 61 
 62 /*
 63  * Serializes faults on the same logical page.  This is used to
 64  * prevent spurious OOMs when the hugepage pool is fully utilized.
 65  */
 66 static int num_fault_mutexes;
 67 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
 68 
 69 /* Forward declaration */
 70 static int hugetlb_acct_memory(struct hstate *h, long delta);
 71 
 72 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
 73 {
 74         bool free = (spool->count == 0) && (spool->used_hpages == 0);
 75 
 76         spin_unlock(&spool->lock);
 77 
 78         /* If no pages are used, and no other handles to the subpool
 79          * remain, give up any reservations mased on minimum size and
 80          * free the subpool */
 81         if (free) {
 82                 if (spool->min_hpages != -1)
 83                         hugetlb_acct_memory(spool->hstate,
 84                                                 -spool->min_hpages);
 85                 kfree(spool);
 86         }
 87 }
 88 
 89 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
 90                                                 long min_hpages)
 91 {
 92         struct hugepage_subpool *spool;
 93 
 94         spool = kzalloc(sizeof(*spool), GFP_KERNEL);
 95         if (!spool)
 96                 return NULL;
 97 
 98         spin_lock_init(&spool->lock);
 99         spool->count = 1;
100         spool->max_hpages = max_hpages;
101         spool->hstate = h;
102         spool->min_hpages = min_hpages;
103 
104         if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
105                 kfree(spool);
106                 return NULL;
107         }
108         spool->rsv_hpages = min_hpages;
109 
110         return spool;
111 }
112 
113 void hugepage_put_subpool(struct hugepage_subpool *spool)
114 {
115         spin_lock(&spool->lock);
116         BUG_ON(!spool->count);
117         spool->count--;
118         unlock_or_release_subpool(spool);
119 }
120 
121 /*
122  * Subpool accounting for allocating and reserving pages.
123  * Return -ENOMEM if there are not enough resources to satisfy the
124  * the request.  Otherwise, return the number of pages by which the
125  * global pools must be adjusted (upward).  The returned value may
126  * only be different than the passed value (delta) in the case where
127  * a subpool minimum size must be manitained.
128  */
129 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
130                                       long delta)
131 {
132         long ret = delta;
133 
134         if (!spool)
135                 return ret;
136 
137         spin_lock(&spool->lock);
138 
139         if (spool->max_hpages != -1) {          /* maximum size accounting */
140                 if ((spool->used_hpages + delta) <= spool->max_hpages)
141                         spool->used_hpages += delta;
142                 else {
143                         ret = -ENOMEM;
144                         goto unlock_ret;
145                 }
146         }
147 
148         /* minimum size accounting */
149         if (spool->min_hpages != -1 && spool->rsv_hpages) {
150                 if (delta > spool->rsv_hpages) {
151                         /*
152                          * Asking for more reserves than those already taken on
153                          * behalf of subpool.  Return difference.
154                          */
155                         ret = delta - spool->rsv_hpages;
156                         spool->rsv_hpages = 0;
157                 } else {
158                         ret = 0;        /* reserves already accounted for */
159                         spool->rsv_hpages -= delta;
160                 }
161         }
162 
163 unlock_ret:
164         spin_unlock(&spool->lock);
165         return ret;
166 }
167 
168 /*
169  * Subpool accounting for freeing and unreserving pages.
170  * Return the number of global page reservations that must be dropped.
171  * The return value may only be different than the passed value (delta)
172  * in the case where a subpool minimum size must be maintained.
173  */
174 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
175                                        long delta)
176 {
177         long ret = delta;
178 
179         if (!spool)
180                 return delta;
181 
182         spin_lock(&spool->lock);
183 
184         if (spool->max_hpages != -1)            /* maximum size accounting */
185                 spool->used_hpages -= delta;
186 
187          /* minimum size accounting */
188         if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {
189                 if (spool->rsv_hpages + delta <= spool->min_hpages)
190                         ret = 0;
191                 else
192                         ret = spool->rsv_hpages + delta - spool->min_hpages;
193 
194                 spool->rsv_hpages += delta;
195                 if (spool->rsv_hpages > spool->min_hpages)
196                         spool->rsv_hpages = spool->min_hpages;
197         }
198 
199         /*
200          * If hugetlbfs_put_super couldn't free spool due to an outstanding
201          * quota reference, free it now.
202          */
203         unlock_or_release_subpool(spool);
204 
205         return ret;
206 }
207 
208 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
209 {
210         return HUGETLBFS_SB(inode->i_sb)->spool;
211 }
212 
213 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
214 {
215         return subpool_inode(file_inode(vma->vm_file));
216 }
217 
218 /*
219  * Region tracking -- allows tracking of reservations and instantiated pages
220  *                    across the pages in a mapping.
221  *
222  * The region data structures are embedded into a resv_map and protected
223  * by a resv_map's lock.  The set of regions within the resv_map represent
224  * reservations for huge pages, or huge pages that have already been
225  * instantiated within the map.  The from and to elements are huge page
226  * indicies into the associated mapping.  from indicates the starting index
227  * of the region.  to represents the first index past the end of  the region.
228  *
229  * For example, a file region structure with from == 0 and to == 4 represents
230  * four huge pages in a mapping.  It is important to note that the to element
231  * represents the first element past the end of the region. This is used in
232  * arithmetic as 4(to) - 0(from) = 4 huge pages in the region.
233  *
234  * Interval notation of the form [from, to) will be used to indicate that
235  * the endpoint from is inclusive and to is exclusive.
236  */
237 struct file_region {
238         struct list_head link;
239         long from;
240         long to;
241 };
242 
243 /*
244  * Add the huge page range represented by [f, t) to the reserve
245  * map.  In the normal case, existing regions will be expanded
246  * to accommodate the specified range.  Sufficient regions should
247  * exist for expansion due to the previous call to region_chg
248  * with the same range.  However, it is possible that region_del
249  * could have been called after region_chg and modifed the map
250  * in such a way that no region exists to be expanded.  In this
251  * case, pull a region descriptor from the cache associated with
252  * the map and use that for the new range.
253  *
254  * Return the number of new huge pages added to the map.  This
255  * number is greater than or equal to zero.
256  */
257 static long region_add(struct resv_map *resv, long f, long t)
258 {
259         struct list_head *head = &resv->regions;
260         struct file_region *rg, *nrg, *trg;
261         long add = 0;
262 
263         spin_lock(&resv->lock);
264         /* Locate the region we are either in or before. */
265         list_for_each_entry(rg, head, link)
266                 if (f <= rg->to)
267                         break;
268 
269         /*
270          * If no region exists which can be expanded to include the
271          * specified range, the list must have been modified by an
272          * interleving call to region_del().  Pull a region descriptor
273          * from the cache and use it for this range.
274          */
275         if (&rg->link == head || t < rg->from) {
276                 VM_BUG_ON(resv->region_cache_count <= 0);
277 
278                 resv->region_cache_count--;
279                 nrg = list_first_entry(&resv->region_cache, struct file_region,
280                                         link);
281                 list_del(&nrg->link);
282 
283                 nrg->from = f;
284                 nrg->to = t;
285                 list_add(&nrg->link, rg->link.prev);
286 
287                 add += t - f;
288                 goto out_locked;
289         }
290 
291         /* Round our left edge to the current segment if it encloses us. */
292         if (f > rg->from)
293                 f = rg->from;
294 
295         /* Check for and consume any regions we now overlap with. */
296         nrg = rg;
297         list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
298                 if (&rg->link == head)
299                         break;
300                 if (rg->from > t)
301                         break;
302 
303                 /* If this area reaches higher then extend our area to
304                  * include it completely.  If this is not the first area
305                  * which we intend to reuse, free it. */
306                 if (rg->to > t)
307                         t = rg->to;
308                 if (rg != nrg) {
309                         /* Decrement return value by the deleted range.
310                          * Another range will span this area so that by
311                          * end of routine add will be >= zero
312                          */
313                         add -= (rg->to - rg->from);
314                         list_del(&rg->link);
315                         kfree(rg);
316                 }
317         }
318 
319         add += (nrg->from - f);         /* Added to beginning of region */
320         nrg->from = f;
321         add += t - nrg->to;             /* Added to end of region */
322         nrg->to = t;
323 
324 out_locked:
325         resv->adds_in_progress--;
326         spin_unlock(&resv->lock);
327         VM_BUG_ON(add < 0);
328         return add;
329 }
330 
331 /*
332  * Examine the existing reserve map and determine how many
333  * huge pages in the specified range [f, t) are NOT currently
334  * represented.  This routine is called before a subsequent
335  * call to region_add that will actually modify the reserve
336  * map to add the specified range [f, t).  region_chg does
337  * not change the number of huge pages represented by the
338  * map.  However, if the existing regions in the map can not
339  * be expanded to represent the new range, a new file_region
340  * structure is added to the map as a placeholder.  This is
341  * so that the subsequent region_add call will have all the
342  * regions it needs and will not fail.
343  *
344  * Upon entry, region_chg will also examine the cache of region descriptors
345  * associated with the map.  If there are not enough descriptors cached, one
346  * will be allocated for the in progress add operation.
347  *
348  * Returns the number of huge pages that need to be added to the existing
349  * reservation map for the range [f, t).  This number is greater or equal to
350  * zero.  -ENOMEM is returned if a new file_region structure or cache entry
351  * is needed and can not be allocated.
352  */
353 static long region_chg(struct resv_map *resv, long f, long t)
354 {
355         struct list_head *head = &resv->regions;
356         struct file_region *rg, *nrg = NULL;
357         long chg = 0;
358 
359 retry:
360         spin_lock(&resv->lock);
361 retry_locked:
362         resv->adds_in_progress++;
363 
364         /*
365          * Check for sufficient descriptors in the cache to accommodate
366          * the number of in progress add operations.
367          */
368         if (resv->adds_in_progress > resv->region_cache_count) {
369                 struct file_region *trg;
370 
371                 VM_BUG_ON(resv->adds_in_progress - resv->region_cache_count > 1);
372                 /* Must drop lock to allocate a new descriptor. */
373                 resv->adds_in_progress--;
374                 spin_unlock(&resv->lock);
375 
376                 trg = kmalloc(sizeof(*trg), GFP_KERNEL);
377                 if (!trg) {
378                         kfree(nrg);
379                         return -ENOMEM;
380                 }
381 
382                 spin_lock(&resv->lock);
383                 list_add(&trg->link, &resv->region_cache);
384                 resv->region_cache_count++;
385                 goto retry_locked;
386         }
387 
388         /* Locate the region we are before or in. */
389         list_for_each_entry(rg, head, link)
390                 if (f <= rg->to)
391                         break;
392 
393         /* If we are below the current region then a new region is required.
394          * Subtle, allocate a new region at the position but make it zero
395          * size such that we can guarantee to record the reservation. */
396         if (&rg->link == head || t < rg->from) {
397                 if (!nrg) {
398                         resv->adds_in_progress--;
399                         spin_unlock(&resv->lock);
400                         nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
401                         if (!nrg)
402                                 return -ENOMEM;
403 
404                         nrg->from = f;
405                         nrg->to   = f;
406                         INIT_LIST_HEAD(&nrg->link);
407                         goto retry;
408                 }
409 
410                 list_add(&nrg->link, rg->link.prev);
411                 chg = t - f;
412                 goto out_nrg;
413         }
414 
415         /* Round our left edge to the current segment if it encloses us. */
416         if (f > rg->from)
417                 f = rg->from;
418         chg = t - f;
419 
420         /* Check for and consume any regions we now overlap with. */
421         list_for_each_entry(rg, rg->link.prev, link) {
422                 if (&rg->link == head)
423                         break;
424                 if (rg->from > t)
425                         goto out;
426 
427                 /* We overlap with this area, if it extends further than
428                  * us then we must extend ourselves.  Account for its
429                  * existing reservation. */
430                 if (rg->to > t) {
431                         chg += rg->to - t;
432                         t = rg->to;
433                 }
434                 chg -= rg->to - rg->from;
435         }
436 
437 out:
438         spin_unlock(&resv->lock);
439         /*  We already know we raced and no longer need the new region */
440         kfree(nrg);
441         return chg;
442 out_nrg:
443         spin_unlock(&resv->lock);
444         return chg;
445 }
446 
447 /*
448  * Abort the in progress add operation.  The adds_in_progress field
449  * of the resv_map keeps track of the operations in progress between
450  * calls to region_chg and region_add.  Operations are sometimes
451  * aborted after the call to region_chg.  In such cases, region_abort
452  * is called to decrement the adds_in_progress counter.
453  *
454  * NOTE: The range arguments [f, t) are not needed or used in this
455  * routine.  They are kept to make reading the calling code easier as
456  * arguments will match the associated region_chg call.
457  */
458 static void region_abort(struct resv_map *resv, long f, long t)
459 {
460         spin_lock(&resv->lock);
461         VM_BUG_ON(!resv->region_cache_count);
462         resv->adds_in_progress--;
463         spin_unlock(&resv->lock);
464 }
465 
466 /*
467  * Delete the specified range [f, t) from the reserve map.  If the
468  * t parameter is LONG_MAX, this indicates that ALL regions after f
469  * should be deleted.  Locate the regions which intersect [f, t)
470  * and either trim, delete or split the existing regions.
471  *
472  * Returns the number of huge pages deleted from the reserve map.
473  * In the normal case, the return value is zero or more.  In the
474  * case where a region must be split, a new region descriptor must
475  * be allocated.  If the allocation fails, -ENOMEM will be returned.
476  * NOTE: If the parameter t == LONG_MAX, then we will never split
477  * a region and possibly return -ENOMEM.  Callers specifying
478  * t == LONG_MAX do not need to check for -ENOMEM error.
479  */
480 static long region_del(struct resv_map *resv, long f, long t)
481 {
482         struct list_head *head = &resv->regions;
483         struct file_region *rg, *trg;
484         struct file_region *nrg = NULL;
485         long del = 0;
486 
487 retry:
488         spin_lock(&resv->lock);
489         list_for_each_entry_safe(rg, trg, head, link) {
490                 /*
491                  * Skip regions before the range to be deleted.  file_region
492                  * ranges are normally of the form [from, to).  However, there
493                  * may be a "placeholder" entry in the map which is of the form
494                  * (from, to) with from == to.  Check for placeholder entries
495                  * at the beginning of the range to be deleted.
496                  */
497                 if (rg->to <= f && (rg->to != rg->from || rg->to != f))
498                         continue;
499 
500                 if (rg->from >= t)
501                         break;
502 
503                 if (f > rg->from && t < rg->to) { /* Must split region */
504                         /*
505                          * Check for an entry in the cache before dropping
506                          * lock and attempting allocation.
507                          */
508                         if (!nrg &&
509                             resv->region_cache_count > resv->adds_in_progress) {
510                                 nrg = list_first_entry(&resv->region_cache,
511                                                         struct file_region,
512                                                         link);
513                                 list_del(&nrg->link);
514                                 resv->region_cache_count--;
515                         }
516 
517                         if (!nrg) {
518                                 spin_unlock(&resv->lock);
519                                 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
520                                 if (!nrg)
521                                         return -ENOMEM;
522                                 goto retry;
523                         }
524 
525                         del += t - f;
526 
527                         /* New entry for end of split region */
528                         nrg->from = t;
529                         nrg->to = rg->to;
530                         INIT_LIST_HEAD(&nrg->link);
531 
532                         /* Original entry is trimmed */
533                         rg->to = f;
534 
535                         list_add(&nrg->link, &rg->link);
536                         nrg = NULL;
537                         break;
538                 }
539 
540                 if (f <= rg->from && t >= rg->to) { /* Remove entire region */
541                         del += rg->to - rg->from;
542                         list_del(&rg->link);
543                         kfree(rg);
544                         continue;
545                 }
546 
547                 if (f <= rg->from) {    /* Trim beginning of region */
548                         del += t - rg->from;
549                         rg->from = t;
550                 } else {                /* Trim end of region */
551                         del += rg->to - f;
552                         rg->to = f;
553                 }
554         }
555 
556         spin_unlock(&resv->lock);
557         kfree(nrg);
558         return del;
559 }
560 
561 /*
562  * A rare out of memory error was encountered which prevented removal of
563  * the reserve map region for a page.  The huge page itself was free'ed
564  * and removed from the page cache.  This routine will adjust the subpool
565  * usage count, and the global reserve count if needed.  By incrementing
566  * these counts, the reserve map entry which could not be deleted will
567  * appear as a "reserved" entry instead of simply dangling with incorrect
568  * counts.
569  */
570 void hugetlb_fix_reserve_counts(struct inode *inode, bool restore_reserve)
571 {
572         struct hugepage_subpool *spool = subpool_inode(inode);
573         long rsv_adjust;
574 
575         rsv_adjust = hugepage_subpool_get_pages(spool, 1);
576         if (restore_reserve && rsv_adjust) {
577                 struct hstate *h = hstate_inode(inode);
578 
579                 hugetlb_acct_memory(h, 1);
580         }
581 }
582 
583 /*
584  * Count and return the number of huge pages in the reserve map
585  * that intersect with the range [f, t).
586  */
587 static long region_count(struct resv_map *resv, long f, long t)
588 {
589         struct list_head *head = &resv->regions;
590         struct file_region *rg;
591         long chg = 0;
592 
593         spin_lock(&resv->lock);
594         /* Locate each segment we overlap with, and count that overlap. */
595         list_for_each_entry(rg, head, link) {
596                 long seg_from;
597                 long seg_to;
598 
599                 if (rg->to <= f)
600                         continue;
601                 if (rg->from >= t)
602                         break;
603 
604                 seg_from = max(rg->from, f);
605                 seg_to = min(rg->to, t);
606 
607                 chg += seg_to - seg_from;
608         }
609         spin_unlock(&resv->lock);
610 
611         return chg;
612 }
613 
614 /*
615  * Convert the address within this vma to the page offset within
616  * the mapping, in pagecache page units; huge pages here.
617  */
618 static pgoff_t vma_hugecache_offset(struct hstate *h,
619                         struct vm_area_struct *vma, unsigned long address)
620 {
621         return ((address - vma->vm_start) >> huge_page_shift(h)) +
622                         (vma->vm_pgoff >> huge_page_order(h));
623 }
624 
625 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
626                                      unsigned long address)
627 {
628         return vma_hugecache_offset(hstate_vma(vma), vma, address);
629 }
630 EXPORT_SYMBOL_GPL(linear_hugepage_index);
631 
632 /*
633  * Return the size of the pages allocated when backing a VMA. In the majority
634  * cases this will be same size as used by the page table entries.
635  */
636 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
637 {
638         struct hstate *hstate;
639 
640         if (!is_vm_hugetlb_page(vma))
641                 return PAGE_SIZE;
642 
643         hstate = hstate_vma(vma);
644 
645         return 1UL << huge_page_shift(hstate);
646 }
647 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
648 
649 /*
650  * Return the page size being used by the MMU to back a VMA. In the majority
651  * of cases, the page size used by the kernel matches the MMU size. On
652  * architectures where it differs, an architecture-specific version of this
653  * function is required.
654  */
655 #ifndef vma_mmu_pagesize
656 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
657 {
658         return vma_kernel_pagesize(vma);
659 }
660 #endif
661 
662 /*
663  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
664  * bits of the reservation map pointer, which are always clear due to
665  * alignment.
666  */
667 #define HPAGE_RESV_OWNER    (1UL << 0)
668 #define HPAGE_RESV_UNMAPPED (1UL << 1)
669 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
670 
671 /*
672  * These helpers are used to track how many pages are reserved for
673  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
674  * is guaranteed to have their future faults succeed.
675  *
676  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
677  * the reserve counters are updated with the hugetlb_lock held. It is safe
678  * to reset the VMA at fork() time as it is not in use yet and there is no
679  * chance of the global counters getting corrupted as a result of the values.
680  *
681  * The private mapping reservation is represented in a subtly different
682  * manner to a shared mapping.  A shared mapping has a region map associated
683  * with the underlying file, this region map represents the backing file
684  * pages which have ever had a reservation assigned which this persists even
685  * after the page is instantiated.  A private mapping has a region map
686  * associated with the original mmap which is attached to all VMAs which
687  * reference it, this region map represents those offsets which have consumed
688  * reservation ie. where pages have been instantiated.
689  */
690 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
691 {
692         return (unsigned long)vma->vm_private_data;
693 }
694 
695 static void set_vma_private_data(struct vm_area_struct *vma,
696                                                         unsigned long value)
697 {
698         vma->vm_private_data = (void *)value;
699 }
700 
701 struct resv_map *resv_map_alloc(void)
702 {
703         struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
704         struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
705 
706         if (!resv_map || !rg) {
707                 kfree(resv_map);
708                 kfree(rg);
709                 return NULL;
710         }
711 
712         kref_init(&resv_map->refs);
713         spin_lock_init(&resv_map->lock);
714         INIT_LIST_HEAD(&resv_map->regions);
715 
716         resv_map->adds_in_progress = 0;
717 
718         INIT_LIST_HEAD(&resv_map->region_cache);
719         list_add(&rg->link, &resv_map->region_cache);
720         resv_map->region_cache_count = 1;
721 
722         return resv_map;
723 }
724 
725 void resv_map_release(struct kref *ref)
726 {
727         struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
728         struct list_head *head = &resv_map->region_cache;
729         struct file_region *rg, *trg;
730 
731         /* Clear out any active regions before we release the map. */
732         region_del(resv_map, 0, LONG_MAX);
733 
734         /* ... and any entries left in the cache */
735         list_for_each_entry_safe(rg, trg, head, link) {
736                 list_del(&rg->link);
737                 kfree(rg);
738         }
739 
740         VM_BUG_ON(resv_map->adds_in_progress);
741 
742         kfree(resv_map);
743 }
744 
745 static inline struct resv_map *inode_resv_map(struct inode *inode)
746 {
747         return inode->i_mapping->private_data;
748 }
749 
750 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
751 {
752         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
753         if (vma->vm_flags & VM_MAYSHARE) {
754                 struct address_space *mapping = vma->vm_file->f_mapping;
755                 struct inode *inode = mapping->host;
756 
757                 return inode_resv_map(inode);
758 
759         } else {
760                 return (struct resv_map *)(get_vma_private_data(vma) &
761                                                         ~HPAGE_RESV_MASK);
762         }
763 }
764 
765 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
766 {
767         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
768         VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
769 
770         set_vma_private_data(vma, (get_vma_private_data(vma) &
771                                 HPAGE_RESV_MASK) | (unsigned long)map);
772 }
773 
774 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
775 {
776         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
777         VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
778 
779         set_vma_private_data(vma, get_vma_private_data(vma) | flags);
780 }
781 
782 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
783 {
784         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
785 
786         return (get_vma_private_data(vma) & flag) != 0;
787 }
788 
789 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
790 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
791 {
792         VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
793         if (!(vma->vm_flags & VM_MAYSHARE))
794                 vma->vm_private_data = (void *)0;
795 }
796 
797 /* Returns true if the VMA has associated reserve pages */
798 static bool vma_has_reserves(struct vm_area_struct *vma, long chg)
799 {
800         if (vma->vm_flags & VM_NORESERVE) {
801                 /*
802                  * This address is already reserved by other process(chg == 0),
803                  * so, we should decrement reserved count. Without decrementing,
804                  * reserve count remains after releasing inode, because this
805                  * allocated page will go into page cache and is regarded as
806                  * coming from reserved pool in releasing step.  Currently, we
807                  * don't have any other solution to deal with this situation
808                  * properly, so add work-around here.
809                  */
810                 if (vma->vm_flags & VM_MAYSHARE && chg == 0)
811                         return true;
812                 else
813                         return false;
814         }
815 
816         /* Shared mappings always use reserves */
817         if (vma->vm_flags & VM_MAYSHARE) {
818                 /*
819                  * We know VM_NORESERVE is not set.  Therefore, there SHOULD
820                  * be a region map for all pages.  The only situation where
821                  * there is no region map is if a hole was punched via
822                  * fallocate.  In this case, there really are no reverves to
823                  * use.  This situation is indicated if chg != 0.
824                  */
825                 if (chg)
826                         return false;
827                 else
828                         return true;
829         }
830 
831         /*
832          * Only the process that called mmap() has reserves for
833          * private mappings.
834          */
835         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
836                 /*
837                  * Like the shared case above, a hole punch or truncate
838                  * could have been performed on the private mapping.
839                  * Examine the value of chg to determine if reserves
840                  * actually exist or were previously consumed.
841                  * Very Subtle - The value of chg comes from a previous
842                  * call to vma_needs_reserves().  The reserve map for
843                  * private mappings has different (opposite) semantics
844                  * than that of shared mappings.  vma_needs_reserves()
845                  * has already taken this difference in semantics into
846                  * account.  Therefore, the meaning of chg is the same
847                  * as in the shared case above.  Code could easily be
848                  * combined, but keeping it separate draws attention to
849                  * subtle differences.
850                  */
851                 if (chg)
852                         return false;
853                 else
854                         return true;
855         }
856 
857         return false;
858 }
859 
860 static void enqueue_huge_page(struct hstate *h, struct page *page)
861 {
862         int nid = page_to_nid(page);
863         list_move(&page->lru, &h->hugepage_freelists[nid]);
864         h->free_huge_pages++;
865         h->free_huge_pages_node[nid]++;
866 }
867 
868 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
869 {
870         struct page *page;
871 
872         list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
873                 if (!is_migrate_isolate_page(page))
874                         break;
875         /*
876          * if 'non-isolated free hugepage' not found on the list,
877          * the allocation fails.
878          */
879         if (&h->hugepage_freelists[nid] == &page->lru)
880                 return NULL;
881         list_move(&page->lru, &h->hugepage_activelist);
882         set_page_refcounted(page);
883         h->free_huge_pages--;
884         h->free_huge_pages_node[nid]--;
885         return page;
886 }
887 
888 /* Movability of hugepages depends on migration support. */
889 static inline gfp_t htlb_alloc_mask(struct hstate *h)
890 {
891         if (hugepages_treat_as_movable || hugepage_migration_supported(h))
892                 return GFP_HIGHUSER_MOVABLE;
893         else
894                 return GFP_HIGHUSER;
895 }
896 
897 static struct page *dequeue_huge_page_vma(struct hstate *h,
898                                 struct vm_area_struct *vma,
899                                 unsigned long address, int avoid_reserve,
900                                 long chg)
901 {
902         struct page *page = NULL;
903         struct mempolicy *mpol;
904         nodemask_t *nodemask;
905         struct zonelist *zonelist;
906         struct zone *zone;
907         struct zoneref *z;
908         unsigned int cpuset_mems_cookie;
909 
910         /*
911          * A child process with MAP_PRIVATE mappings created by their parent
912          * have no page reserves. This check ensures that reservations are
913          * not "stolen". The child may still get SIGKILLed
914          */
915         if (!vma_has_reserves(vma, chg) &&
916                         h->free_huge_pages - h->resv_huge_pages == 0)
917                 goto err;
918 
919         /* If reserves cannot be used, ensure enough pages are in the pool */
920         if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
921                 goto err;
922 
923 retry_cpuset:
924         cpuset_mems_cookie = read_mems_allowed_begin();
925         zonelist = huge_zonelist(vma, address,
926                                         htlb_alloc_mask(h), &mpol, &nodemask);
927 
928         for_each_zone_zonelist_nodemask(zone, z, zonelist,
929                                                 MAX_NR_ZONES - 1, nodemask) {
930                 if (cpuset_zone_allowed(zone, htlb_alloc_mask(h))) {
931                         page = dequeue_huge_page_node(h, zone_to_nid(zone));
932                         if (page) {
933                                 if (avoid_reserve)
934                                         break;
935                                 if (!vma_has_reserves(vma, chg))
936                                         break;
937 
938                                 SetPagePrivate(page);
939                                 h->resv_huge_pages--;
940                                 break;
941                         }
942                 }
943         }
944 
945         mpol_cond_put(mpol);
946         if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
947                 goto retry_cpuset;
948         return page;
949 
950 err:
951         return NULL;
952 }
953 
954 /*
955  * common helper functions for hstate_next_node_to_{alloc|free}.
956  * We may have allocated or freed a huge page based on a different
957  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
958  * be outside of *nodes_allowed.  Ensure that we use an allowed
959  * node for alloc or free.
960  */
961 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
962 {
963         nid = next_node_in(nid, *nodes_allowed);
964         VM_BUG_ON(nid >= MAX_NUMNODES);
965 
966         return nid;
967 }
968 
969 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
970 {
971         if (!node_isset(nid, *nodes_allowed))
972                 nid = next_node_allowed(nid, nodes_allowed);
973         return nid;
974 }
975 
976 /*
977  * returns the previously saved node ["this node"] from which to
978  * allocate a persistent huge page for the pool and advance the
979  * next node from which to allocate, handling wrap at end of node
980  * mask.
981  */
982 static int hstate_next_node_to_alloc(struct hstate *h,
983                                         nodemask_t *nodes_allowed)
984 {
985         int nid;
986 
987         VM_BUG_ON(!nodes_allowed);
988 
989         nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
990         h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
991 
992         return nid;
993 }
994 
995 /*
996  * helper for free_pool_huge_page() - return the previously saved
997  * node ["this node"] from which to free a huge page.  Advance the
998  * next node id whether or not we find a free huge page to free so
999  * that the next attempt to free addresses the next node.
1000  */
1001 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
1002 {
1003         int nid;
1004 
1005         VM_BUG_ON(!nodes_allowed);
1006 
1007         nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
1008         h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
1009 
1010         return nid;
1011 }
1012 
1013 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)           \
1014         for (nr_nodes = nodes_weight(*mask);                            \
1015                 nr_nodes > 0 &&                                         \
1016                 ((node = hstate_next_node_to_alloc(hs, mask)) || 1);    \
1017                 nr_nodes--)
1018 
1019 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask)            \
1020         for (nr_nodes = nodes_weight(*mask);                            \
1021                 nr_nodes > 0 &&                                         \
1022                 ((node = hstate_next_node_to_free(hs, mask)) || 1);     \
1023                 nr_nodes--)
1024 
1025 #if defined(CONFIG_X86_64) && ((defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || defined(CONFIG_CMA))
1026 static void destroy_compound_gigantic_page(struct page *page,
1027                                         unsigned int order)
1028 {
1029         int i;
1030         int nr_pages = 1 << order;
1031         struct page *p = page + 1;
1032 
1033         atomic_set(compound_mapcount_ptr(page), 0);
1034         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1035                 clear_compound_head(p);
1036                 set_page_refcounted(p);
1037         }
1038 
1039         set_compound_order(page, 0);
1040         __ClearPageHead(page);
1041 }
1042 
1043 static void free_gigantic_page(struct page *page, unsigned int order)
1044 {
1045         free_contig_range(page_to_pfn(page), 1 << order);
1046 }
1047 
1048 static int __alloc_gigantic_page(unsigned long start_pfn,
1049                                 unsigned long nr_pages)
1050 {
1051         unsigned long end_pfn = start_pfn + nr_pages;
1052         return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
1053 }
1054 
1055 static bool pfn_range_valid_gigantic(struct zone *z,
1056                         unsigned long start_pfn, unsigned long nr_pages)
1057 {
1058         unsigned long i, end_pfn = start_pfn + nr_pages;
1059         struct page *page;
1060 
1061         for (i = start_pfn; i < end_pfn; i++) {
1062                 if (!pfn_valid(i))
1063                         return false;
1064 
1065                 page = pfn_to_page(i);
1066 
1067                 if (page_zone(page) != z)
1068                         return false;
1069 
1070                 if (PageReserved(page))
1071                         return false;
1072 
1073                 if (page_count(page) > 0)
1074                         return false;
1075 
1076                 if (PageHuge(page))
1077                         return false;
1078         }
1079 
1080         return true;
1081 }
1082 
1083 static bool zone_spans_last_pfn(const struct zone *zone,
1084                         unsigned long start_pfn, unsigned long nr_pages)
1085 {
1086         unsigned long last_pfn = start_pfn + nr_pages - 1;
1087         return zone_spans_pfn(zone, last_pfn);
1088 }
1089 
1090 static struct page *alloc_gigantic_page(int nid, unsigned int order)
1091 {
1092         unsigned long nr_pages = 1 << order;
1093         unsigned long ret, pfn, flags;
1094         struct zone *z;
1095 
1096         z = NODE_DATA(nid)->node_zones;
1097         for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) {
1098                 spin_lock_irqsave(&z->lock, flags);
1099 
1100                 pfn = ALIGN(z->zone_start_pfn, nr_pages);
1101                 while (zone_spans_last_pfn(z, pfn, nr_pages)) {
1102                         if (pfn_range_valid_gigantic(z, pfn, nr_pages)) {
1103                                 /*
1104                                  * We release the zone lock here because
1105                                  * alloc_contig_range() will also lock the zone
1106                                  * at some point. If there's an allocation
1107                                  * spinning on this lock, it may win the race
1108                                  * and cause alloc_contig_range() to fail...
1109                                  */
1110                                 spin_unlock_irqrestore(&z->lock, flags);
1111                                 ret = __alloc_gigantic_page(pfn, nr_pages);
1112                                 if (!ret)
1113                                         return pfn_to_page(pfn);
1114                                 spin_lock_irqsave(&z->lock, flags);
1115                         }
1116                         pfn += nr_pages;
1117                 }
1118 
1119                 spin_unlock_irqrestore(&z->lock, flags);
1120         }
1121 
1122         return NULL;
1123 }
1124 
1125 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
1126 static void prep_compound_gigantic_page(struct page *page, unsigned int order);
1127 
1128 static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid)
1129 {
1130         struct page *page;
1131 
1132         page = alloc_gigantic_page(nid, huge_page_order(h));
1133         if (page) {
1134                 prep_compound_gigantic_page(page, huge_page_order(h));
1135                 prep_new_huge_page(h, page, nid);
1136         }
1137 
1138         return page;
1139 }
1140 
1141 static int alloc_fresh_gigantic_page(struct hstate *h,
1142                                 nodemask_t *nodes_allowed)
1143 {
1144         struct page *page = NULL;
1145         int nr_nodes, node;
1146 
1147         for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1148                 page = alloc_fresh_gigantic_page_node(h, node);
1149                 if (page)
1150                         return 1;
1151         }
1152 
1153         return 0;
1154 }
1155 
1156 static inline bool gigantic_page_supported(void) { return true; }
1157 #else
1158 static inline bool gigantic_page_supported(void) { return false; }
1159 static inline void free_gigantic_page(struct page *page, unsigned int order) { }
1160 static inline void destroy_compound_gigantic_page(struct page *page,
1161                                                 unsigned int order) { }
1162 static inline int alloc_fresh_gigantic_page(struct hstate *h,
1163                                         nodemask_t *nodes_allowed) { return 0; }
1164 #endif
1165 
1166 static void update_and_free_page(struct hstate *h, struct page *page)
1167 {
1168         int i;
1169 
1170         if (hstate_is_gigantic(h) && !gigantic_page_supported())
1171                 return;
1172 
1173         h->nr_huge_pages--;
1174         h->nr_huge_pages_node[page_to_nid(page)]--;
1175         for (i = 0; i < pages_per_huge_page(h); i++) {
1176                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
1177                                 1 << PG_referenced | 1 << PG_dirty |
1178                                 1 << PG_active | 1 << PG_private |
1179                                 1 << PG_writeback);
1180         }
1181         VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
1182         set_compound_page_dtor(page, NULL_COMPOUND_DTOR);
1183         set_page_refcounted(page);
1184         if (hstate_is_gigantic(h)) {
1185                 destroy_compound_gigantic_page(page, huge_page_order(h));
1186                 free_gigantic_page(page, huge_page_order(h));
1187         } else {
1188                 __free_pages(page, huge_page_order(h));
1189         }
1190 }
1191 
1192 struct hstate *size_to_hstate(unsigned long size)
1193 {
1194         struct hstate *h;
1195 
1196         for_each_hstate(h) {
1197                 if (huge_page_size(h) == size)
1198                         return h;
1199         }
1200         return NULL;
1201 }
1202 
1203 /*
1204  * Test to determine whether the hugepage is "active/in-use" (i.e. being linked
1205  * to hstate->hugepage_activelist.)
1206  *
1207  * This function can be called for tail pages, but never returns true for them.
1208  */
1209 bool page_huge_active(struct page *page)
1210 {
1211         VM_BUG_ON_PAGE(!PageHuge(page), page);
1212         return PageHead(page) && PagePrivate(&page[1]);
1213 }
1214 
1215 /* never called for tail page */
1216 static void set_page_huge_active(struct page *page)
1217 {
1218         VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
1219         SetPagePrivate(&page[1]);
1220 }
1221 
1222 static void clear_page_huge_active(struct page *page)
1223 {
1224         VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
1225         ClearPagePrivate(&page[1]);
1226 }
1227 
1228 void free_huge_page(struct page *page)
1229 {
1230         /*
1231          * Can't pass hstate in here because it is called from the
1232          * compound page destructor.
1233          */
1234         struct hstate *h = page_hstate(page);
1235         int nid = page_to_nid(page);
1236         struct hugepage_subpool *spool =
1237                 (struct hugepage_subpool *)page_private(page);
1238         bool restore_reserve;
1239 
1240         set_page_private(page, 0);
1241         page->mapping = NULL;
1242         VM_BUG_ON_PAGE(page_count(page), page);
1243         VM_BUG_ON_PAGE(page_mapcount(page), page);
1244         restore_reserve = PagePrivate(page);
1245         ClearPagePrivate(page);
1246 
1247         /*
1248          * A return code of zero implies that the subpool will be under its
1249          * minimum size if the reservation is not restored after page is free.
1250          * Therefore, force restore_reserve operation.
1251          */
1252         if (hugepage_subpool_put_pages(spool, 1) == 0)
1253                 restore_reserve = true;
1254 
1255         spin_lock(&hugetlb_lock);
1256         clear_page_huge_active(page);
1257         hugetlb_cgroup_uncharge_page(hstate_index(h),
1258                                      pages_per_huge_page(h), page);
1259         if (restore_reserve)
1260                 h->resv_huge_pages++;
1261 
1262         if (h->surplus_huge_pages_node[nid]) {
1263                 /* remove the page from active list */
1264                 list_del(&page->lru);
1265                 update_and_free_page(h, page);
1266                 h->surplus_huge_pages--;
1267                 h->surplus_huge_pages_node[nid]--;
1268         } else {
1269                 arch_clear_hugepage_flags(page);
1270                 enqueue_huge_page(h, page);
1271         }
1272         spin_unlock(&hugetlb_lock);
1273 }
1274 
1275 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
1276 {
1277         INIT_LIST_HEAD(&page->lru);
1278         set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1279         spin_lock(&hugetlb_lock);
1280         set_hugetlb_cgroup(page, NULL);
1281         h->nr_huge_pages++;
1282         h->nr_huge_pages_node[nid]++;
1283         spin_unlock(&hugetlb_lock);
1284         put_page(page); /* free it into the hugepage allocator */
1285 }
1286 
1287 static void prep_compound_gigantic_page(struct page *page, unsigned int order)
1288 {
1289         int i;
1290         int nr_pages = 1 << order;
1291         struct page *p = page + 1;
1292 
1293         /* we rely on prep_new_huge_page to set the destructor */
1294         set_compound_order(page, order);
1295         __ClearPageReserved(page);
1296         __SetPageHead(page);
1297         for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1298                 /*
1299                  * For gigantic hugepages allocated through bootmem at
1300                  * boot, it's safer to be consistent with the not-gigantic
1301                  * hugepages and clear the PG_reserved bit from all tail pages
1302                  * too.  Otherwse drivers using get_user_pages() to access tail
1303                  * pages may get the reference counting wrong if they see
1304                  * PG_reserved set on a tail page (despite the head page not
1305                  * having PG_reserved set).  Enforcing this consistency between
1306                  * head and tail pages allows drivers to optimize away a check
1307                  * on the head page when they need know if put_page() is needed
1308                  * after get_user_pages().
1309                  */
1310                 __ClearPageReserved(p);
1311                 set_page_count(p, 0);
1312                 set_compound_head(p, page);
1313         }
1314         atomic_set(compound_mapcount_ptr(page), -1);
1315 }
1316 
1317 /*
1318  * PageHuge() only returns true for hugetlbfs pages, but not for normal or
1319  * transparent huge pages.  See the PageTransHuge() documentation for more
1320  * details.
1321  */
1322 int PageHuge(struct page *page)
1323 {
1324         if (!PageCompound(page))
1325                 return 0;
1326 
1327         page = compound_head(page);
1328         return page[1].compound_dtor == HUGETLB_PAGE_DTOR;
1329 }
1330 EXPORT_SYMBOL_GPL(PageHuge);
1331 
1332 /*
1333  * PageHeadHuge() only returns true for hugetlbfs head page, but not for
1334  * normal or transparent huge pages.
1335  */
1336 int PageHeadHuge(struct page *page_head)
1337 {
1338         if (!PageHead(page_head))
1339                 return 0;
1340 
1341         return get_compound_page_dtor(page_head) == free_huge_page;
1342 }
1343 
1344 pgoff_t __basepage_index(struct page *page)
1345 {
1346         struct page *page_head = compound_head(page);
1347         pgoff_t index = page_index(page_head);
1348         unsigned long compound_idx;
1349 
1350         if (!PageHuge(page_head))
1351                 return page_index(page);
1352 
1353         if (compound_order(page_head) >= MAX_ORDER)
1354                 compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
1355         else
1356                 compound_idx = page - page_head;
1357 
1358         return (index << compound_order(page_head)) + compound_idx;
1359 }
1360 
1361 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
1362 {
1363         struct page *page;
1364 
1365         page = __alloc_pages_node(nid,
1366                 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
1367                                                 __GFP_REPEAT|__GFP_NOWARN,
1368                 huge_page_order(h));
1369         if (page) {
1370                 prep_new_huge_page(h, page, nid);
1371         }
1372 
1373         return page;
1374 }
1375 
1376 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
1377 {
1378         struct page *page;
1379         int nr_nodes, node;
1380         int ret = 0;
1381 
1382         for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1383                 page = alloc_fresh_huge_page_node(h, node);
1384                 if (page) {
1385                         ret = 1;
1386                         break;
1387                 }
1388         }
1389 
1390         if (ret)
1391                 count_vm_event(HTLB_BUDDY_PGALLOC);
1392         else
1393                 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1394 
1395         return ret;
1396 }
1397 
1398 /*
1399  * Free huge page from pool from next node to free.
1400  * Attempt to keep persistent huge pages more or less
1401  * balanced over allowed nodes.
1402  * Called with hugetlb_lock locked.
1403  */
1404 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1405                                                          bool acct_surplus)
1406 {
1407         int nr_nodes, node;
1408         int ret = 0;
1409 
1410         for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1411                 /*
1412                  * If we're returning unused surplus pages, only examine
1413                  * nodes with surplus pages.
1414                  */
1415                 if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
1416                     !list_empty(&h->hugepage_freelists[node])) {
1417                         struct page *page =
1418                                 list_entry(h->hugepage_freelists[node].next,
1419                                           struct page, lru);
1420                         list_del(&page->lru);
1421                         h->free_huge_pages--;
1422                         h->free_huge_pages_node[node]--;
1423                         if (acct_surplus) {
1424                                 h->surplus_huge_pages--;
1425                                 h->surplus_huge_pages_node[node]--;
1426                         }
1427                         update_and_free_page(h, page);
1428                         ret = 1;
1429                         break;
1430                 }
1431         }
1432 
1433         return ret;
1434 }
1435 
1436 /*
1437  * Dissolve a given free hugepage into free buddy pages. This function does
1438  * nothing for in-use (including surplus) hugepages.
1439  */
1440 static void dissolve_free_huge_page(struct page *page)
1441 {
1442         spin_lock(&hugetlb_lock);
1443         if (PageHuge(page) && !page_count(page)) {
1444                 struct hstate *h = page_hstate(page);
1445                 int nid = page_to_nid(page);
1446                 list_del(&page->lru);
1447                 h->free_huge_pages--;
1448                 h->free_huge_pages_node[nid]--;
1449                 update_and_free_page(h, page);
1450         }
1451         spin_unlock(&hugetlb_lock);
1452 }
1453 
1454 /*
1455  * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
1456  * make specified memory blocks removable from the system.
1457  * Note that start_pfn should aligned with (minimum) hugepage size.
1458  */
1459 void dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
1460 {
1461         unsigned long pfn;
1462 
1463         if (!hugepages_supported())
1464                 return;
1465 
1466         VM_BUG_ON(!IS_ALIGNED(start_pfn, 1 << minimum_order));
1467         for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order)
1468                 dissolve_free_huge_page(pfn_to_page(pfn));
1469 }
1470 
1471 /*
1472  * There are 3 ways this can get called:
1473  * 1. With vma+addr: we use the VMA's memory policy
1474  * 2. With !vma, but nid=NUMA_NO_NODE:  We try to allocate a huge
1475  *    page from any node, and let the buddy allocator itself figure
1476  *    it out.
1477  * 3. With !vma, but nid!=NUMA_NO_NODE.  We allocate a huge page
1478  *    strictly from 'nid'
1479  */
1480 static struct page *__hugetlb_alloc_buddy_huge_page(struct hstate *h,
1481                 struct vm_area_struct *vma, unsigned long addr, int nid)
1482 {
1483         int order = huge_page_order(h);
1484         gfp_t gfp = htlb_alloc_mask(h)|__GFP_COMP|__GFP_REPEAT|__GFP_NOWARN;
1485         unsigned int cpuset_mems_cookie;
1486 
1487         /*
1488          * We need a VMA to get a memory policy.  If we do not
1489          * have one, we use the 'nid' argument.
1490          *
1491          * The mempolicy stuff below has some non-inlined bits
1492          * and calls ->vm_ops.  That makes it hard to optimize at
1493          * compile-time, even when NUMA is off and it does
1494          * nothing.  This helps the compiler optimize it out.
1495          */
1496         if (!IS_ENABLED(CONFIG_NUMA) || !vma) {
1497                 /*
1498                  * If a specific node is requested, make sure to
1499                  * get memory from there, but only when a node
1500                  * is explicitly specified.
1501                  */
1502                 if (nid != NUMA_NO_NODE)
1503                         gfp |= __GFP_THISNODE;
1504                 /*
1505                  * Make sure to call something that can handle
1506                  * nid=NUMA_NO_NODE
1507                  */
1508                 return alloc_pages_node(nid, gfp, order);
1509         }
1510 
1511         /*
1512          * OK, so we have a VMA.  Fetch the mempolicy and try to
1513          * allocate a huge page with it.  We will only reach this
1514          * when CONFIG_NUMA=y.
1515          */
1516         do {
1517                 struct page *page;
1518                 struct mempolicy *mpol;
1519                 struct zonelist *zl;
1520                 nodemask_t *nodemask;
1521 
1522                 cpuset_mems_cookie = read_mems_allowed_begin();
1523                 zl = huge_zonelist(vma, addr, gfp, &mpol, &nodemask);
1524                 mpol_cond_put(mpol);
1525                 page = __alloc_pages_nodemask(gfp, order, zl, nodemask);
1526                 if (page)
1527                         return page;
1528         } while (read_mems_allowed_retry(cpuset_mems_cookie));
1529 
1530         return NULL;
1531 }
1532 
1533 /*
1534  * There are two ways to allocate a huge page:
1535  * 1. When you have a VMA and an address (like a fault)
1536  * 2. When you have no VMA (like when setting /proc/.../nr_hugepages)
1537  *
1538  * 'vma' and 'addr' are only for (1).  'nid' is always NUMA_NO_NODE in
1539  * this case which signifies that the allocation should be done with
1540  * respect for the VMA's memory policy.
1541  *
1542  * For (2), we ignore 'vma' and 'addr' and use 'nid' exclusively. This
1543  * implies that memory policies will not be taken in to account.
1544  */
1545 static struct page *__alloc_buddy_huge_page(struct hstate *h,
1546                 struct vm_area_struct *vma, unsigned long addr, int nid)
1547 {
1548         struct page *page;
1549         unsigned int r_nid;
1550 
1551         if (hstate_is_gigantic(h))
1552                 return NULL;
1553 
1554         /*
1555          * Make sure that anyone specifying 'nid' is not also specifying a VMA.
1556          * This makes sure the caller is picking _one_ of the modes with which
1557          * we can call this function, not both.
1558          */
1559         if (vma || (addr != -1)) {
1560                 VM_WARN_ON_ONCE(addr == -1);
1561                 VM_WARN_ON_ONCE(nid != NUMA_NO_NODE);
1562         }
1563         /*
1564          * Assume we will successfully allocate the surplus page to
1565          * prevent racing processes from causing the surplus to exceed
1566          * overcommit
1567          *
1568          * This however introduces a different race, where a process B
1569          * tries to grow the static hugepage pool while alloc_pages() is
1570          * called by process A. B will only examine the per-node
1571          * counters in determining if surplus huge pages can be
1572          * converted to normal huge pages in adjust_pool_surplus(). A
1573          * won't be able to increment the per-node counter, until the
1574          * lock is dropped by B, but B doesn't drop hugetlb_lock until
1575          * no more huge pages can be converted from surplus to normal
1576          * state (and doesn't try to convert again). Thus, we have a
1577          * case where a surplus huge page exists, the pool is grown, and
1578          * the surplus huge page still exists after, even though it
1579          * should just have been converted to a normal huge page. This
1580          * does not leak memory, though, as the hugepage will be freed
1581          * once it is out of use. It also does not allow the counters to
1582          * go out of whack in adjust_pool_surplus() as we don't modify
1583          * the node values until we've gotten the hugepage and only the
1584          * per-node value is checked there.
1585          */
1586         spin_lock(&hugetlb_lock);
1587         if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
1588                 spin_unlock(&hugetlb_lock);
1589                 return NULL;
1590         } else {
1591                 h->nr_huge_pages++;
1592                 h->surplus_huge_pages++;
1593         }
1594         spin_unlock(&hugetlb_lock);
1595 
1596         page = __hugetlb_alloc_buddy_huge_page(h, vma, addr, nid);
1597 
1598         spin_lock(&hugetlb_lock);
1599         if (page) {
1600                 INIT_LIST_HEAD(&page->lru);
1601                 r_nid = page_to_nid(page);
1602                 set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1603                 set_hugetlb_cgroup(page, NULL);
1604                 /*
1605                  * We incremented the global counters already
1606                  */
1607                 h->nr_huge_pages_node[r_nid]++;
1608                 h->surplus_huge_pages_node[r_nid]++;
1609                 __count_vm_event(HTLB_BUDDY_PGALLOC);
1610         } else {
1611                 h->nr_huge_pages--;
1612                 h->surplus_huge_pages--;
1613                 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1614         }
1615         spin_unlock(&hugetlb_lock);
1616 
1617         return page;
1618 }
1619 
1620 /*
1621  * Allocate a huge page from 'nid'.  Note, 'nid' may be
1622  * NUMA_NO_NODE, which means that it may be allocated
1623  * anywhere.
1624  */
1625 static
1626 struct page *__alloc_buddy_huge_page_no_mpol(struct hstate *h, int nid)
1627 {
1628         unsigned long addr = -1;
1629 
1630         return __alloc_buddy_huge_page(h, NULL, addr, nid);
1631 }
1632 
1633 /*
1634  * Use the VMA's mpolicy to allocate a huge page from the buddy.
1635  */
1636 static
1637 struct page *__alloc_buddy_huge_page_with_mpol(struct hstate *h,
1638                 struct vm_area_struct *vma, unsigned long addr)
1639 {
1640         return __alloc_buddy_huge_page(h, vma, addr, NUMA_NO_NODE);
1641 }
1642 
1643 /*
1644  * This allocation function is useful in the context where vma is irrelevant.
1645  * E.g. soft-offlining uses this function because it only cares physical
1646  * address of error page.
1647  */
1648 struct page *alloc_huge_page_node(struct hstate *h, int nid)
1649 {
1650         struct page *page = NULL;
1651 
1652         spin_lock(&hugetlb_lock);
1653         if (h->free_huge_pages - h->resv_huge_pages > 0)
1654                 page = dequeue_huge_page_node(h, nid);
1655         spin_unlock(&hugetlb_lock);
1656 
1657         if (!page)
1658                 page = __alloc_buddy_huge_page_no_mpol(h, nid);
1659 
1660         return page;
1661 }
1662 
1663 /*
1664  * Increase the hugetlb pool such that it can accommodate a reservation
1665  * of size 'delta'.
1666  */
1667 static int gather_surplus_pages(struct hstate *h, int delta)
1668 {
1669         struct list_head surplus_list;
1670         struct page *page, *tmp;
1671         int ret, i;
1672         int needed, allocated;
1673         bool alloc_ok = true;
1674 
1675         needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
1676         if (needed <= 0) {
1677                 h->resv_huge_pages += delta;
1678                 return 0;
1679         }
1680 
1681         allocated = 0;
1682         INIT_LIST_HEAD(&surplus_list);
1683 
1684         ret = -ENOMEM;
1685 retry:
1686         spin_unlock(&hugetlb_lock);
1687         for (i = 0; i < needed; i++) {
1688                 page = __alloc_buddy_huge_page_no_mpol(h, NUMA_NO_NODE);
1689                 if (!page) {
1690                         alloc_ok = false;
1691                         break;
1692                 }
1693                 list_add(&page->lru, &surplus_list);
1694         }
1695         allocated += i;
1696 
1697         /*
1698          * After retaking hugetlb_lock, we need to recalculate 'needed'
1699          * because either resv_huge_pages or free_huge_pages may have changed.
1700          */
1701         spin_lock(&hugetlb_lock);
1702         needed = (h->resv_huge_pages + delta) -
1703                         (h->free_huge_pages + allocated);
1704         if (needed > 0) {
1705                 if (alloc_ok)
1706                         goto retry;
1707                 /*
1708                  * We were not able to allocate enough pages to
1709                  * satisfy the entire reservation so we free what
1710                  * we've allocated so far.
1711                  */
1712                 goto free;
1713         }
1714         /*
1715          * The surplus_list now contains _at_least_ the number of extra pages
1716          * needed to accommodate the reservation.  Add the appropriate number
1717          * of pages to the hugetlb pool and free the extras back to the buddy
1718          * allocator.  Commit the entire reservation here to prevent another
1719          * process from stealing the pages as they are added to the pool but
1720          * before they are reserved.
1721          */
1722         needed += allocated;
1723         h->resv_huge_pages += delta;
1724         ret = 0;
1725 
1726         /* Free the needed pages to the hugetlb pool */
1727         list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1728                 if ((--needed) < 0)
1729                         break;
1730                 /*
1731                  * This page is now managed by the hugetlb allocator and has
1732                  * no users -- drop the buddy allocator's reference.
1733                  */
1734                 put_page_testzero(page);
1735                 VM_BUG_ON_PAGE(page_count(page), page);
1736                 enqueue_huge_page(h, page);
1737         }
1738 free:
1739         spin_unlock(&hugetlb_lock);
1740 
1741         /* Free unnecessary surplus pages to the buddy allocator */
1742         list_for_each_entry_safe(page, tmp, &surplus_list, lru)
1743                 put_page(page);
1744         spin_lock(&hugetlb_lock);
1745 
1746         return ret;
1747 }
1748 
1749 /*
1750  * When releasing a hugetlb pool reservation, any surplus pages that were
1751  * allocated to satisfy the reservation must be explicitly freed if they were
1752  * never used.
1753  * Called with hugetlb_lock held.
1754  */
1755 static void return_unused_surplus_pages(struct hstate *h,
1756                                         unsigned long unused_resv_pages)
1757 {
1758         unsigned long nr_pages;
1759 
1760         /* Uncommit the reservation */
1761         h->resv_huge_pages -= unused_resv_pages;
1762 
1763         /* Cannot return gigantic pages currently */
1764         if (hstate_is_gigantic(h))
1765                 return;
1766 
1767         nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1768 
1769         /*
1770          * We want to release as many surplus pages as possible, spread
1771          * evenly across all nodes with memory. Iterate across these nodes
1772          * until we can no longer free unreserved surplus pages. This occurs
1773          * when the nodes with surplus pages have no free pages.
1774          * free_pool_huge_page() will balance the the freed pages across the
1775          * on-line nodes with memory and will handle the hstate accounting.
1776          */
1777         while (nr_pages--) {
1778                 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))
1779                         break;
1780                 cond_resched_lock(&hugetlb_lock);
1781         }
1782 }
1783 
1784 
1785 /*
1786  * vma_needs_reservation, vma_commit_reservation and vma_end_reservation
1787  * are used by the huge page allocation routines to manage reservations.
1788  *
1789  * vma_needs_reservation is called to determine if the huge page at addr
1790  * within the vma has an associated reservation.  If a reservation is
1791  * needed, the value 1 is returned.  The caller is then responsible for
1792  * managing the global reservation and subpool usage counts.  After
1793  * the huge page has been allocated, vma_commit_reservation is called
1794  * to add the page to the reservation map.  If the page allocation fails,
1795  * the reservation must be ended instead of committed.  vma_end_reservation
1796  * is called in such cases.
1797  *
1798  * In the normal case, vma_commit_reservation returns the same value
1799  * as the preceding vma_needs_reservation call.  The only time this
1800  * is not the case is if a reserve map was changed between calls.  It
1801  * is the responsibility of the caller to notice the difference and
1802  * take appropriate action.
1803  */
1804 enum vma_resv_mode {
1805         VMA_NEEDS_RESV,
1806         VMA_COMMIT_RESV,
1807         VMA_END_RESV,
1808 };
1809 static long __vma_reservation_common(struct hstate *h,
1810                                 struct vm_area_struct *vma, unsigned long addr,
1811                                 enum vma_resv_mode mode)
1812 {
1813         struct resv_map *resv;
1814         pgoff_t idx;
1815         long ret;
1816 
1817         resv = vma_resv_map(vma);
1818         if (!resv)
1819                 return 1;
1820 
1821         idx = vma_hugecache_offset(h, vma, addr);
1822         switch (mode) {
1823         case VMA_NEEDS_RESV:
1824                 ret = region_chg(resv, idx, idx + 1);
1825                 break;
1826         case VMA_COMMIT_RESV:
1827                 ret = region_add(resv, idx, idx + 1);
1828                 break;
1829         case VMA_END_RESV:
1830                 region_abort(resv, idx, idx + 1);
1831                 ret = 0;
1832                 break;
1833         default:
1834                 BUG();
1835         }
1836 
1837         if (vma->vm_flags & VM_MAYSHARE)
1838                 return ret;
1839         else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && ret >= 0) {
1840                 /*
1841                  * In most cases, reserves always exist for private mappings.
1842                  * However, a file associated with mapping could have been
1843                  * hole punched or truncated after reserves were consumed.
1844                  * As subsequent fault on such a range will not use reserves.
1845                  * Subtle - The reserve map for private mappings has the
1846                  * opposite meaning than that of shared mappings.  If NO
1847                  * entry is in the reserve map, it means a reservation exists.
1848                  * If an entry exists in the reserve map, it means the
1849                  * reservation has already been consumed.  As a result, the
1850                  * return value of this routine is the opposite of the
1851                  * value returned from reserve map manipulation routines above.
1852                  */
1853                 if (ret)
1854                         return 0;
1855                 else
1856                         return 1;
1857         }
1858         else
1859                 return ret < 0 ? ret : 0;
1860 }
1861 
1862 static long vma_needs_reservation(struct hstate *h,
1863                         struct vm_area_struct *vma, unsigned long addr)
1864 {
1865         return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
1866 }
1867 
1868 static long vma_commit_reservation(struct hstate *h,
1869                         struct vm_area_struct *vma, unsigned long addr)
1870 {
1871         return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
1872 }
1873 
1874 static void vma_end_reservation(struct hstate *h,
1875                         struct vm_area_struct *vma, unsigned long addr)
1876 {
1877         (void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
1878 }
1879 
1880 struct page *alloc_huge_page(struct vm_area_struct *vma,
1881                                     unsigned long addr, int avoid_reserve)
1882 {
1883         struct hugepage_subpool *spool = subpool_vma(vma);
1884         struct hstate *h = hstate_vma(vma);
1885         struct page *page;
1886         long map_chg, map_commit;
1887         long gbl_chg;
1888         int ret, idx;
1889         struct hugetlb_cgroup *h_cg;
1890 
1891         idx = hstate_index(h);
1892         /*
1893          * Examine the region/reserve map to determine if the process
1894          * has a reservation for the page to be allocated.  A return
1895          * code of zero indicates a reservation exists (no change).
1896          */
1897         map_chg = gbl_chg = vma_needs_reservation(h, vma, addr);
1898         if (map_chg < 0)
1899                 return ERR_PTR(-ENOMEM);
1900 
1901         /*
1902          * Processes that did not create the mapping will have no
1903          * reserves as indicated by the region/reserve map. Check
1904          * that the allocation will not exceed the subpool limit.
1905          * Allocations for MAP_NORESERVE mappings also need to be
1906          * checked against any subpool limit.
1907          */
1908         if (map_chg || avoid_reserve) {
1909                 gbl_chg = hugepage_subpool_get_pages(spool, 1);
1910                 if (gbl_chg < 0) {
1911                         vma_end_reservation(h, vma, addr);
1912                         return ERR_PTR(-ENOSPC);
1913                 }
1914 
1915                 /*
1916                  * Even though there was no reservation in the region/reserve
1917                  * map, there could be reservations associated with the
1918                  * subpool that can be used.  This would be indicated if the
1919                  * return value of hugepage_subpool_get_pages() is zero.
1920                  * However, if avoid_reserve is specified we still avoid even
1921                  * the subpool reservations.
1922                  */
1923                 if (avoid_reserve)
1924                         gbl_chg = 1;
1925         }
1926 
1927         ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
1928         if (ret)
1929                 goto out_subpool_put;
1930 
1931         spin_lock(&hugetlb_lock);
1932         /*
1933          * glb_chg is passed to indicate whether or not a page must be taken
1934          * from the global free pool (global change).  gbl_chg == 0 indicates
1935          * a reservation exists for the allocation.
1936          */
1937         page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg);
1938         if (!page) {
1939                 spin_unlock(&hugetlb_lock);
1940                 page = __alloc_buddy_huge_page_with_mpol(h, vma, addr);
1941                 if (!page)
1942                         goto out_uncharge_cgroup;
1943                 if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) {
1944                         SetPagePrivate(page);
1945                         h->resv_huge_pages--;
1946                 }
1947                 spin_lock(&hugetlb_lock);
1948                 list_move(&page->lru, &h->hugepage_activelist);
1949                 /* Fall through */
1950         }
1951         hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
1952         spin_unlock(&hugetlb_lock);
1953 
1954         set_page_private(page, (unsigned long)spool);
1955 
1956         map_commit = vma_commit_reservation(h, vma, addr);
1957         if (unlikely(map_chg > map_commit)) {
1958                 /*
1959                  * The page was added to the reservation map between
1960                  * vma_needs_reservation and vma_commit_reservation.
1961                  * This indicates a race with hugetlb_reserve_pages.
1962                  * Adjust for the subpool count incremented above AND
1963                  * in hugetlb_reserve_pages for the same page.  Also,
1964                  * the reservation count added in hugetlb_reserve_pages
1965                  * no longer applies.
1966                  */
1967                 long rsv_adjust;
1968 
1969                 rsv_adjust = hugepage_subpool_put_pages(spool, 1);
1970                 hugetlb_acct_memory(h, -rsv_adjust);
1971         }
1972         return page;
1973 
1974 out_uncharge_cgroup:
1975         hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
1976 out_subpool_put:
1977         if (map_chg || avoid_reserve)
1978                 hugepage_subpool_put_pages(spool, 1);
1979         vma_end_reservation(h, vma, addr);
1980         return ERR_PTR(-ENOSPC);
1981 }
1982 
1983 /*
1984  * alloc_huge_page()'s wrapper which simply returns the page if allocation
1985  * succeeds, otherwise NULL. This function is called from new_vma_page(),
1986  * where no ERR_VALUE is expected to be returned.
1987  */
1988 struct page *alloc_huge_page_noerr(struct vm_area_struct *vma,
1989                                 unsigned long addr, int avoid_reserve)
1990 {
1991         struct page *page = alloc_huge_page(vma, addr, avoid_reserve);
1992         if (IS_ERR(page))
1993                 page = NULL;
1994         return page;
1995 }
1996 
1997 int __weak alloc_bootmem_huge_page(struct hstate *h)
1998 {
1999         struct huge_bootmem_page *m;
2000         int nr_nodes, node;
2001 
2002         for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
2003                 void *addr;
2004 
2005                 addr = memblock_virt_alloc_try_nid_nopanic(
2006                                 huge_page_size(h), huge_page_size(h),
2007                                 0, BOOTMEM_ALLOC_ACCESSIBLE, node);
2008                 if (addr) {
2009                         /*
2010                          * Use the beginning of the huge page to store the
2011                          * huge_bootmem_page struct (until gather_bootmem
2012                          * puts them into the mem_map).
2013                          */
2014                         m = addr;
2015                         goto found;
2016                 }
2017         }
2018         return 0;
2019 
2020 found:
2021         BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h)));
2022         /* Put them into a private list first because mem_map is not up yet */
2023         list_add(&m->list, &huge_boot_pages);
2024         m->hstate = h;
2025         return 1;
2026 }
2027 
2028 static void __init prep_compound_huge_page(struct page *page,
2029                 unsigned int order)
2030 {
2031         if (unlikely(order > (MAX_ORDER - 1)))
2032                 prep_compound_gigantic_page(page, order);
2033         else
2034                 prep_compound_page(page, order);
2035 }
2036 
2037 /* Put bootmem huge pages into the standard lists after mem_map is up */
2038 static void __init gather_bootmem_prealloc(void)
2039 {
2040         struct huge_bootmem_page *m;
2041 
2042         list_for_each_entry(m, &huge_boot_pages, list) {
2043                 struct hstate *h = m->hstate;
2044                 struct page *page;
2045 
2046 #ifdef CONFIG_HIGHMEM
2047                 page = pfn_to_page(m->phys >> PAGE_SHIFT);
2048                 memblock_free_late(__pa(m),
2049                                    sizeof(struct huge_bootmem_page));
2050 #else
2051                 page = virt_to_page(m);
2052 #endif
2053                 WARN_ON(page_count(page) != 1);
2054                 prep_compound_huge_page(page, h->order);
2055                 WARN_ON(PageReserved(page));
2056                 prep_new_huge_page(h, page, page_to_nid(page));
2057                 /*
2058                  * If we had gigantic hugepages allocated at boot time, we need
2059                  * to restore the 'stolen' pages to totalram_pages in order to
2060                  * fix confusing memory reports from free(1) and another
2061                  * side-effects, like CommitLimit going negative.
2062                  */
2063                 if (hstate_is_gigantic(h))
2064                         adjust_managed_page_count(page, 1 << h->order);
2065         }
2066 }
2067 
2068 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
2069 {
2070         unsigned long i;
2071 
2072         for (i = 0; i < h->max_huge_pages; ++i) {
2073                 if (hstate_is_gigantic(h)) {
2074                         if (!alloc_bootmem_huge_page(h))
2075                                 break;
2076                 } else if (!alloc_fresh_huge_page(h,
2077                                          &node_states[N_MEMORY]))
2078                         break;
2079         }
2080         h->max_huge_pages = i;
2081 }
2082 
2083 static void __init hugetlb_init_hstates(void)
2084 {
2085         struct hstate *h;
2086 
2087         for_each_hstate(h) {
2088                 if (minimum_order > huge_page_order(h))
2089                         minimum_order = huge_page_order(h);
2090 
2091                 /* oversize hugepages were init'ed in early boot */
2092                 if (!hstate_is_gigantic(h))
2093                         hugetlb_hstate_alloc_pages(h);
2094         }
2095         VM_BUG_ON(minimum_order == UINT_MAX);
2096 }
2097 
2098 static char * __init memfmt(char *buf, unsigned long n)
2099 {
2100         if (n >= (1UL << 30))
2101                 sprintf(buf, "%lu GB", n >> 30);
2102         else if (n >= (1UL << 20))
2103                 sprintf(buf, "%lu MB", n >> 20);
2104         else
2105                 sprintf(buf, "%lu KB", n >> 10);
2106         return buf;
2107 }
2108 
2109 static void __init report_hugepages(void)
2110 {
2111         struct hstate *h;
2112 
2113         for_each_hstate(h) {
2114                 char buf[32];
2115                 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
2116                         memfmt(buf, huge_page_size(h)),
2117                         h->free_huge_pages);
2118         }
2119 }
2120 
2121 #ifdef CONFIG_HIGHMEM
2122 static void try_to_free_low(struct hstate *h, unsigned long count,
2123                                                 nodemask_t *nodes_allowed)
2124 {
2125         int i;
2126 
2127         if (hstate_is_gigantic(h))
2128                 return;
2129 
2130         for_each_node_mask(i, *nodes_allowed) {
2131                 struct page *page, *next;
2132                 struct list_head *freel = &h->hugepage_freelists[i];
2133                 list_for_each_entry_safe(page, next, freel, lru) {
2134                         if (count >= h->nr_huge_pages)
2135                                 return;
2136                         if (PageHighMem(page))
2137                                 continue;
2138                         list_del(&page->lru);
2139                         update_and_free_page(h, page);
2140                         h->free_huge_pages--;
2141                         h->free_huge_pages_node[page_to_nid(page)]--;
2142                 }
2143         }
2144 }
2145 #else
2146 static inline void try_to_free_low(struct hstate *h, unsigned long count,
2147                                                 nodemask_t *nodes_allowed)
2148 {
2149 }
2150 #endif
2151 
2152 /*
2153  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
2154  * balanced by operating on them in a round-robin fashion.
2155  * Returns 1 if an adjustment was made.
2156  */
2157 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
2158                                 int delta)
2159 {
2160         int nr_nodes, node;
2161 
2162         VM_BUG_ON(delta != -1 && delta != 1);
2163 
2164         if (delta < 0) {
2165                 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
2166                         if (h->surplus_huge_pages_node[node])
2167                                 goto found;
2168                 }
2169         } else {
2170                 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
2171                         if (h->surplus_huge_pages_node[node] <
2172                                         h->nr_huge_pages_node[node])
2173                                 goto found;
2174                 }
2175         }
2176         return 0;
2177 
2178 found:
2179         h->surplus_huge_pages += delta;
2180         h->surplus_huge_pages_node[node] += delta;
2181         return 1;
2182 }
2183 
2184 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
2185 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
2186                                                 nodemask_t *nodes_allowed)
2187 {
2188         unsigned long min_count, ret;
2189 
2190         if (hstate_is_gigantic(h) && !gigantic_page_supported())
2191                 return h->max_huge_pages;
2192 
2193         /*
2194          * Increase the pool size
2195          * First take pages out of surplus state.  Then make up the
2196          * remaining difference by allocating fresh huge pages.
2197          *
2198          * We might race with __alloc_buddy_huge_page() here and be unable
2199          * to convert a surplus huge page to a normal huge page. That is
2200          * not critical, though, it just means the overall size of the
2201          * pool might be one hugepage larger than it needs to be, but
2202          * within all the constraints specified by the sysctls.
2203          */
2204         spin_lock(&hugetlb_lock);
2205         while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
2206                 if (!adjust_pool_surplus(h, nodes_allowed, -1))
2207                         break;
2208         }
2209 
2210         while (count > persistent_huge_pages(h)) {
2211                 /*
2212                  * If this allocation races such that we no longer need the
2213                  * page, free_huge_page will handle it by freeing the page
2214                  * and reducing the surplus.
2215                  */
2216                 spin_unlock(&hugetlb_lock);
2217 
2218                 /* yield cpu to avoid soft lockup */
2219                 cond_resched();
2220 
2221                 if (hstate_is_gigantic(h))
2222                         ret = alloc_fresh_gigantic_page(h, nodes_allowed);
2223                 else
2224                         ret = alloc_fresh_huge_page(h, nodes_allowed);
2225                 spin_lock(&hugetlb_lock);
2226                 if (!ret)
2227                         goto out;
2228 
2229                 /* Bail for signals. Probably ctrl-c from user */
2230                 if (signal_pending(current))
2231                         goto out;
2232         }
2233 
2234         /*
2235          * Decrease the pool size
2236          * First return free pages to the buddy allocator (being careful
2237          * to keep enough around to satisfy reservations).  Then place
2238          * pages into surplus state as needed so the pool will shrink
2239          * to the desired size as pages become free.
2240          *
2241          * By placing pages into the surplus state independent of the
2242          * overcommit value, we are allowing the surplus pool size to
2243          * exceed overcommit. There are few sane options here. Since
2244          * __alloc_buddy_huge_page() is checking the global counter,
2245          * though, we'll note that we're not allowed to exceed surplus
2246          * and won't grow the pool anywhere else. Not until one of the
2247          * sysctls are changed, or the surplus pages go out of use.
2248          */
2249         min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
2250         min_count = max(count, min_count);
2251         try_to_free_low(h, min_count, nodes_allowed);
2252         while (min_count < persistent_huge_pages(h)) {
2253                 if (!free_pool_huge_page(h, nodes_allowed, 0))
2254                         break;
2255                 cond_resched_lock(&hugetlb_lock);
2256         }
2257         while (count < persistent_huge_pages(h)) {
2258                 if (!adjust_pool_surplus(h, nodes_allowed, 1))
2259                         break;
2260         }
2261 out:
2262         ret = persistent_huge_pages(h);
2263         spin_unlock(&hugetlb_lock);
2264         return ret;
2265 }
2266 
2267 #define HSTATE_ATTR_RO(_name) \
2268         static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
2269 
2270 #define HSTATE_ATTR(_name) \
2271         static struct kobj_attribute _name##_attr = \
2272                 __ATTR(_name, 0644, _name##_show, _name##_store)
2273 
2274 static struct kobject *hugepages_kobj;
2275 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
2276 
2277 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
2278 
2279 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
2280 {
2281         int i;
2282 
2283         for (i = 0; i < HUGE_MAX_HSTATE; i++)
2284                 if (hstate_kobjs[i] == kobj) {
2285                         if (nidp)
2286                                 *nidp = NUMA_NO_NODE;
2287                         return &hstates[i];
2288                 }
2289 
2290         return kobj_to_node_hstate(kobj, nidp);
2291 }
2292 
2293 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
2294                                         struct kobj_attribute *attr, char *buf)
2295 {
2296         struct hstate *h;
2297         unsigned long nr_huge_pages;
2298         int nid;
2299 
2300         h = kobj_to_hstate(kobj, &nid);
2301         if (nid == NUMA_NO_NODE)
2302                 nr_huge_pages = h->nr_huge_pages;
2303         else
2304                 nr_huge_pages = h->nr_huge_pages_node[nid];
2305 
2306         return sprintf(buf, "%lu\n", nr_huge_pages);
2307 }
2308 
2309 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
2310                                            struct hstate *h, int nid,
2311                                            unsigned long count, size_t len)
2312 {
2313         int err;
2314         NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
2315 
2316         if (hstate_is_gigantic(h) && !gigantic_page_supported()) {
2317                 err = -EINVAL;
2318                 goto out;
2319         }
2320 
2321         if (nid == NUMA_NO_NODE) {
2322                 /*
2323                  * global hstate attribute
2324                  */
2325                 if (!(obey_mempolicy &&
2326                                 init_nodemask_of_mempolicy(nodes_allowed))) {
2327                         NODEMASK_FREE(nodes_allowed);
2328                         nodes_allowed = &node_states[N_MEMORY];
2329                 }
2330         } else if (nodes_allowed) {
2331                 /*
2332                  * per node hstate attribute: adjust count to global,
2333                  * but restrict alloc/free to the specified node.
2334                  */
2335                 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
2336                 init_nodemask_of_node(nodes_allowed, nid);
2337         } else
2338                 nodes_allowed = &node_states[N_MEMORY];
2339 
2340         h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
2341 
2342         if (nodes_allowed != &node_states[N_MEMORY])
2343                 NODEMASK_FREE(nodes_allowed);
2344 
2345         return len;
2346 out:
2347         NODEMASK_FREE(nodes_allowed);
2348         return err;
2349 }
2350 
2351 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
2352                                          struct kobject *kobj, const char *buf,
2353                                          size_t len)
2354 {
2355         struct hstate *h;
2356         unsigned long count;
2357         int nid;
2358         int err;
2359 
2360         err = kstrtoul(buf, 10, &count);
2361         if (err)
2362                 return err;
2363 
2364         h = kobj_to_hstate(kobj, &nid);
2365         return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
2366 }
2367 
2368 static ssize_t nr_hugepages_show(struct kobject *kobj,
2369                                        struct kobj_attribute *attr, char *buf)
2370 {
2371         return nr_hugepages_show_common(kobj, attr, buf);
2372 }
2373 
2374 static ssize_t nr_hugepages_store(struct kobject *kobj,
2375                struct kobj_attribute *attr, const char *buf, size_t len)
2376 {
2377         return nr_hugepages_store_common(false, kobj, buf, len);
2378 }
2379 HSTATE_ATTR(nr_hugepages);
2380 
2381 #ifdef CONFIG_NUMA
2382 
2383 /*
2384  * hstate attribute for optionally mempolicy-based constraint on persistent
2385  * huge page alloc/free.
2386  */
2387 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
2388                                        struct kobj_attribute *attr, char *buf)
2389 {
2390         return nr_hugepages_show_common(kobj, attr, buf);
2391 }
2392 
2393 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
2394                struct kobj_attribute *attr, const char *buf, size_t len)
2395 {
2396         return nr_hugepages_store_common(true, kobj, buf, len);
2397 }
2398 HSTATE_ATTR(nr_hugepages_mempolicy);
2399 #endif
2400 
2401 
2402 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
2403                                         struct kobj_attribute *attr, char *buf)
2404 {
2405         struct hstate *h = kobj_to_hstate(kobj, NULL);
2406         return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
2407 }
2408 
2409 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
2410                 struct kobj_attribute *attr, const char *buf, size_t count)
2411 {
2412         int err;
2413         unsigned long input;
2414         struct hstate *h = kobj_to_hstate(kobj, NULL);
2415 
2416         if (hstate_is_gigantic(h))
2417                 return -EINVAL;
2418 
2419         err = kstrtoul(buf, 10, &input);
2420         if (err)
2421                 return err;
2422 
2423         spin_lock(&hugetlb_lock);
2424         h->nr_overcommit_huge_pages = input;
2425         spin_unlock(&hugetlb_lock);
2426 
2427         return count;
2428 }
2429 HSTATE_ATTR(nr_overcommit_hugepages);
2430 
2431 static ssize_t free_hugepages_show(struct kobject *kobj,
2432                                         struct kobj_attribute *attr, char *buf)
2433 {
2434         struct hstate *h;
2435         unsigned long free_huge_pages;
2436         int nid;
2437 
2438         h = kobj_to_hstate(kobj, &nid);
2439         if (nid == NUMA_NO_NODE)
2440                 free_huge_pages = h->free_huge_pages;
2441         else
2442                 free_huge_pages = h->free_huge_pages_node[nid];
2443 
2444         return sprintf(buf, "%lu\n", free_huge_pages);
2445 }
2446 HSTATE_ATTR_RO(free_hugepages);
2447 
2448 static ssize_t resv_hugepages_show(struct kobject *kobj,
2449                                         struct kobj_attribute *attr, char *buf)
2450 {
2451         struct hstate *h = kobj_to_hstate(kobj, NULL);
2452         return sprintf(buf, "%lu\n", h->resv_huge_pages);
2453 }
2454 HSTATE_ATTR_RO(resv_hugepages);
2455 
2456 static ssize_t surplus_hugepages_show(struct kobject *kobj,
2457                                         struct kobj_attribute *attr, char *buf)
2458 {
2459         struct hstate *h;
2460         unsigned long surplus_huge_pages;
2461         int nid;
2462 
2463         h = kobj_to_hstate(kobj, &nid);
2464         if (nid == NUMA_NO_NODE)
2465                 surplus_huge_pages = h->surplus_huge_pages;
2466         else
2467                 surplus_huge_pages = h->surplus_huge_pages_node[nid];
2468 
2469         return sprintf(buf, "%lu\n", surplus_huge_pages);
2470 }
2471 HSTATE_ATTR_RO(surplus_hugepages);
2472 
2473 static struct attribute *hstate_attrs[] = {
2474         &nr_hugepages_attr.attr,
2475         &nr_overcommit_hugepages_attr.attr,
2476         &free_hugepages_attr.attr,
2477         &resv_hugepages_attr.attr,
2478         &surplus_hugepages_attr.attr,
2479 #ifdef CONFIG_NUMA
2480         &nr_hugepages_mempolicy_attr.attr,
2481 #endif
2482         NULL,
2483 };
2484 
2485 static struct attribute_group hstate_attr_group = {
2486         .attrs = hstate_attrs,
2487 };
2488 
2489 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
2490                                     struct kobject **hstate_kobjs,
2491                                     struct attribute_group *hstate_attr_group)
2492 {
2493         int retval;
2494         int hi = hstate_index(h);
2495 
2496         hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
2497         if (!hstate_kobjs[hi])
2498                 return -ENOMEM;
2499 
2500         retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
2501         if (retval)
2502                 kobject_put(hstate_kobjs[hi]);
2503 
2504         return retval;
2505 }
2506 
2507 static void __init hugetlb_sysfs_init(void)
2508 {
2509         struct hstate *h;
2510         int err;
2511 
2512         hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
2513         if (!hugepages_kobj)
2514                 return;
2515 
2516         for_each_hstate(h) {
2517                 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
2518                                          hstate_kobjs, &hstate_attr_group);
2519                 if (err)
2520                         pr_err("Hugetlb: Unable to add hstate %s", h->name);
2521         }
2522 }
2523 
2524 #ifdef CONFIG_NUMA
2525 
2526 /*
2527  * node_hstate/s - associate per node hstate attributes, via their kobjects,
2528  * with node devices in node_devices[] using a parallel array.  The array
2529  * index of a node device or _hstate == node id.
2530  * This is here to avoid any static dependency of the node device driver, in
2531  * the base kernel, on the hugetlb module.
2532  */
2533 struct node_hstate {
2534         struct kobject          *hugepages_kobj;
2535         struct kobject          *hstate_kobjs[HUGE_MAX_HSTATE];
2536 };
2537 static struct node_hstate node_hstates[MAX_NUMNODES];
2538 
2539 /*
2540  * A subset of global hstate attributes for node devices
2541  */
2542 static struct attribute *per_node_hstate_attrs[] = {
2543         &nr_hugepages_attr.attr,
2544         &free_hugepages_attr.attr,
2545         &surplus_hugepages_attr.attr,
2546         NULL,
2547 };
2548 
2549 static struct attribute_group per_node_hstate_attr_group = {
2550         .attrs = per_node_hstate_attrs,
2551 };
2552 
2553 /*
2554  * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
2555  * Returns node id via non-NULL nidp.
2556  */
2557 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2558 {
2559         int nid;
2560 
2561         for (nid = 0; nid < nr_node_ids; nid++) {
2562                 struct node_hstate *nhs = &node_hstates[nid];
2563                 int i;
2564                 for (i = 0; i < HUGE_MAX_HSTATE; i++)
2565                         if (nhs->hstate_kobjs[i] == kobj) {
2566                                 if (nidp)
2567                                         *nidp = nid;
2568                                 return &hstates[i];
2569                         }
2570         }
2571 
2572         BUG();
2573         return NULL;
2574 }
2575 
2576 /*
2577  * Unregister hstate attributes from a single node device.
2578  * No-op if no hstate attributes attached.
2579  */
2580 static void hugetlb_unregister_node(struct node *node)
2581 {
2582         struct hstate *h;
2583         struct node_hstate *nhs = &node_hstates[node->dev.id];
2584 
2585         if (!nhs->hugepages_kobj)
2586                 return;         /* no hstate attributes */
2587 
2588         for_each_hstate(h) {
2589                 int idx = hstate_index(h);
2590                 if (nhs->hstate_kobjs[idx]) {
2591                         kobject_put(nhs->hstate_kobjs[idx]);
2592                         nhs->hstate_kobjs[idx] = NULL;
2593                 }
2594         }
2595 
2596         kobject_put(nhs->hugepages_kobj);
2597         nhs->hugepages_kobj = NULL;
2598 }
2599 
2600 
2601 /*
2602  * Register hstate attributes for a single node device.
2603  * No-op if attributes already registered.
2604  */
2605 static void hugetlb_register_node(struct node *node)
2606 {
2607         struct hstate *h;
2608         struct node_hstate *nhs = &node_hstates[node->dev.id];
2609         int err;
2610 
2611         if (nhs->hugepages_kobj)
2612                 return;         /* already allocated */
2613 
2614         nhs->hugepages_kobj = kobject_create_and_add("hugepages",
2615                                                         &node->dev.kobj);
2616         if (!nhs->hugepages_kobj)
2617                 return;
2618 
2619         for_each_hstate(h) {
2620                 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
2621                                                 nhs->hstate_kobjs,
2622                                                 &per_node_hstate_attr_group);
2623                 if (err) {
2624                         pr_err("Hugetlb: Unable to add hstate %s for node %d\n",
2625                                 h->name, node->dev.id);
2626                         hugetlb_unregister_node(node);
2627                         break;
2628                 }
2629         }
2630 }
2631 
2632 /*
2633  * hugetlb init time:  register hstate attributes for all registered node
2634  * devices of nodes that have memory.  All on-line nodes should have
2635  * registered their associated device by this time.
2636  */
2637 static void __init hugetlb_register_all_nodes(void)
2638 {
2639         int nid;
2640 
2641         for_each_node_state(nid, N_MEMORY) {
2642                 struct node *node = node_devices[nid];
2643                 if (node->dev.id == nid)
2644                         hugetlb_register_node(node);
2645         }
2646 
2647         /*
2648          * Let the node device driver know we're here so it can
2649          * [un]register hstate attributes on node hotplug.
2650          */
2651         register_hugetlbfs_with_node(hugetlb_register_node,
2652                                      hugetlb_unregister_node);
2653 }
2654 #else   /* !CONFIG_NUMA */
2655 
2656 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2657 {
2658         BUG();
2659         if (nidp)
2660                 *nidp = -1;
2661         return NULL;
2662 }
2663 
2664 static void hugetlb_register_all_nodes(void) { }
2665 
2666 #endif
2667 
2668 static int __init hugetlb_init(void)
2669 {
2670         int i;
2671 
2672         if (!hugepages_supported())
2673                 return 0;
2674 
2675         if (!size_to_hstate(default_hstate_size)) {
2676                 default_hstate_size = HPAGE_SIZE;
2677                 if (!size_to_hstate(default_hstate_size))
2678                         hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
2679         }
2680         default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size));
2681         if (default_hstate_max_huge_pages) {
2682                 if (!default_hstate.max_huge_pages)
2683                         default_hstate.max_huge_pages = default_hstate_max_huge_pages;
2684         }
2685 
2686         hugetlb_init_hstates();
2687         gather_bootmem_prealloc();
2688         report_hugepages();
2689 
2690         hugetlb_sysfs_init();
2691         hugetlb_register_all_nodes();
2692         hugetlb_cgroup_file_init();
2693 
2694 #ifdef CONFIG_SMP
2695         num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
2696 #else
2697         num_fault_mutexes = 1;
2698 #endif
2699         hugetlb_fault_mutex_table =
2700                 kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL);
2701         BUG_ON(!hugetlb_fault_mutex_table);
2702 
2703         for (i = 0; i < num_fault_mutexes; i++)
2704                 mutex_init(&hugetlb_fault_mutex_table[i]);
2705         return 0;
2706 }
2707 subsys_initcall(hugetlb_init);
2708 
2709 /* Should be called on processing a hugepagesz=... option */
2710 void __init hugetlb_bad_size(void)
2711 {
2712         parsed_valid_hugepagesz = false;
2713 }
2714 
2715 void __init hugetlb_add_hstate(unsigned int order)
2716 {
2717         struct hstate *h;
2718         unsigned long i;
2719 
2720         if (size_to_hstate(PAGE_SIZE << order)) {
2721                 pr_warn("hugepagesz= specified twice, ignoring\n");
2722                 return;
2723         }
2724         BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
2725         BUG_ON(order == 0);
2726         h = &hstates[hugetlb_max_hstate++];
2727         h->order = order;
2728         h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
2729         h->nr_huge_pages = 0;
2730         h->free_huge_pages = 0;
2731         for (i = 0; i < MAX_NUMNODES; ++i)
2732                 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
2733         INIT_LIST_HEAD(&h->hugepage_activelist);
2734         h->next_nid_to_alloc = first_memory_node;
2735         h->next_nid_to_free = first_memory_node;
2736         snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
2737                                         huge_page_size(h)/1024);
2738 
2739         parsed_hstate = h;
2740 }
2741 
2742 static int __init hugetlb_nrpages_setup(char *s)
2743 {
2744         unsigned long *mhp;
2745         static unsigned long *last_mhp;
2746 
2747         if (!parsed_valid_hugepagesz) {
2748                 pr_warn("hugepages = %s preceded by "
2749                         "an unsupported hugepagesz, ignoring\n", s);
2750                 parsed_valid_hugepagesz = true;
2751                 return 1;
2752         }
2753         /*
2754          * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet,
2755          * so this hugepages= parameter goes to the "default hstate".
2756          */
2757         else if (!hugetlb_max_hstate)
2758                 mhp = &default_hstate_max_huge_pages;
2759         else
2760                 mhp = &parsed_hstate->max_huge_pages;
2761 
2762         if (mhp == last_mhp) {
2763                 pr_warn("hugepages= specified twice without interleaving hugepagesz=, ignoring\n");
2764                 return 1;
2765         }
2766 
2767         if (sscanf(s, "%lu", mhp) <= 0)
2768                 *mhp = 0;
2769 
2770         /*
2771          * Global state is always initialized later in hugetlb_init.
2772          * But we need to allocate >= MAX_ORDER hstates here early to still
2773          * use the bootmem allocator.
2774          */
2775         if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER)
2776                 hugetlb_hstate_alloc_pages(parsed_hstate);
2777 
2778         last_mhp = mhp;
2779 
2780         return 1;
2781 }
2782 __setup("hugepages=", hugetlb_nrpages_setup);
2783 
2784 static int __init hugetlb_default_setup(char *s)
2785 {
2786         default_hstate_size = memparse(s, &s);
2787         return 1;
2788 }
2789 __setup("default_hugepagesz=", hugetlb_default_setup);
2790 
2791 static unsigned int cpuset_mems_nr(unsigned int *array)
2792 {
2793         int node;
2794         unsigned int nr = 0;
2795 
2796         for_each_node_mask(node, cpuset_current_mems_allowed)
2797                 nr += array[node];
2798 
2799         return nr;
2800 }
2801 
2802 #ifdef CONFIG_SYSCTL
2803 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
2804                          struct ctl_table *table, int write,
2805                          void __user *buffer, size_t *length, loff_t *ppos)
2806 {
2807         struct hstate *h = &default_hstate;
2808         unsigned long tmp = h->max_huge_pages;
2809         int ret;
2810 
2811         if (!hugepages_supported())
2812                 return -EOPNOTSUPP;
2813 
2814         table->data = &tmp;
2815         table->maxlen = sizeof(unsigned long);
2816         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2817         if (ret)
2818                 goto out;
2819 
2820         if (write)
2821                 ret = __nr_hugepages_store_common(obey_mempolicy, h,
2822                                                   NUMA_NO_NODE, tmp, *length);
2823 out:
2824         return ret;
2825 }
2826 
2827 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2828                           void __user *buffer, size_t *length, loff_t *ppos)
2829 {
2830 
2831         return hugetlb_sysctl_handler_common(false, table, write,
2832                                                         buffer, length, ppos);
2833 }
2834 
2835 #ifdef CONFIG_NUMA
2836 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2837                           void __user *buffer, size_t *length, loff_t *ppos)
2838 {
2839         return hugetlb_sysctl_handler_common(true, table, write,
2840                                                         buffer, length, ppos);
2841 }
2842 #endif /* CONFIG_NUMA */
2843 
2844 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
2845                         void __user *buffer,
2846                         size_t *length, loff_t *ppos)
2847 {
2848         struct hstate *h = &default_hstate;
2849         unsigned long tmp;
2850         int ret;
2851 
2852         if (!hugepages_supported())
2853                 return -EOPNOTSUPP;
2854 
2855         tmp = h->nr_overcommit_huge_pages;
2856 
2857         if (write && hstate_is_gigantic(h))
2858                 return -EINVAL;
2859 
2860         table->data = &tmp;
2861         table->maxlen = sizeof(unsigned long);
2862         ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2863         if (ret)
2864                 goto out;
2865 
2866         if (write) {
2867                 spin_lock(&hugetlb_lock);
2868                 h->nr_overcommit_huge_pages = tmp;
2869                 spin_unlock(&hugetlb_lock);
2870         }
2871 out:
2872         return ret;
2873 }
2874 
2875 #endif /* CONFIG_SYSCTL */
2876 
2877 void hugetlb_report_meminfo(struct seq_file *m)
2878 {
2879         struct hstate *h = &default_hstate;
2880         if (!hugepages_supported())
2881                 return;
2882         seq_printf(m,
2883                         "HugePages_Total:   %5lu\n"
2884                         "HugePages_Free:    %5lu\n"
2885                         "HugePages_Rsvd:    %5lu\n"
2886                         "HugePages_Surp:    %5lu\n"
2887                         "Hugepagesize:   %8lu kB\n",
2888                         h->nr_huge_pages,
2889                         h->free_huge_pages,
2890                         h->resv_huge_pages,
2891                         h->surplus_huge_pages,
2892                         1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2893 }
2894 
2895 int hugetlb_report_node_meminfo(int nid, char *buf)
2896 {
2897         struct hstate *h = &default_hstate;
2898         if (!hugepages_supported())
2899                 return 0;
2900         return sprintf(buf,
2901                 "Node %d HugePages_Total: %5u\n"
2902                 "Node %d HugePages_Free:  %5u\n"
2903                 "Node %d HugePages_Surp:  %5u\n",
2904                 nid, h->nr_huge_pages_node[nid],
2905                 nid, h->free_huge_pages_node[nid],
2906                 nid, h->surplus_huge_pages_node[nid]);
2907 }
2908 
2909 void hugetlb_show_meminfo(void)
2910 {
2911         struct hstate *h;
2912         int nid;
2913 
2914         if (!hugepages_supported())
2915                 return;
2916 
2917         for_each_node_state(nid, N_MEMORY)
2918                 for_each_hstate(h)
2919                         pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
2920                                 nid,
2921                                 h->nr_huge_pages_node[nid],
2922                                 h->free_huge_pages_node[nid],
2923                                 h->surplus_huge_pages_node[nid],
2924                                 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2925 }
2926 
2927 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm)
2928 {
2929         seq_printf(m, "HugetlbPages:\t%8lu kB\n",
2930                    atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10));
2931 }
2932 
2933 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
2934 unsigned long hugetlb_total_pages(void)
2935 {
2936         struct hstate *h;
2937         unsigned long nr_total_pages = 0;
2938 
2939         for_each_hstate(h)
2940                 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
2941         return nr_total_pages;
2942 }
2943 
2944 static int hugetlb_acct_memory(struct hstate *h, long delta)
2945 {
2946         int ret = -ENOMEM;
2947 
2948         spin_lock(&hugetlb_lock);
2949         /*
2950          * When cpuset is configured, it breaks the strict hugetlb page
2951          * reservation as the accounting is done on a global variable. Such
2952          * reservation is completely rubbish in the presence of cpuset because
2953          * the reservation is not checked against page availability for the
2954          * current cpuset. Application can still potentially OOM'ed by kernel
2955          * with lack of free htlb page in cpuset that the task is in.
2956          * Attempt to enforce strict accounting with cpuset is almost
2957          * impossible (or too ugly) because cpuset is too fluid that
2958          * task or memory node can be dynamically moved between cpusets.
2959          *
2960          * The change of semantics for shared hugetlb mapping with cpuset is
2961          * undesirable. However, in order to preserve some of the semantics,
2962          * we fall back to check against current free page availability as
2963          * a best attempt and hopefully to minimize the impact of changing
2964          * semantics that cpuset has.
2965          */
2966         if (delta > 0) {
2967                 if (gather_surplus_pages(h, delta) < 0)
2968                         goto out;
2969 
2970                 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
2971                         return_unused_surplus_pages(h, delta);
2972                         goto out;
2973                 }
2974         }
2975 
2976         ret = 0;
2977         if (delta < 0)
2978                 return_unused_surplus_pages(h, (unsigned long) -delta);
2979 
2980 out:
2981         spin_unlock(&hugetlb_lock);
2982         return ret;
2983 }
2984 
2985 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
2986 {
2987         struct resv_map *resv = vma_resv_map(vma);
2988 
2989         /*
2990          * This new VMA should share its siblings reservation map if present.
2991          * The VMA will only ever have a valid reservation map pointer where
2992          * it is being copied for another still existing VMA.  As that VMA
2993          * has a reference to the reservation map it cannot disappear until
2994          * after this open call completes.  It is therefore safe to take a
2995          * new reference here without additional locking.
2996          */
2997         if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
2998                 kref_get(&resv->refs);
2999 }
3000 
3001 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
3002 {
3003         struct hstate *h = hstate_vma(vma);
3004         struct resv_map *resv = vma_resv_map(vma);
3005         struct hugepage_subpool *spool = subpool_vma(vma);
3006         unsigned long reserve, start, end;
3007         long gbl_reserve;
3008 
3009         if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3010                 return;
3011 
3012         start = vma_hugecache_offset(h, vma, vma->vm_start);
3013         end = vma_hugecache_offset(h, vma, vma->vm_end);
3014 
3015         reserve = (end - start) - region_count(resv, start, end);
3016 
3017         kref_put(&resv->refs, resv_map_release);
3018 
3019         if (reserve) {
3020                 /*
3021                  * Decrement reserve counts.  The global reserve count may be
3022                  * adjusted if the subpool has a minimum size.
3023                  */
3024                 gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
3025                 hugetlb_acct_memory(h, -gbl_reserve);
3026         }
3027 }
3028 
3029 /*
3030  * We cannot handle pagefaults against hugetlb pages at all.  They cause
3031  * handle_mm_fault() to try to instantiate regular-sized pages in the
3032  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
3033  * this far.
3034  */
3035 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
3036 {
3037         BUG();
3038         return 0;
3039 }
3040 
3041 const struct vm_operations_struct hugetlb_vm_ops = {
3042         .fault = hugetlb_vm_op_fault,
3043         .open = hugetlb_vm_op_open,
3044         .close = hugetlb_vm_op_close,
3045 };
3046 
3047 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
3048                                 int writable)
3049 {
3050         pte_t entry;
3051 
3052         if (writable) {
3053                 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
3054                                          vma->vm_page_prot)));
3055         } else {
3056                 entry = huge_pte_wrprotect(mk_huge_pte(page,
3057                                            vma->vm_page_prot));
3058         }
3059         entry = pte_mkyoung(entry);
3060         entry = pte_mkhuge(entry);
3061         entry = arch_make_huge_pte(entry, vma, page, writable);
3062 
3063         return entry;
3064 }
3065 
3066 static void set_huge_ptep_writable(struct vm_area_struct *vma,
3067                                    unsigned long address, pte_t *ptep)
3068 {
3069         pte_t entry;
3070 
3071         entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
3072         if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
3073                 update_mmu_cache(vma, address, ptep);
3074 }
3075 
3076 static int is_hugetlb_entry_migration(pte_t pte)
3077 {
3078         swp_entry_t swp;
3079 
3080         if (huge_pte_none(pte) || pte_present(pte))
3081                 return 0;
3082         swp = pte_to_swp_entry(pte);
3083         if (non_swap_entry(swp) && is_migration_entry(swp))
3084                 return 1;
3085         else
3086                 return 0;
3087 }
3088 
3089 static int is_hugetlb_entry_hwpoisoned(pte_t pte)
3090 {
3091         swp_entry_t swp;
3092 
3093         if (huge_pte_none(pte) || pte_present(pte))
3094                 return 0;
3095         swp = pte_to_swp_entry(pte);
3096         if (non_swap_entry(swp) && is_hwpoison_entry(swp))
3097                 return 1;
3098         else
3099                 return 0;
3100 }
3101 
3102 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
3103                             struct vm_area_struct *vma)
3104 {
3105         pte_t *src_pte, *dst_pte, entry;
3106         struct page *ptepage;
3107         unsigned long addr;
3108         int cow;
3109         struct hstate *h = hstate_vma(vma);
3110         unsigned long sz = huge_page_size(h);
3111         unsigned long mmun_start;       /* For mmu_notifiers */
3112         unsigned long mmun_end;         /* For mmu_notifiers */
3113         int ret = 0;
3114 
3115         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
3116 
3117         mmun_start = vma->vm_start;
3118         mmun_end = vma->vm_end;
3119         if (cow)
3120                 mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end);
3121 
3122         for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
3123                 spinlock_t *src_ptl, *dst_ptl;
3124                 src_pte = huge_pte_offset(src, addr);
3125                 if (!src_pte)
3126                         continue;
3127                 dst_pte = huge_pte_alloc(dst, addr, sz);
3128                 if (!dst_pte) {
3129                         ret = -ENOMEM;
3130                         break;
3131                 }
3132 
3133                 /* If the pagetables are shared don't copy or take references */
3134                 if (dst_pte == src_pte)
3135                         continue;
3136 
3137                 dst_ptl = huge_pte_lock(h, dst, dst_pte);
3138                 src_ptl = huge_pte_lockptr(h, src, src_pte);
3139                 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
3140                 entry = huge_ptep_get(src_pte);
3141                 if (huge_pte_none(entry)) { /* skip none entry */
3142                         ;
3143                 } else if (unlikely(is_hugetlb_entry_migration(entry) ||
3144                                     is_hugetlb_entry_hwpoisoned(entry))) {
3145                         swp_entry_t swp_entry = pte_to_swp_entry(entry);
3146 
3147                         if (is_write_migration_entry(swp_entry) && cow) {
3148                                 /*
3149                                  * COW mappings require pages in both
3150                                  * parent and child to be set to read.
3151                                  */
3152                                 make_migration_entry_read(&swp_entry);
3153                                 entry = swp_entry_to_pte(swp_entry);
3154                                 set_huge_pte_at(src, addr, src_pte, entry);
3155                         }
3156                         set_huge_pte_at(dst, addr, dst_pte, entry);
3157                 } else {
3158                         if (cow) {
3159                                 huge_ptep_set_wrprotect(src, addr, src_pte);
3160                                 mmu_notifier_invalidate_range(src, mmun_start,
3161                                                                    mmun_end);
3162                         }
3163                         entry = huge_ptep_get(src_pte);
3164                         ptepage = pte_page(entry);
3165                         get_page(ptepage);
3166                         page_dup_rmap(ptepage, true);
3167                         set_huge_pte_at(dst, addr, dst_pte, entry);
3168                         hugetlb_count_add(pages_per_huge_page(h), dst);
3169                 }
3170                 spin_unlock(src_ptl);
3171                 spin_unlock(dst_ptl);
3172         }
3173 
3174         if (cow)
3175                 mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end);
3176 
3177         return ret;
3178 }
3179 
3180 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
3181                             unsigned long start, unsigned long end,
3182                             struct page *ref_page)
3183 {
3184         int force_flush = 0;
3185         struct mm_struct *mm = vma->vm_mm;
3186         unsigned long address;
3187         pte_t *ptep;
3188         pte_t pte;
3189         spinlock_t *ptl;
3190         struct page *page;
3191         struct hstate *h = hstate_vma(vma);
3192         unsigned long sz = huge_page_size(h);
3193         const unsigned long mmun_start = start; /* For mmu_notifiers */
3194         const unsigned long mmun_end   = end;   /* For mmu_notifiers */
3195 
3196         WARN_ON(!is_vm_hugetlb_page(vma));
3197         BUG_ON(start & ~huge_page_mask(h));
3198         BUG_ON(end & ~huge_page_mask(h));
3199 
3200         tlb_start_vma(tlb, vma);
3201         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
3202         address = start;
3203 again:
3204         for (; address < end; address += sz) {
3205                 ptep = huge_pte_offset(mm, address);
3206                 if (!ptep)
3207                         continue;
3208 
3209                 ptl = huge_pte_lock(h, mm, ptep);
3210                 if (huge_pmd_unshare(mm, &address, ptep))
3211                         goto unlock;
3212 
3213                 pte = huge_ptep_get(ptep);
3214                 if (huge_pte_none(pte))
3215                         goto unlock;
3216 
3217                 /*
3218                  * Migrating hugepage or HWPoisoned hugepage is already
3219                  * unmapped and its refcount is dropped, so just clear pte here.
3220                  */
3221                 if (unlikely(!pte_present(pte))) {
3222                         huge_pte_clear(mm, address, ptep);
3223                         goto unlock;
3224                 }
3225 
3226                 page = pte_page(pte);
3227                 /*
3228                  * If a reference page is supplied, it is because a specific
3229                  * page is being unmapped, not a range. Ensure the page we
3230                  * are about to unmap is the actual page of interest.
3231                  */
3232                 if (ref_page) {
3233                         if (page != ref_page)
3234                                 goto unlock;
3235 
3236                         /*
3237                          * Mark the VMA as having unmapped its page so that
3238                          * future faults in this VMA will fail rather than
3239                          * looking like data was lost
3240                          */
3241                         set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
3242                 }
3243 
3244                 pte = huge_ptep_get_and_clear(mm, address, ptep);
3245                 tlb_remove_tlb_entry(tlb, ptep, address);
3246                 if (huge_pte_dirty(pte))
3247                         set_page_dirty(page);
3248 
3249                 hugetlb_count_sub(pages_per_huge_page(h), mm);
3250                 page_remove_rmap(page, true);
3251                 force_flush = !__tlb_remove_page(tlb, page);
3252                 if (force_flush) {
3253                         address += sz;
3254                         spin_unlock(ptl);
3255                         break;
3256                 }
3257                 /* Bail out after unmapping reference page if supplied */
3258                 if (ref_page) {
3259                         spin_unlock(ptl);
3260                         break;
3261                 }
3262 unlock:
3263                 spin_unlock(ptl);
3264         }
3265         /*
3266          * mmu_gather ran out of room to batch pages, we break out of
3267          * the PTE lock to avoid doing the potential expensive TLB invalidate
3268          * and page-free while holding it.
3269          */
3270         if (force_flush) {
3271                 force_flush = 0;
3272                 tlb_flush_mmu(tlb);
3273                 if (address < end && !ref_page)
3274                         goto again;
3275         }
3276         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
3277         tlb_end_vma(tlb, vma);
3278 }
3279 
3280 void __unmap_hugepage_range_final(struct mmu_gather *tlb,
3281                           struct vm_area_struct *vma, unsigned long start,
3282                           unsigned long end, struct page *ref_page)
3283 {
3284         __unmap_hugepage_range(tlb, vma, start, end, ref_page);
3285 
3286         /*
3287          * Clear this flag so that x86's huge_pmd_share page_table_shareable
3288          * test will fail on a vma being torn down, and not grab a page table
3289          * on its way out.  We're lucky that the flag has such an appropriate
3290          * name, and can in fact be safely cleared here. We could clear it
3291          * before the __unmap_hugepage_range above, but all that's necessary
3292          * is to clear it before releasing the i_mmap_rwsem. This works
3293          * because in the context this is called, the VMA is about to be
3294          * destroyed and the i_mmap_rwsem is held.
3295          */
3296         vma->vm_flags &= ~VM_MAYSHARE;
3297 }
3298 
3299 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
3300                           unsigned long end, struct page *ref_page)
3301 {
3302         struct mm_struct *mm;
3303         struct mmu_gather tlb;
3304 
3305         mm = vma->vm_mm;
3306 
3307         tlb_gather_mmu(&tlb, mm, start, end);
3308         __unmap_hugepage_range(&tlb, vma, start, end, ref_page);
3309         tlb_finish_mmu(&tlb, start, end);
3310 }
3311 
3312 /*
3313  * This is called when the original mapper is failing to COW a MAP_PRIVATE
3314  * mappping it owns the reserve page for. The intention is to unmap the page
3315  * from other VMAs and let the children be SIGKILLed if they are faulting the
3316  * same region.
3317  */
3318 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
3319                               struct page *page, unsigned long address)
3320 {
3321         struct hstate *h = hstate_vma(vma);
3322         struct vm_area_struct *iter_vma;
3323         struct address_space *mapping;
3324         pgoff_t pgoff;
3325 
3326         /*
3327          * vm_pgoff is in PAGE_SIZE units, hence the different calculation
3328          * from page cache lookup which is in HPAGE_SIZE units.
3329          */
3330         address = address & huge_page_mask(h);
3331         pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
3332                         vma->vm_pgoff;
3333         mapping = file_inode(vma->vm_file)->i_mapping;
3334 
3335         /*
3336          * Take the mapping lock for the duration of the table walk. As
3337          * this mapping should be shared between all the VMAs,
3338          * __unmap_hugepage_range() is called as the lock is already held
3339          */
3340         i_mmap_lock_write(mapping);
3341         vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
3342                 /* Do not unmap the current VMA */
3343                 if (iter_vma == vma)
3344                         continue;
3345 
3346                 /*
3347                  * Shared VMAs have their own reserves and do not affect
3348                  * MAP_PRIVATE accounting but it is possible that a shared
3349                  * VMA is using the same page so check and skip such VMAs.
3350                  */
3351                 if (iter_vma->vm_flags & VM_MAYSHARE)
3352                         continue;
3353 
3354                 /*
3355                  * Unmap the page from other VMAs without their own reserves.
3356                  * They get marked to be SIGKILLed if they fault in these
3357                  * areas. This is because a future no-page fault on this VMA
3358                  * could insert a zeroed page instead of the data existing
3359                  * from the time of fork. This would look like data corruption
3360                  */
3361                 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
3362                         unmap_hugepage_range(iter_vma, address,
3363                                              address + huge_page_size(h), page);
3364         }
3365         i_mmap_unlock_write(mapping);
3366 }
3367 
3368 /*
3369  * Hugetlb_cow() should be called with page lock of the original hugepage held.
3370  * Called with hugetlb_instantiation_mutex held and pte_page locked so we
3371  * cannot race with other handlers or page migration.
3372  * Keep the pte_same checks anyway to make transition from the mutex easier.
3373  */
3374 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
3375                         unsigned long address, pte_t *ptep, pte_t pte,
3376                         struct page *pagecache_page, spinlock_t *ptl)
3377 {
3378         struct hstate *h = hstate_vma(vma);
3379         struct page *old_page, *new_page;
3380         int ret = 0, outside_reserve = 0;
3381         unsigned long mmun_start;       /* For mmu_notifiers */
3382         unsigned long mmun_end;         /* For mmu_notifiers */
3383 
3384         old_page = pte_page(pte);
3385 
3386 retry_avoidcopy:
3387         /* If no-one else is actually using this page, avoid the copy
3388          * and just make the page writable */
3389         if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
3390                 page_move_anon_rmap(old_page, vma);
3391                 set_huge_ptep_writable(vma, address, ptep);
3392                 return 0;
3393         }
3394 
3395         /*
3396          * If the process that created a MAP_PRIVATE mapping is about to
3397          * perform a COW due to a shared page count, attempt to satisfy
3398          * the allocation without using the existing reserves. The pagecache
3399          * page is used to determine if the reserve at this address was
3400          * consumed or not. If reserves were used, a partial faulted mapping
3401          * at the time of fork() could consume its reserves on COW instead
3402          * of the full address range.
3403          */
3404         if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
3405                         old_page != pagecache_page)
3406                 outside_reserve = 1;
3407 
3408         get_page(old_page);
3409 
3410         /*
3411          * Drop page table lock as buddy allocator may be called. It will
3412          * be acquired again before returning to the caller, as expected.
3413          */
3414         spin_unlock(ptl);
3415         new_page = alloc_huge_page(vma, address, outside_reserve);
3416 
3417         if (IS_ERR(new_page)) {
3418                 /*
3419                  * If a process owning a MAP_PRIVATE mapping fails to COW,
3420                  * it is due to references held by a child and an insufficient
3421                  * huge page pool. To guarantee the original mappers
3422                  * reliability, unmap the page from child processes. The child
3423                  * may get SIGKILLed if it later faults.
3424                  */
3425                 if (outside_reserve) {
3426                         put_page(old_page);
3427                         BUG_ON(huge_pte_none(pte));
3428                         unmap_ref_private(mm, vma, old_page, address);
3429                         BUG_ON(huge_pte_none(pte));
3430                         spin_lock(ptl);
3431                         ptep = huge_pte_offset(mm, address & huge_page_mask(h));
3432                         if (likely(ptep &&
3433                                    pte_same(huge_ptep_get(ptep), pte)))
3434                                 goto retry_avoidcopy;
3435                         /*
3436                          * race occurs while re-acquiring page table
3437                          * lock, and our job is done.
3438                          */
3439                         return 0;
3440                 }
3441 
3442                 ret = (PTR_ERR(new_page) == -ENOMEM) ?
3443                         VM_FAULT_OOM : VM_FAULT_SIGBUS;
3444                 goto out_release_old;
3445         }
3446 
3447         /*
3448          * When the original hugepage is shared one, it does not have
3449          * anon_vma prepared.
3450          */
3451         if (unlikely(anon_vma_prepare(vma))) {
3452                 ret = VM_FAULT_OOM;
3453                 goto out_release_all;
3454         }
3455 
3456         copy_user_huge_page(new_page, old_page, address, vma,
3457                             pages_per_huge_page(h));
3458         __SetPageUptodate(new_page);
3459         set_page_huge_active(new_page);
3460 
3461         mmun_start = address & huge_page_mask(h);
3462         mmun_end = mmun_start + huge_page_size(h);
3463         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
3464 
3465         /*
3466          * Retake the page table lock to check for racing updates
3467          * before the page tables are altered
3468          */
3469         spin_lock(ptl);
3470         ptep = huge_pte_offset(mm, address & huge_page_mask(h));
3471         if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
3472                 ClearPagePrivate(new_page);
3473 
3474                 /* Break COW */
3475                 huge_ptep_clear_flush(vma, address, ptep);
3476                 mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
3477                 set_huge_pte_at(mm, address, ptep,
3478                                 make_huge_pte(vma, new_page, 1));
3479                 page_remove_rmap(old_page, true);
3480                 hugepage_add_new_anon_rmap(new_page, vma, address);
3481                 /* Make the old page be freed below */
3482                 new_page = old_page;
3483         }
3484         spin_unlock(ptl);
3485         mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
3486 out_release_all:
3487         put_page(new_page);
3488 out_release_old:
3489         put_page(old_page);
3490 
3491         spin_lock(ptl); /* Caller expects lock to be held */
3492         return ret;
3493 }
3494 
3495 /* Return the pagecache page at a given address within a VMA */
3496 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
3497                         struct vm_area_struct *vma, unsigned long address)
3498 {
3499         struct address_space *mapping;
3500         pgoff_t idx;
3501 
3502         mapping = vma->vm_file->f_mapping;
3503         idx = vma_hugecache_offset(h, vma, address);
3504 
3505         return find_lock_page(mapping, idx);
3506 }
3507 
3508 /*
3509  * Return whether there is a pagecache page to back given address within VMA.
3510  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
3511  */
3512 static bool hugetlbfs_pagecache_present(struct hstate *h,
3513                         struct vm_area_struct *vma, unsigned long address)
3514 {
3515         struct address_space *mapping;
3516         pgoff_t idx;
3517         struct page *page;
3518 
3519         mapping = vma->vm_file->f_mapping;
3520         idx = vma_hugecache_offset(h, vma, address);
3521 
3522         page = find_get_page(mapping, idx);
3523         if (page)
3524                 put_page(page);
3525         return page != NULL;
3526 }
3527 
3528 int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
3529                            pgoff_t idx)
3530 {
3531         struct inode *inode = mapping->host;
3532         struct hstate *h = hstate_inode(inode);
3533         int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
3534 
3535         if (err)
3536                 return err;
3537         ClearPagePrivate(page);
3538 
3539         spin_lock(&inode->i_lock);
3540         inode->i_blocks += blocks_per_huge_page(h);
3541         spin_unlock(&inode->i_lock);
3542         return 0;
3543 }
3544 
3545 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
3546                            struct address_space *mapping, pgoff_t idx,
3547                            unsigned long address, pte_t *ptep, unsigned int flags)
3548 {
3549         struct hstate *h = hstate_vma(vma);
3550         int ret = VM_FAULT_SIGBUS;
3551         int anon_rmap = 0;
3552         unsigned long size;
3553         struct page *page;
3554         pte_t new_pte;
3555         spinlock_t *ptl;
3556 
3557         /*
3558          * Currently, we are forced to kill the process in the event the
3559          * original mapper has unmapped pages from the child due to a failed
3560          * COW. Warn that such a situation has occurred as it may not be obvious
3561          */
3562         if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
3563                 pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
3564                            current->pid);
3565                 return ret;
3566         }
3567 
3568         /*
3569          * Use page lock to guard against racing truncation
3570          * before we get page_table_lock.
3571          */
3572 retry:
3573         page = find_lock_page(mapping, idx);
3574         if (!page) {
3575                 size = i_size_read(mapping->host) >> huge_page_shift(h);
3576                 if (idx >= size)
3577                         goto out;
3578                 page = alloc_huge_page(vma, address, 0);
3579                 if (IS_ERR(page)) {
3580                         ret = PTR_ERR(page);
3581                         if (ret == -ENOMEM)
3582                                 ret = VM_FAULT_OOM;
3583                         else
3584                                 ret = VM_FAULT_SIGBUS;
3585                         goto out;
3586                 }
3587                 clear_huge_page(page, address, pages_per_huge_page(h));
3588                 __SetPageUptodate(page);
3589                 set_page_huge_active(page);
3590 
3591                 if (vma->vm_flags & VM_MAYSHARE) {
3592                         int err = huge_add_to_page_cache(page, mapping, idx);
3593                         if (err) {
3594                                 put_page(page);
3595                                 if (err == -EEXIST)
3596                                         goto retry;
3597                                 goto out;
3598                         }
3599                 } else {
3600                         lock_page(page);
3601                         if (unlikely(anon_vma_prepare(vma))) {
3602                                 ret = VM_FAULT_OOM;
3603                                 goto backout_unlocked;
3604                         }
3605                         anon_rmap = 1;
3606                 }
3607         } else {
3608                 /*
3609                  * If memory error occurs between mmap() and fault, some process
3610                  * don't have hwpoisoned swap entry for errored virtual address.
3611                  * So we need to block hugepage fault by PG_hwpoison bit check.
3612                  */
3613                 if (unlikely(PageHWPoison(page))) {
3614                         ret = VM_FAULT_HWPOISON |
3615                                 VM_FAULT_SET_HINDEX(hstate_index(h));
3616                         goto backout_unlocked;
3617                 }
3618         }
3619 
3620         /*
3621          * If we are going to COW a private mapping later, we examine the
3622          * pending reservations for this page now. This will ensure that
3623          * any allocations necessary to record that reservation occur outside
3624          * the spinlock.
3625          */
3626         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3627                 if (vma_needs_reservation(h, vma, address) < 0) {
3628                         ret = VM_FAULT_OOM;
3629                         goto backout_unlocked;
3630                 }
3631                 /* Just decrements count, does not deallocate */
3632                 vma_end_reservation(h, vma, address);
3633         }
3634 
3635         ptl = huge_pte_lockptr(h, mm, ptep);
3636         spin_lock(ptl);
3637         size = i_size_read(mapping->host) >> huge_page_shift(h);
3638         if (idx >= size)
3639                 goto backout;
3640 
3641         ret = 0;
3642         if (!huge_pte_none(huge_ptep_get(ptep)))
3643                 goto backout;
3644 
3645         if (anon_rmap) {
3646                 ClearPagePrivate(page);
3647                 hugepage_add_new_anon_rmap(page, vma, address);
3648         } else
3649                 page_dup_rmap(page, true);
3650         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
3651                                 && (vma->vm_flags & VM_SHARED)));
3652         set_huge_pte_at(mm, address, ptep, new_pte);
3653 
3654         hugetlb_count_add(pages_per_huge_page(h), mm);
3655         if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3656                 /* Optimization, do the COW without a second fault */
3657                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page, ptl);
3658         }
3659 
3660         spin_unlock(ptl);
3661         unlock_page(page);
3662 out:
3663         return ret;
3664 
3665 backout:
3666         spin_unlock(ptl);
3667 backout_unlocked:
3668         unlock_page(page);
3669         put_page(page);
3670         goto out;
3671 }
3672 
3673 #ifdef CONFIG_SMP
3674 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3675                             struct vm_area_struct *vma,
3676                             struct address_space *mapping,
3677                             pgoff_t idx, unsigned long address)
3678 {
3679         unsigned long key[2];
3680         u32 hash;
3681 
3682         if (vma->vm_flags & VM_SHARED) {
3683                 key[0] = (unsigned long) mapping;
3684                 key[1] = idx;
3685         } else {
3686                 key[0] = (unsigned long) mm;
3687                 key[1] = address >> huge_page_shift(h);
3688         }
3689 
3690         hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0);
3691 
3692         return hash & (num_fault_mutexes - 1);
3693 }
3694 #else
3695 /*
3696  * For uniprocesor systems we always use a single mutex, so just
3697  * return 0 and avoid the hashing overhead.
3698  */
3699 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3700                             struct vm_area_struct *vma,
3701                             struct address_space *mapping,
3702                             pgoff_t idx, unsigned long address)
3703 {
3704         return 0;
3705 }
3706 #endif
3707 
3708 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
3709                         unsigned long address, unsigned int flags)
3710 {
3711         pte_t *ptep, entry;
3712         spinlock_t *ptl;
3713         int ret;
3714         u32 hash;
3715         pgoff_t idx;
3716         struct page *page = NULL;
3717         struct page *pagecache_page = NULL;
3718         struct hstate *h = hstate_vma(vma);
3719         struct address_space *mapping;
3720         int need_wait_lock = 0;
3721 
3722         address &= huge_page_mask(h);
3723 
3724         ptep = huge_pte_offset(mm, address);
3725         if (ptep) {
3726                 entry = huge_ptep_get(ptep);
3727                 if (unlikely(is_hugetlb_entry_migration(entry))) {
3728                         migration_entry_wait_huge(vma, mm, ptep);
3729                         return 0;
3730                 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
3731                         return VM_FAULT_HWPOISON_LARGE |
3732                                 VM_FAULT_SET_HINDEX(hstate_index(h));
3733         } else {
3734                 ptep = huge_pte_alloc(mm, address, huge_page_size(h));
3735                 if (!ptep)
3736                         return VM_FAULT_OOM;
3737         }
3738 
3739         mapping = vma->vm_file->f_mapping;
3740         idx = vma_hugecache_offset(h, vma, address);
3741 
3742         /*
3743          * Serialize hugepage allocation and instantiation, so that we don't
3744          * get spurious allocation failures if two CPUs race to instantiate
3745          * the same page in the page cache.
3746          */
3747         hash = hugetlb_fault_mutex_hash(h, mm, vma, mapping, idx, address);
3748         mutex_lock(&hugetlb_fault_mutex_table[hash]);
3749 
3750         entry = huge_ptep_get(ptep);
3751         if (huge_pte_none(entry)) {
3752                 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
3753                 goto out_mutex;
3754         }
3755 
3756         ret = 0;
3757 
3758         /*
3759          * entry could be a migration/hwpoison entry at this point, so this
3760          * check prevents the kernel from going below assuming that we have
3761          * a active hugepage in pagecache. This goto expects the 2nd page fault,
3762          * and is_hugetlb_entry_(migration|hwpoisoned) check will properly
3763          * handle it.
3764          */
3765         if (!pte_present(entry))
3766                 goto out_mutex;
3767 
3768         /*
3769          * If we are going to COW the mapping later, we examine the pending
3770          * reservations for this page now. This will ensure that any
3771          * allocations necessary to record that reservation occur outside the
3772          * spinlock. For private mappings, we also lookup the pagecache
3773          * page now as it is used to determine if a reservation has been
3774          * consumed.
3775          */
3776         if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
3777                 if (vma_needs_reservation(h, vma, address) < 0) {
3778                         ret = VM_FAULT_OOM;
3779                         goto out_mutex;
3780                 }
3781                 /* Just decrements count, does not deallocate */
3782                 vma_end_reservation(h, vma, address);
3783 
3784                 if (!(vma->vm_flags & VM_MAYSHARE))
3785                         pagecache_page = hugetlbfs_pagecache_page(h,
3786                                                                 vma, address);
3787         }
3788 
3789         ptl = huge_pte_lock(h, mm, ptep);
3790 
3791         /* Check for a racing update before calling hugetlb_cow */
3792         if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
3793                 goto out_ptl;
3794 
3795         /*
3796          * hugetlb_cow() requires page locks of pte_page(entry) and
3797          * pagecache_page, so here we need take the former one
3798          * when page != pagecache_page or !pagecache_page.
3799          */
3800         page = pte_page(entry);
3801         if (page != pagecache_page)
3802                 if (!trylock_page(page)) {
3803                         need_wait_lock = 1;
3804                         goto out_ptl;
3805                 }
3806 
3807         get_page(page);
3808 
3809         if (flags & FAULT_FLAG_WRITE) {
3810                 if (!huge_pte_write(entry)) {
3811                         ret = hugetlb_cow(mm, vma, address, ptep, entry,
3812                                         pagecache_page, ptl);
3813                         goto out_put_page;
3814                 }
3815                 entry = huge_pte_mkdirty(entry);
3816         }
3817         entry = pte_mkyoung(entry);
3818         if (huge_ptep_set_access_flags(vma, address, ptep, entry,
3819                                                 flags & FAULT_FLAG_WRITE))
3820                 update_mmu_cache(vma, address, ptep);
3821 out_put_page:
3822         if (page != pagecache_page)
3823                 unlock_page(page);
3824         put_page(page);
3825 out_ptl:
3826         spin_unlock(ptl);
3827 
3828         if (pagecache_page) {
3829                 unlock_page(pagecache_page);
3830                 put_page(pagecache_page);
3831         }
3832 out_mutex:
3833         mutex_unlock(&hugetlb_fault_mutex_table[hash]);
3834         /*
3835          * Generally it's safe to hold refcount during waiting page lock. But
3836          * here we just wait to defer the next page fault to avoid busy loop and
3837          * the page is not used after unlocked before returning from the current
3838          * page fault. So we are safe from accessing freed page, even if we wait
3839          * here without taking refcount.
3840          */
3841         if (need_wait_lock)
3842                 wait_on_page_locked(page);
3843         return ret;
3844 }
3845 
3846 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
3847                          struct page **pages, struct vm_area_struct **vmas,
3848                          unsigned long *position, unsigned long *nr_pages,
3849                          long i, unsigned int flags)
3850 {
3851         unsigned long pfn_offset;
3852         unsigned long vaddr = *position;
3853         unsigned long remainder = *nr_pages;
3854         struct hstate *h = hstate_vma(vma);
3855 
3856         while (vaddr < vma->vm_end && remainder) {
3857                 pte_t *pte;
3858                 spinlock_t *ptl = NULL;
3859                 int absent;
3860                 struct page *page;
3861 
3862                 /*
3863                  * If we have a pending SIGKILL, don't keep faulting pages and
3864                  * potentially allocating memory.
3865                  */
3866                 if (unlikely(fatal_signal_pending(current))) {
3867                         remainder = 0;
3868                         break;
3869                 }
3870 
3871                 /*
3872                  * Some archs (sparc64, sh*) have multiple pte_ts to
3873                  * each hugepage.  We have to make sure we get the
3874                  * first, for the page indexing below to work.
3875                  *
3876                  * Note that page table lock is not held when pte is null.
3877                  */
3878                 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
3879                 if (pte)
3880                         ptl = huge_pte_lock(h, mm, pte);
3881                 absent = !pte || huge_pte_none(huge_ptep_get(pte));
3882 
3883                 /*
3884                  * When coredumping, it suits get_dump_page if we just return
3885                  * an error where there's an empty slot with no huge pagecache
3886                  * to back it.  This way, we avoid allocating a hugepage, and
3887                  * the sparse dumpfile avoids allocating disk blocks, but its
3888                  * huge holes still show up with zeroes where they need to be.
3889                  */
3890                 if (absent && (flags & FOLL_DUMP) &&
3891                     !hugetlbfs_pagecache_present(h, vma, vaddr)) {
3892                         if (pte)
3893                                 spin_unlock(ptl);
3894                         remainder = 0;
3895                         break;
3896                 }
3897 
3898                 /*
3899                  * We need call hugetlb_fault for both hugepages under migration
3900                  * (in which case hugetlb_fault waits for the migration,) and
3901                  * hwpoisoned hugepages (in which case we need to prevent the
3902                  * caller from accessing to them.) In order to do this, we use
3903                  * here is_swap_pte instead of is_hugetlb_entry_migration and
3904                  * is_hugetlb_entry_hwpoisoned. This is because it simply covers
3905                  * both cases, and because we can't follow correct pages
3906                  * directly from any kind of swap entries.
3907                  */
3908                 if (absent || is_swap_pte(huge_ptep_get(pte)) ||
3909                     ((flags & FOLL_WRITE) &&
3910                       !huge_pte_write(huge_ptep_get(pte)))) {
3911                         int ret;
3912 
3913                         if (pte)
3914                                 spin_unlock(ptl);
3915                         ret = hugetlb_fault(mm, vma, vaddr,
3916                                 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
3917                         if (!(ret & VM_FAULT_ERROR))
3918                                 continue;
3919 
3920                         remainder = 0;
3921                         break;
3922                 }
3923 
3924                 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
3925                 page = pte_page(huge_ptep_get(pte));
3926 same_page:
3927                 if (pages) {
3928                         pages[i] = mem_map_offset(page, pfn_offset);
3929                         get_page(pages[i]);
3930                 }
3931 
3932                 if (vmas)
3933                         vmas[i] = vma;
3934 
3935                 vaddr += PAGE_SIZE;
3936                 ++pfn_offset;
3937                 --remainder;
3938                 ++i;
3939                 if (vaddr < vma->vm_end && remainder &&
3940                                 pfn_offset < pages_per_huge_page(h)) {
3941                         /*
3942                          * We use pfn_offset to avoid touching the pageframes
3943                          * of this compound page.
3944                          */
3945                         goto same_page;
3946                 }
3947                 spin_unlock(ptl);
3948         }
3949         *nr_pages = remainder;
3950         *position = vaddr;
3951 
3952         return i ? i : -EFAULT;
3953 }
3954 
3955 unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
3956                 unsigned long address, unsigned long end, pgprot_t newprot)
3957 {
3958         struct mm_struct *mm = vma->vm_mm;
3959         unsigned long start = address;
3960         pte_t *ptep;
3961         pte_t pte;
3962         struct hstate *h = hstate_vma(vma);
3963         unsigned long pages = 0;
3964 
3965         BUG_ON(address >= end);
3966         flush_cache_range(vma, address, end);
3967 
3968         mmu_notifier_invalidate_range_start(mm, start, end);
3969         i_mmap_lock_write(vma->vm_file->f_mapping);
3970         for (; address < end; address += huge_page_size(h)) {
3971                 spinlock_t *ptl;
3972                 ptep = huge_pte_offset(mm, address);
3973                 if (!ptep)
3974                         continue;
3975                 ptl = huge_pte_lock(h, mm, ptep);
3976                 if (huge_pmd_unshare(mm, &address, ptep)) {
3977                         pages++;
3978                         spin_unlock(ptl);
3979                         continue;
3980                 }
3981                 pte = huge_ptep_get(ptep);
3982                 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
3983                         spin_unlock(ptl);
3984                         continue;
3985                 }
3986                 if (unlikely(is_hugetlb_entry_migration(pte))) {
3987                         swp_entry_t entry = pte_to_swp_entry(pte);
3988 
3989                         if (is_write_migration_entry(entry)) {
3990                                 pte_t newpte;
3991 
3992                                 make_migration_entry_read(&entry);
3993                                 newpte = swp_entry_to_pte(entry);
3994                                 set_huge_pte_at(mm, address, ptep, newpte);
3995                                 pages++;
3996                         }
3997                         spin_unlock(ptl);
3998                         continue;
3999                 }
4000                 if (!huge_pte_none(pte)) {
4001                         pte = huge_ptep_get_and_clear(mm, address, ptep);
4002                         pte = pte_mkhuge(huge_pte_modify(pte, newprot));
4003                         pte = arch_make_huge_pte(pte, vma, NULL, 0);
4004                         set_huge_pte_at(mm, address, ptep, pte);
4005                         pages++;
4006                 }
4007                 spin_unlock(ptl);
4008         }
4009         /*
4010          * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
4011          * may have cleared our pud entry and done put_page on the page table:
4012          * once we release i_mmap_rwsem, another task can do the final put_page
4013          * and that page table be reused and filled with junk.
4014          */
4015         flush_tlb_range(vma, start, end);
4016         mmu_notifier_invalidate_range(mm, start, end);
4017         i_mmap_unlock_write(vma->vm_file->f_mapping);
4018         mmu_notifier_invalidate_range_end(mm, start, end);
4019 
4020         return pages << h->order;
4021 }
4022 
4023 int hugetlb_reserve_pages(struct inode *inode,
4024                                         long from, long to,
4025                                         struct vm_area_struct *vma,
4026                                         vm_flags_t vm_flags)
4027 {
4028         long ret, chg;
4029         struct hstate *h = hstate_inode(inode);
4030         struct hugepage_subpool *spool = subpool_inode(inode);
4031         struct resv_map *resv_map;
4032         long gbl_reserve;
4033 
4034         /*
4035          * Only apply hugepage reservation if asked. At fault time, an
4036          * attempt will be made for VM_NORESERVE to allocate a page
4037          * without using reserves
4038          */
4039         if (vm_flags & VM_NORESERVE)
4040                 return 0;
4041 
4042         /*
4043          * Shared mappings base their reservation on the number of pages that
4044          * are already allocated on behalf of the file. Private mappings need
4045          * to reserve the full area even if read-only as mprotect() may be
4046          * called to make the mapping read-write. Assume !vma is a shm mapping
4047          */
4048         if (!vma || vma->vm_flags & VM_MAYSHARE) {
4049                 resv_map = inode_resv_map(inode);
4050 
4051                 chg = region_chg(resv_map, from, to);
4052 
4053         } else {
4054                 resv_map = resv_map_alloc();
4055                 if (!resv_map)
4056                         return -ENOMEM;
4057 
4058                 chg = to - from;
4059 
4060                 set_vma_resv_map(vma, resv_map);
4061                 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
4062         }
4063 
4064         if (chg < 0) {
4065                 ret = chg;
4066                 goto out_err;
4067         }
4068 
4069         /*
4070          * There must be enough pages in the subpool for the mapping. If
4071          * the subpool has a minimum size, there may be some global
4072          * reservations already in place (gbl_reserve).
4073          */
4074         gbl_reserve = hugepage_subpool_get_pages(spool, chg);
4075         if (gbl_reserve < 0) {
4076                 ret = -ENOSPC;
4077                 goto out_err;
4078         }
4079 
4080         /*
4081          * Check enough hugepages are available for the reservation.
4082          * Hand the pages back to the subpool if there are not
4083          */
4084         ret = hugetlb_acct_memory(h, gbl_reserve);
4085         if (ret < 0) {
4086                 /* put back original number of pages, chg */
4087                 (void)hugepage_subpool_put_pages(spool, chg);
4088                 goto out_err;
4089         }
4090 
4091         /*
4092          * Account for the reservations made. Shared mappings record regions
4093          * that have reservations as they are shared by multiple VMAs.
4094          * When the last VMA disappears, the region map says how much
4095          * the reservation was and the page cache tells how much of
4096          * the reservation was consumed. Private mappings are per-VMA and
4097          * only the consumed reservations are tracked. When the VMA
4098          * disappears, the original reservation is the VMA size and the
4099          * consumed reservations are stored in the map. Hence, nothing
4100          * else has to be done for private mappings here
4101          */
4102         if (!vma || vma->vm_flags & VM_MAYSHARE) {
4103                 long add = region_add(resv_map, from, to);
4104 
4105                 if (unlikely(chg > add)) {
4106                         /*
4107                          * pages in this range were added to the reserve
4108                          * map between region_chg and region_add.  This
4109                          * indicates a race with alloc_huge_page.  Adjust
4110                          * the subpool and reserve counts modified above
4111                          * based on the difference.
4112                          */
4113                         long rsv_adjust;
4114 
4115                         rsv_adjust = hugepage_subpool_put_pages(spool,
4116                                                                 chg - add);
4117                         hugetlb_acct_memory(h, -rsv_adjust);
4118                 }
4119         }
4120         return 0;
4121 out_err:
4122         if (!vma || vma->vm_flags & VM_MAYSHARE)
4123                 region_abort(resv_map, from, to);
4124         if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
4125                 kref_put(&resv_map->refs, resv_map_release);
4126         return ret;
4127 }
4128 
4129 long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
4130                                                                 long freed)
4131 {
4132         struct hstate *h = hstate_inode(inode);
4133         struct resv_map *resv_map = inode_resv_map(inode);
4134         long chg = 0;
4135         struct hugepage_subpool *spool = subpool_inode(inode);
4136         long gbl_reserve;
4137 
4138         if (resv_map) {
4139                 chg = region_del(resv_map, start, end);
4140                 /*
4141                  * region_del() can fail in the rare case where a region
4142                  * must be split and another region descriptor can not be
4143                  * allocated.  If end == LONG_MAX, it will not fail.
4144                  */
4145                 if (chg < 0)
4146                         return chg;
4147         }
4148 
4149         spin_lock(&inode->i_lock);
4150         inode->i_blocks -= (blocks_per_huge_page(h) * freed);
4151         spin_unlock(&inode->i_lock);
4152 
4153         /*
4154          * If the subpool has a minimum size, the number of global
4155          * reservations to be released may be adjusted.
4156          */
4157         gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
4158         hugetlb_acct_memory(h, -gbl_reserve);
4159 
4160         return 0;
4161 }
4162 
4163 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
4164 static unsigned long page_table_shareable(struct vm_area_struct *svma,
4165                                 struct vm_area_struct *vma,
4166                                 unsigned long addr, pgoff_t idx)
4167 {
4168         unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
4169                                 svma->vm_start;
4170         unsigned long sbase = saddr & PUD_MASK;
4171         unsigned long s_end = sbase + PUD_SIZE;
4172 
4173         /* Allow segments to share if only one is marked locked */
4174         unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
4175         unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK;
4176 
4177         /*
4178          * match the virtual addresses, permission and the alignment of the
4179          * page table page.
4180          */
4181         if (pmd_index(addr) != pmd_index(saddr) ||
4182             vm_flags != svm_flags ||
4183             sbase < svma->vm_start || svma->vm_end < s_end)
4184                 return 0;
4185 
4186         return saddr;
4187 }
4188 
4189 static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
4190 {
4191         unsigned long base = addr & PUD_MASK;
4192         unsigned long end = base + PUD_SIZE;
4193 
4194         /*
4195          * check on proper vm_flags and page table alignment
4196          */
4197         if (vma->vm_flags & VM_MAYSHARE &&
4198             vma->vm_start <= base && end <= vma->vm_end)
4199                 return true;
4200         return false;
4201 }
4202 
4203 /*
4204  * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
4205  * and returns the corresponding pte. While this is not necessary for the
4206  * !shared pmd case because we can allocate the pmd later as well, it makes the
4207  * code much cleaner. pmd allocation is essential for the shared case because
4208  * pud has to be populated inside the same i_mmap_rwsem section - otherwise
4209  * racing tasks could either miss the sharing (see huge_pte_offset) or select a
4210  * bad pmd for sharing.
4211  */
4212 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
4213 {
4214         struct vm_area_struct *vma = find_vma(mm, addr);
4215         struct address_space *mapping = vma->vm_file->f_mapping;
4216         pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
4217                         vma->vm_pgoff;
4218         struct vm_area_struct *svma;
4219         unsigned long saddr;
4220         pte_t *spte = NULL;
4221         pte_t *pte;
4222         spinlock_t *ptl;
4223 
4224         if (!vma_shareable(vma, addr))
4225                 return (pte_t *)pmd_alloc(mm, pud, addr);
4226 
4227         i_mmap_lock_write(mapping);
4228         vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
4229                 if (svma == vma)
4230                         continue;
4231 
4232                 saddr = page_table_shareable(svma, vma, addr, idx);
4233                 if (saddr) {
4234                         spte = huge_pte_offset(svma->vm_mm, saddr);
4235                         if (spte) {
4236                                 get_page(virt_to_page(spte));
4237                                 break;
4238                         }
4239                 }
4240         }
4241 
4242         if (!spte)
4243                 goto out;
4244 
4245         ptl = huge_pte_lockptr(hstate_vma(vma), mm, spte);
4246         spin_lock(ptl);
4247         if (pud_none(*pud)) {
4248                 pud_populate(mm, pud,
4249                                 (pmd_t *)((unsigned long)spte & PAGE_MASK));
4250                 mm_inc_nr_pmds(mm);
4251         } else {
4252                 put_page(virt_to_page(spte));
4253         }
4254         spin_unlock(ptl);
4255 out:
4256         pte = (pte_t *)pmd_alloc(mm, pud, addr);
4257         i_mmap_unlock_write(mapping);
4258         return pte;
4259 }
4260 
4261 /*
4262  * unmap huge page backed by shared pte.
4263  *
4264  * Hugetlb pte page is ref counted at the time of mapping.  If pte is shared
4265  * indicated by page_count > 1, unmap is achieved by clearing pud and
4266  * decrementing the ref count. If count == 1, the pte page is not shared.
4267  *
4268  * called with page table lock held.
4269  *
4270  * returns: 1 successfully unmapped a shared pte page
4271  *          0 the underlying pte page is not shared, or it is the last user
4272  */
4273 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4274 {
4275         pgd_t *pgd = pgd_offset(mm, *addr);
4276         pud_t *pud = pud_offset(pgd, *addr);
4277 
4278         BUG_ON(page_count(virt_to_page(ptep)) == 0);
4279         if (page_count(virt_to_page(ptep)) == 1)
4280                 return 0;
4281 
4282         pud_clear(pud);
4283         put_page(virt_to_page(ptep));
4284         mm_dec_nr_pmds(mm);
4285         *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
4286         return 1;
4287 }
4288 #define want_pmd_share()        (1)
4289 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
4290 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
4291 {
4292         return NULL;
4293 }
4294 
4295 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4296 {
4297         return 0;
4298 }
4299 #define want_pmd_share()        (0)
4300 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
4301 
4302 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
4303 pte_t *huge_pte_alloc(struct mm_struct *mm,
4304                         unsigned long addr, unsigned long sz)
4305 {
4306         pgd_t *pgd;
4307         pud_t *pud;
4308         pte_t *pte = NULL;
4309 
4310         pgd = pgd_offset(mm, addr);
4311         pud = pud_alloc(mm, pgd, addr);
4312         if (pud) {
4313                 if (sz == PUD_SIZE) {
4314                         pte = (pte_t *)pud;
4315                 } else {
4316                         BUG_ON(sz != PMD_SIZE);
4317                         if (want_pmd_share() && pud_none(*pud))
4318                                 pte = huge_pmd_share(mm, addr, pud);
4319                         else
4320                                 pte = (pte_t *)pmd_alloc(mm, pud, addr);
4321                 }
4322         }
4323         BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte));
4324 
4325         return pte;
4326 }
4327 
4328 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
4329 {
4330         pgd_t *pgd;
4331         pud_t *pud;
4332         pmd_t *pmd = NULL;
4333 
4334         pgd = pgd_offset(mm, addr);
4335         if (pgd_present(*pgd)) {
4336                 pud = pud_offset(pgd, addr);
4337                 if (pud_present(*pud)) {
4338                         if (pud_huge(*pud))
4339                                 return (pte_t *)pud;
4340                         pmd = pmd_offset(pud, addr);
4341                 }
4342         }
4343         return (pte_t *) pmd;
4344 }
4345 
4346 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
4347 
4348 /*
4349  * These functions are overwritable if your architecture needs its own
4350  * behavior.
4351  */
4352 struct page * __weak
4353 follow_huge_addr(struct mm_struct *mm, unsigned long address,
4354                               int write)
4355 {
4356         return ERR_PTR(-EINVAL);
4357 }
4358 
4359 struct page * __weak
4360 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
4361                 pmd_t *pmd, int flags)
4362 {
4363         struct page *page = NULL;
4364         spinlock_t *ptl;
4365 retry:
4366         ptl = pmd_lockptr(mm, pmd);
4367         spin_lock(ptl);
4368         /*
4369          * make sure that the address range covered by this pmd is not
4370          * unmapped from other threads.
4371          */
4372         if (!pmd_huge(*pmd))
4373                 goto out;
4374         if (pmd_present(*pmd)) {
4375                 page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT);
4376                 if (flags & FOLL_GET)
4377                         get_page(page);
4378         } else {
4379                 if (is_hugetlb_entry_migration(huge_ptep_get((pte_t *)pmd))) {
4380                         spin_unlock(ptl);
4381                         __migration_entry_wait(mm, (pte_t *)pmd, ptl);
4382                         goto retry;
4383                 }
4384                 /*
4385                  * hwpoisoned entry is treated as no_page_table in
4386                  * follow_page_mask().
4387                  */
4388         }
4389 out:
4390         spin_unlock(ptl);
4391         return page;
4392 }
4393 
4394 struct page * __weak
4395 follow_huge_pud(struct mm_struct *mm, unsigned long address,
4396                 pud_t *pud, int flags)
4397 {
4398         if (flags & FOLL_GET)
4399                 return NULL;
4400 
4401         return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
4402 }
4403 
4404 #ifdef CONFIG_MEMORY_FAILURE
4405 
4406 /*
4407  * This function is called from memory failure code.
4408  * Assume the caller holds page lock of the head page.
4409  */
4410 int dequeue_hwpoisoned_huge_page(struct page *hpage)
4411 {
4412         struct hstate *h = page_hstate(hpage);
4413         int nid = page_to_nid(hpage);
4414         int ret = -EBUSY;
4415 
4416         spin_lock(&hugetlb_lock);
4417         /*
4418          * Just checking !page_huge_active is not enough, because that could be
4419          * an isolated/hwpoisoned hugepage (which have >0 refcount).
4420          */
4421         if (!page_huge_active(hpage) && !page_count(hpage)) {
4422                 /*
4423                  * Hwpoisoned hugepage isn't linked to activelist or freelist,
4424                  * but dangling hpage->lru can trigger list-debug warnings
4425                  * (this happens when we call unpoison_memory() on it),
4426                  * so let it point to itself with list_del_init().
4427                  */
4428                 list_del_init(&hpage->lru);
4429                 set_page_refcounted(hpage);
4430                 h->free_huge_pages--;
4431                 h->free_huge_pages_node[nid]--;
4432                 ret = 0;
4433         }
4434         spin_unlock(&hugetlb_lock);
4435         return ret;
4436 }
4437 #endif
4438 
4439 bool isolate_huge_page(struct page *page, struct list_head *list)
4440 {
4441         bool ret = true;
4442 
4443         VM_BUG_ON_PAGE(!PageHead(page), page);
4444         spin_lock(&hugetlb_lock);
4445         if (!page_huge_active(page) || !get_page_unless_zero(page)) {
4446                 ret = false;
4447                 goto unlock;
4448         }
4449         clear_page_huge_active(page);
4450         list_move_tail(&page->lru, list);
4451 unlock:
4452         spin_unlock(&hugetlb_lock);
4453         return ret;
4454 }
4455 
4456 void putback_active_hugepage(struct page *page)
4457 {
4458         VM_BUG_ON_PAGE(!PageHead(page), page);
4459         spin_lock(&hugetlb_lock);
4460         set_page_huge_active(page);
4461         list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
4462         spin_unlock(&hugetlb_lock);
4463         put_page(page);
4464 }
4465 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp