~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/page_alloc.c

Version: ~ [ linux-5.1-rc1 ] ~ [ linux-5.0.3 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.30 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.107 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.164 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.176 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.136 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.63 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *  linux/mm/page_alloc.c
  3  *
  4  *  Manages the free list, the system allocates free pages here.
  5  *  Note that kmalloc() lives in slab.c
  6  *
  7  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
  8  *  Swap reorganised 29.12.95, Stephen Tweedie
  9  *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
 10  *  Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
 11  *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
 12  *  Zone balancing, Kanoj Sarcar, SGI, Jan 2000
 13  *  Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
 14  *          (lots of bits borrowed from Ingo Molnar & Andrew Morton)
 15  */
 16 
 17 #include <linux/stddef.h>
 18 #include <linux/mm.h>
 19 #include <linux/highmem.h>
 20 #include <linux/swap.h>
 21 #include <linux/interrupt.h>
 22 #include <linux/pagemap.h>
 23 #include <linux/jiffies.h>
 24 #include <linux/memblock.h>
 25 #include <linux/compiler.h>
 26 #include <linux/kernel.h>
 27 #include <linux/kasan.h>
 28 #include <linux/module.h>
 29 #include <linux/suspend.h>
 30 #include <linux/pagevec.h>
 31 #include <linux/blkdev.h>
 32 #include <linux/slab.h>
 33 #include <linux/ratelimit.h>
 34 #include <linux/oom.h>
 35 #include <linux/topology.h>
 36 #include <linux/sysctl.h>
 37 #include <linux/cpu.h>
 38 #include <linux/cpuset.h>
 39 #include <linux/memory_hotplug.h>
 40 #include <linux/nodemask.h>
 41 #include <linux/vmalloc.h>
 42 #include <linux/vmstat.h>
 43 #include <linux/mempolicy.h>
 44 #include <linux/memremap.h>
 45 #include <linux/stop_machine.h>
 46 #include <linux/sort.h>
 47 #include <linux/pfn.h>
 48 #include <linux/backing-dev.h>
 49 #include <linux/fault-inject.h>
 50 #include <linux/page-isolation.h>
 51 #include <linux/page_ext.h>
 52 #include <linux/debugobjects.h>
 53 #include <linux/kmemleak.h>
 54 #include <linux/compaction.h>
 55 #include <trace/events/kmem.h>
 56 #include <trace/events/oom.h>
 57 #include <linux/prefetch.h>
 58 #include <linux/mm_inline.h>
 59 #include <linux/migrate.h>
 60 #include <linux/hugetlb.h>
 61 #include <linux/sched/rt.h>
 62 #include <linux/sched/mm.h>
 63 #include <linux/page_owner.h>
 64 #include <linux/kthread.h>
 65 #include <linux/memcontrol.h>
 66 #include <linux/ftrace.h>
 67 #include <linux/lockdep.h>
 68 #include <linux/nmi.h>
 69 #include <linux/psi.h>
 70 
 71 #include <asm/sections.h>
 72 #include <asm/tlbflush.h>
 73 #include <asm/div64.h>
 74 #include "internal.h"
 75 
 76 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
 77 static DEFINE_MUTEX(pcp_batch_high_lock);
 78 #define MIN_PERCPU_PAGELIST_FRACTION    (8)
 79 
 80 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
 81 DEFINE_PER_CPU(int, numa_node);
 82 EXPORT_PER_CPU_SYMBOL(numa_node);
 83 #endif
 84 
 85 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);
 86 
 87 #ifdef CONFIG_HAVE_MEMORYLESS_NODES
 88 /*
 89  * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
 90  * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
 91  * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
 92  * defined in <linux/topology.h>.
 93  */
 94 DEFINE_PER_CPU(int, _numa_mem_);                /* Kernel "local memory" node */
 95 EXPORT_PER_CPU_SYMBOL(_numa_mem_);
 96 int _node_numa_mem_[MAX_NUMNODES];
 97 #endif
 98 
 99 /* work_structs for global per-cpu drains */
100 struct pcpu_drain {
101         struct zone *zone;
102         struct work_struct work;
103 };
104 DEFINE_MUTEX(pcpu_drain_mutex);
105 DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain);
106 
107 #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
108 volatile unsigned long latent_entropy __latent_entropy;
109 EXPORT_SYMBOL(latent_entropy);
110 #endif
111 
112 /*
113  * Array of node states.
114  */
115 nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
116         [N_POSSIBLE] = NODE_MASK_ALL,
117         [N_ONLINE] = { { [0] = 1UL } },
118 #ifndef CONFIG_NUMA
119         [N_NORMAL_MEMORY] = { { [0] = 1UL } },
120 #ifdef CONFIG_HIGHMEM
121         [N_HIGH_MEMORY] = { { [0] = 1UL } },
122 #endif
123         [N_MEMORY] = { { [0] = 1UL } },
124         [N_CPU] = { { [0] = 1UL } },
125 #endif  /* NUMA */
126 };
127 EXPORT_SYMBOL(node_states);
128 
129 atomic_long_t _totalram_pages __read_mostly;
130 EXPORT_SYMBOL(_totalram_pages);
131 unsigned long totalreserve_pages __read_mostly;
132 unsigned long totalcma_pages __read_mostly;
133 
134 int percpu_pagelist_fraction;
135 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
136 
137 /*
138  * A cached value of the page's pageblock's migratetype, used when the page is
139  * put on a pcplist. Used to avoid the pageblock migratetype lookup when
140  * freeing from pcplists in most cases, at the cost of possibly becoming stale.
141  * Also the migratetype set in the page does not necessarily match the pcplist
142  * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
143  * other index - this ensures that it will be put on the correct CMA freelist.
144  */
145 static inline int get_pcppage_migratetype(struct page *page)
146 {
147         return page->index;
148 }
149 
150 static inline void set_pcppage_migratetype(struct page *page, int migratetype)
151 {
152         page->index = migratetype;
153 }
154 
155 #ifdef CONFIG_PM_SLEEP
156 /*
157  * The following functions are used by the suspend/hibernate code to temporarily
158  * change gfp_allowed_mask in order to avoid using I/O during memory allocations
159  * while devices are suspended.  To avoid races with the suspend/hibernate code,
160  * they should always be called with system_transition_mutex held
161  * (gfp_allowed_mask also should only be modified with system_transition_mutex
162  * held, unless the suspend/hibernate code is guaranteed not to run in parallel
163  * with that modification).
164  */
165 
166 static gfp_t saved_gfp_mask;
167 
168 void pm_restore_gfp_mask(void)
169 {
170         WARN_ON(!mutex_is_locked(&system_transition_mutex));
171         if (saved_gfp_mask) {
172                 gfp_allowed_mask = saved_gfp_mask;
173                 saved_gfp_mask = 0;
174         }
175 }
176 
177 void pm_restrict_gfp_mask(void)
178 {
179         WARN_ON(!mutex_is_locked(&system_transition_mutex));
180         WARN_ON(saved_gfp_mask);
181         saved_gfp_mask = gfp_allowed_mask;
182         gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS);
183 }
184 
185 bool pm_suspended_storage(void)
186 {
187         if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
188                 return false;
189         return true;
190 }
191 #endif /* CONFIG_PM_SLEEP */
192 
193 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
194 unsigned int pageblock_order __read_mostly;
195 #endif
196 
197 static void __free_pages_ok(struct page *page, unsigned int order);
198 
199 /*
200  * results with 256, 32 in the lowmem_reserve sysctl:
201  *      1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
202  *      1G machine -> (16M dma, 784M normal, 224M high)
203  *      NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
204  *      HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
205  *      HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA
206  *
207  * TBD: should special case ZONE_DMA32 machines here - in those we normally
208  * don't need any ZONE_NORMAL reservation
209  */
210 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = {
211 #ifdef CONFIG_ZONE_DMA
212         [ZONE_DMA] = 256,
213 #endif
214 #ifdef CONFIG_ZONE_DMA32
215         [ZONE_DMA32] = 256,
216 #endif
217         [ZONE_NORMAL] = 32,
218 #ifdef CONFIG_HIGHMEM
219         [ZONE_HIGHMEM] = 0,
220 #endif
221         [ZONE_MOVABLE] = 0,
222 };
223 
224 EXPORT_SYMBOL(totalram_pages);
225 
226 static char * const zone_names[MAX_NR_ZONES] = {
227 #ifdef CONFIG_ZONE_DMA
228          "DMA",
229 #endif
230 #ifdef CONFIG_ZONE_DMA32
231          "DMA32",
232 #endif
233          "Normal",
234 #ifdef CONFIG_HIGHMEM
235          "HighMem",
236 #endif
237          "Movable",
238 #ifdef CONFIG_ZONE_DEVICE
239          "Device",
240 #endif
241 };
242 
243 const char * const migratetype_names[MIGRATE_TYPES] = {
244         "Unmovable",
245         "Movable",
246         "Reclaimable",
247         "HighAtomic",
248 #ifdef CONFIG_CMA
249         "CMA",
250 #endif
251 #ifdef CONFIG_MEMORY_ISOLATION
252         "Isolate",
253 #endif
254 };
255 
256 compound_page_dtor * const compound_page_dtors[] = {
257         NULL,
258         free_compound_page,
259 #ifdef CONFIG_HUGETLB_PAGE
260         free_huge_page,
261 #endif
262 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
263         free_transhuge_page,
264 #endif
265 };
266 
267 int min_free_kbytes = 1024;
268 int user_min_free_kbytes = -1;
269 int watermark_boost_factor __read_mostly = 15000;
270 int watermark_scale_factor = 10;
271 
272 static unsigned long nr_kernel_pages __initdata;
273 static unsigned long nr_all_pages __initdata;
274 static unsigned long dma_reserve __initdata;
275 
276 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
277 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata;
278 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata;
279 static unsigned long required_kernelcore __initdata;
280 static unsigned long required_kernelcore_percent __initdata;
281 static unsigned long required_movablecore __initdata;
282 static unsigned long required_movablecore_percent __initdata;
283 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata;
284 static bool mirrored_kernelcore __meminitdata;
285 
286 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
287 int movable_zone;
288 EXPORT_SYMBOL(movable_zone);
289 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
290 
291 #if MAX_NUMNODES > 1
292 unsigned int nr_node_ids __read_mostly = MAX_NUMNODES;
293 unsigned int nr_online_nodes __read_mostly = 1;
294 EXPORT_SYMBOL(nr_node_ids);
295 EXPORT_SYMBOL(nr_online_nodes);
296 #endif
297 
298 int page_group_by_mobility_disabled __read_mostly;
299 
300 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
301 /*
302  * During boot we initialize deferred pages on-demand, as needed, but once
303  * page_alloc_init_late() has finished, the deferred pages are all initialized,
304  * and we can permanently disable that path.
305  */
306 static DEFINE_STATIC_KEY_TRUE(deferred_pages);
307 
308 /*
309  * Calling kasan_free_pages() only after deferred memory initialization
310  * has completed. Poisoning pages during deferred memory init will greatly
311  * lengthen the process and cause problem in large memory systems as the
312  * deferred pages initialization is done with interrupt disabled.
313  *
314  * Assuming that there will be no reference to those newly initialized
315  * pages before they are ever allocated, this should have no effect on
316  * KASAN memory tracking as the poison will be properly inserted at page
317  * allocation time. The only corner case is when pages are allocated by
318  * on-demand allocation and then freed again before the deferred pages
319  * initialization is done, but this is not likely to happen.
320  */
321 static inline void kasan_free_nondeferred_pages(struct page *page, int order)
322 {
323         if (!static_branch_unlikely(&deferred_pages))
324                 kasan_free_pages(page, order);
325 }
326 
327 /* Returns true if the struct page for the pfn is uninitialised */
328 static inline bool __meminit early_page_uninitialised(unsigned long pfn)
329 {
330         int nid = early_pfn_to_nid(pfn);
331 
332         if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
333                 return true;
334 
335         return false;
336 }
337 
338 /*
339  * Returns true when the remaining initialisation should be deferred until
340  * later in the boot cycle when it can be parallelised.
341  */
342 static bool __meminit
343 defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
344 {
345         static unsigned long prev_end_pfn, nr_initialised;
346 
347         /*
348          * prev_end_pfn static that contains the end of previous zone
349          * No need to protect because called very early in boot before smp_init.
350          */
351         if (prev_end_pfn != end_pfn) {
352                 prev_end_pfn = end_pfn;
353                 nr_initialised = 0;
354         }
355 
356         /* Always populate low zones for address-constrained allocations */
357         if (end_pfn < pgdat_end_pfn(NODE_DATA(nid)))
358                 return false;
359 
360         /*
361          * We start only with one section of pages, more pages are added as
362          * needed until the rest of deferred pages are initialized.
363          */
364         nr_initialised++;
365         if ((nr_initialised > PAGES_PER_SECTION) &&
366             (pfn & (PAGES_PER_SECTION - 1)) == 0) {
367                 NODE_DATA(nid)->first_deferred_pfn = pfn;
368                 return true;
369         }
370         return false;
371 }
372 #else
373 #define kasan_free_nondeferred_pages(p, o)      kasan_free_pages(p, o)
374 
375 static inline bool early_page_uninitialised(unsigned long pfn)
376 {
377         return false;
378 }
379 
380 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
381 {
382         return false;
383 }
384 #endif
385 
386 /* Return a pointer to the bitmap storing bits affecting a block of pages */
387 static inline unsigned long *get_pageblock_bitmap(struct page *page,
388                                                         unsigned long pfn)
389 {
390 #ifdef CONFIG_SPARSEMEM
391         return __pfn_to_section(pfn)->pageblock_flags;
392 #else
393         return page_zone(page)->pageblock_flags;
394 #endif /* CONFIG_SPARSEMEM */
395 }
396 
397 static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)
398 {
399 #ifdef CONFIG_SPARSEMEM
400         pfn &= (PAGES_PER_SECTION-1);
401         return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
402 #else
403         pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
404         return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
405 #endif /* CONFIG_SPARSEMEM */
406 }
407 
408 /**
409  * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
410  * @page: The page within the block of interest
411  * @pfn: The target page frame number
412  * @end_bitidx: The last bit of interest to retrieve
413  * @mask: mask of bits that the caller is interested in
414  *
415  * Return: pageblock_bits flags
416  */
417 static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page,
418                                         unsigned long pfn,
419                                         unsigned long end_bitidx,
420                                         unsigned long mask)
421 {
422         unsigned long *bitmap;
423         unsigned long bitidx, word_bitidx;
424         unsigned long word;
425 
426         bitmap = get_pageblock_bitmap(page, pfn);
427         bitidx = pfn_to_bitidx(page, pfn);
428         word_bitidx = bitidx / BITS_PER_LONG;
429         bitidx &= (BITS_PER_LONG-1);
430 
431         word = bitmap[word_bitidx];
432         bitidx += end_bitidx;
433         return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;
434 }
435 
436 unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
437                                         unsigned long end_bitidx,
438                                         unsigned long mask)
439 {
440         return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask);
441 }
442 
443 static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
444 {
445         return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK);
446 }
447 
448 /**
449  * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
450  * @page: The page within the block of interest
451  * @flags: The flags to set
452  * @pfn: The target page frame number
453  * @end_bitidx: The last bit of interest
454  * @mask: mask of bits that the caller is interested in
455  */
456 void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
457                                         unsigned long pfn,
458                                         unsigned long end_bitidx,
459                                         unsigned long mask)
460 {
461         unsigned long *bitmap;
462         unsigned long bitidx, word_bitidx;
463         unsigned long old_word, word;
464 
465         BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
466         BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits));
467 
468         bitmap = get_pageblock_bitmap(page, pfn);
469         bitidx = pfn_to_bitidx(page, pfn);
470         word_bitidx = bitidx / BITS_PER_LONG;
471         bitidx &= (BITS_PER_LONG-1);
472 
473         VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
474 
475         bitidx += end_bitidx;
476         mask <<= (BITS_PER_LONG - bitidx - 1);
477         flags <<= (BITS_PER_LONG - bitidx - 1);
478 
479         word = READ_ONCE(bitmap[word_bitidx]);
480         for (;;) {
481                 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
482                 if (word == old_word)
483                         break;
484                 word = old_word;
485         }
486 }
487 
488 void set_pageblock_migratetype(struct page *page, int migratetype)
489 {
490         if (unlikely(page_group_by_mobility_disabled &&
491                      migratetype < MIGRATE_PCPTYPES))
492                 migratetype = MIGRATE_UNMOVABLE;
493 
494         set_pageblock_flags_group(page, (unsigned long)migratetype,
495                                         PB_migrate, PB_migrate_end);
496 }
497 
498 #ifdef CONFIG_DEBUG_VM
499 static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
500 {
501         int ret = 0;
502         unsigned seq;
503         unsigned long pfn = page_to_pfn(page);
504         unsigned long sp, start_pfn;
505 
506         do {
507                 seq = zone_span_seqbegin(zone);
508                 start_pfn = zone->zone_start_pfn;
509                 sp = zone->spanned_pages;
510                 if (!zone_spans_pfn(zone, pfn))
511                         ret = 1;
512         } while (zone_span_seqretry(zone, seq));
513 
514         if (ret)
515                 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
516                         pfn, zone_to_nid(zone), zone->name,
517                         start_pfn, start_pfn + sp);
518 
519         return ret;
520 }
521 
522 static int page_is_consistent(struct zone *zone, struct page *page)
523 {
524         if (!pfn_valid_within(page_to_pfn(page)))
525                 return 0;
526         if (zone != page_zone(page))
527                 return 0;
528 
529         return 1;
530 }
531 /*
532  * Temporary debugging check for pages not lying within a given zone.
533  */
534 static int __maybe_unused bad_range(struct zone *zone, struct page *page)
535 {
536         if (page_outside_zone_boundaries(zone, page))
537                 return 1;
538         if (!page_is_consistent(zone, page))
539                 return 1;
540 
541         return 0;
542 }
543 #else
544 static inline int __maybe_unused bad_range(struct zone *zone, struct page *page)
545 {
546         return 0;
547 }
548 #endif
549 
550 static void bad_page(struct page *page, const char *reason,
551                 unsigned long bad_flags)
552 {
553         static unsigned long resume;
554         static unsigned long nr_shown;
555         static unsigned long nr_unshown;
556 
557         /*
558          * Allow a burst of 60 reports, then keep quiet for that minute;
559          * or allow a steady drip of one report per second.
560          */
561         if (nr_shown == 60) {
562                 if (time_before(jiffies, resume)) {
563                         nr_unshown++;
564                         goto out;
565                 }
566                 if (nr_unshown) {
567                         pr_alert(
568                               "BUG: Bad page state: %lu messages suppressed\n",
569                                 nr_unshown);
570                         nr_unshown = 0;
571                 }
572                 nr_shown = 0;
573         }
574         if (nr_shown++ == 0)
575                 resume = jiffies + 60 * HZ;
576 
577         pr_alert("BUG: Bad page state in process %s  pfn:%05lx\n",
578                 current->comm, page_to_pfn(page));
579         __dump_page(page, reason);
580         bad_flags &= page->flags;
581         if (bad_flags)
582                 pr_alert("bad because of flags: %#lx(%pGp)\n",
583                                                 bad_flags, &bad_flags);
584         dump_page_owner(page);
585 
586         print_modules();
587         dump_stack();
588 out:
589         /* Leave bad fields for debug, except PageBuddy could make trouble */
590         page_mapcount_reset(page); /* remove PageBuddy */
591         add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
592 }
593 
594 /*
595  * Higher-order pages are called "compound pages".  They are structured thusly:
596  *
597  * The first PAGE_SIZE page is called the "head page" and have PG_head set.
598  *
599  * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded
600  * in bit 0 of page->compound_head. The rest of bits is pointer to head page.
601  *
602  * The first tail page's ->compound_dtor holds the offset in array of compound
603  * page destructors. See compound_page_dtors.
604  *
605  * The first tail page's ->compound_order holds the order of allocation.
606  * This usage means that zero-order pages may not be compound.
607  */
608 
609 void free_compound_page(struct page *page)
610 {
611         __free_pages_ok(page, compound_order(page));
612 }
613 
614 void prep_compound_page(struct page *page, unsigned int order)
615 {
616         int i;
617         int nr_pages = 1 << order;
618 
619         set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
620         set_compound_order(page, order);
621         __SetPageHead(page);
622         for (i = 1; i < nr_pages; i++) {
623                 struct page *p = page + i;
624                 set_page_count(p, 0);
625                 p->mapping = TAIL_MAPPING;
626                 set_compound_head(p, page);
627         }
628         atomic_set(compound_mapcount_ptr(page), -1);
629 }
630 
631 #ifdef CONFIG_DEBUG_PAGEALLOC
632 unsigned int _debug_guardpage_minorder;
633 bool _debug_pagealloc_enabled __read_mostly
634                         = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
635 EXPORT_SYMBOL(_debug_pagealloc_enabled);
636 bool _debug_guardpage_enabled __read_mostly;
637 
638 static int __init early_debug_pagealloc(char *buf)
639 {
640         if (!buf)
641                 return -EINVAL;
642         return kstrtobool(buf, &_debug_pagealloc_enabled);
643 }
644 early_param("debug_pagealloc", early_debug_pagealloc);
645 
646 static bool need_debug_guardpage(void)
647 {
648         /* If we don't use debug_pagealloc, we don't need guard page */
649         if (!debug_pagealloc_enabled())
650                 return false;
651 
652         if (!debug_guardpage_minorder())
653                 return false;
654 
655         return true;
656 }
657 
658 static void init_debug_guardpage(void)
659 {
660         if (!debug_pagealloc_enabled())
661                 return;
662 
663         if (!debug_guardpage_minorder())
664                 return;
665 
666         _debug_guardpage_enabled = true;
667 }
668 
669 struct page_ext_operations debug_guardpage_ops = {
670         .need = need_debug_guardpage,
671         .init = init_debug_guardpage,
672 };
673 
674 static int __init debug_guardpage_minorder_setup(char *buf)
675 {
676         unsigned long res;
677 
678         if (kstrtoul(buf, 10, &res) < 0 ||  res > MAX_ORDER / 2) {
679                 pr_err("Bad debug_guardpage_minorder value\n");
680                 return 0;
681         }
682         _debug_guardpage_minorder = res;
683         pr_info("Setting debug_guardpage_minorder to %lu\n", res);
684         return 0;
685 }
686 early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup);
687 
688 static inline bool set_page_guard(struct zone *zone, struct page *page,
689                                 unsigned int order, int migratetype)
690 {
691         struct page_ext *page_ext;
692 
693         if (!debug_guardpage_enabled())
694                 return false;
695 
696         if (order >= debug_guardpage_minorder())
697                 return false;
698 
699         page_ext = lookup_page_ext(page);
700         if (unlikely(!page_ext))
701                 return false;
702 
703         __set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
704 
705         INIT_LIST_HEAD(&page->lru);
706         set_page_private(page, order);
707         /* Guard pages are not available for any usage */
708         __mod_zone_freepage_state(zone, -(1 << order), migratetype);
709 
710         return true;
711 }
712 
713 static inline void clear_page_guard(struct zone *zone, struct page *page,
714                                 unsigned int order, int migratetype)
715 {
716         struct page_ext *page_ext;
717 
718         if (!debug_guardpage_enabled())
719                 return;
720 
721         page_ext = lookup_page_ext(page);
722         if (unlikely(!page_ext))
723                 return;
724 
725         __clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
726 
727         set_page_private(page, 0);
728         if (!is_migrate_isolate(migratetype))
729                 __mod_zone_freepage_state(zone, (1 << order), migratetype);
730 }
731 #else
732 struct page_ext_operations debug_guardpage_ops;
733 static inline bool set_page_guard(struct zone *zone, struct page *page,
734                         unsigned int order, int migratetype) { return false; }
735 static inline void clear_page_guard(struct zone *zone, struct page *page,
736                                 unsigned int order, int migratetype) {}
737 #endif
738 
739 static inline void set_page_order(struct page *page, unsigned int order)
740 {
741         set_page_private(page, order);
742         __SetPageBuddy(page);
743 }
744 
745 static inline void rmv_page_order(struct page *page)
746 {
747         __ClearPageBuddy(page);
748         set_page_private(page, 0);
749 }
750 
751 /*
752  * This function checks whether a page is free && is the buddy
753  * we can coalesce a page and its buddy if
754  * (a) the buddy is not in a hole (check before calling!) &&
755  * (b) the buddy is in the buddy system &&
756  * (c) a page and its buddy have the same order &&
757  * (d) a page and its buddy are in the same zone.
758  *
759  * For recording whether a page is in the buddy system, we set PageBuddy.
760  * Setting, clearing, and testing PageBuddy is serialized by zone->lock.
761  *
762  * For recording page's order, we use page_private(page).
763  */
764 static inline int page_is_buddy(struct page *page, struct page *buddy,
765                                                         unsigned int order)
766 {
767         if (page_is_guard(buddy) && page_order(buddy) == order) {
768                 if (page_zone_id(page) != page_zone_id(buddy))
769                         return 0;
770 
771                 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
772 
773                 return 1;
774         }
775 
776         if (PageBuddy(buddy) && page_order(buddy) == order) {
777                 /*
778                  * zone check is done late to avoid uselessly
779                  * calculating zone/node ids for pages that could
780                  * never merge.
781                  */
782                 if (page_zone_id(page) != page_zone_id(buddy))
783                         return 0;
784 
785                 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
786 
787                 return 1;
788         }
789         return 0;
790 }
791 
792 #ifdef CONFIG_COMPACTION
793 static inline struct capture_control *task_capc(struct zone *zone)
794 {
795         struct capture_control *capc = current->capture_control;
796 
797         return capc &&
798                 !(current->flags & PF_KTHREAD) &&
799                 !capc->page &&
800                 capc->cc->zone == zone &&
801                 capc->cc->direct_compaction ? capc : NULL;
802 }
803 
804 static inline bool
805 compaction_capture(struct capture_control *capc, struct page *page,
806                    int order, int migratetype)
807 {
808         if (!capc || order != capc->cc->order)
809                 return false;
810 
811         /* Do not accidentally pollute CMA or isolated regions*/
812         if (is_migrate_cma(migratetype) ||
813             is_migrate_isolate(migratetype))
814                 return false;
815 
816         /*
817          * Do not let lower order allocations polluate a movable pageblock.
818          * This might let an unmovable request use a reclaimable pageblock
819          * and vice-versa but no more than normal fallback logic which can
820          * have trouble finding a high-order free page.
821          */
822         if (order < pageblock_order && migratetype == MIGRATE_MOVABLE)
823                 return false;
824 
825         capc->page = page;
826         return true;
827 }
828 
829 #else
830 static inline struct capture_control *task_capc(struct zone *zone)
831 {
832         return NULL;
833 }
834 
835 static inline bool
836 compaction_capture(struct capture_control *capc, struct page *page,
837                    int order, int migratetype)
838 {
839         return false;
840 }
841 #endif /* CONFIG_COMPACTION */
842 
843 /*
844  * Freeing function for a buddy system allocator.
845  *
846  * The concept of a buddy system is to maintain direct-mapped table
847  * (containing bit values) for memory blocks of various "orders".
848  * The bottom level table contains the map for the smallest allocatable
849  * units of memory (here, pages), and each level above it describes
850  * pairs of units from the levels below, hence, "buddies".
851  * At a high level, all that happens here is marking the table entry
852  * at the bottom level available, and propagating the changes upward
853  * as necessary, plus some accounting needed to play nicely with other
854  * parts of the VM system.
855  * At each level, we keep a list of pages, which are heads of continuous
856  * free pages of length of (1 << order) and marked with PageBuddy.
857  * Page's order is recorded in page_private(page) field.
858  * So when we are allocating or freeing one, we can derive the state of the
859  * other.  That is, if we allocate a small block, and both were
860  * free, the remainder of the region must be split into blocks.
861  * If a block is freed, and its buddy is also free, then this
862  * triggers coalescing into a block of larger size.
863  *
864  * -- nyc
865  */
866 
867 static inline void __free_one_page(struct page *page,
868                 unsigned long pfn,
869                 struct zone *zone, unsigned int order,
870                 int migratetype)
871 {
872         unsigned long combined_pfn;
873         unsigned long uninitialized_var(buddy_pfn);
874         struct page *buddy;
875         unsigned int max_order;
876         struct capture_control *capc = task_capc(zone);
877 
878         max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1);
879 
880         VM_BUG_ON(!zone_is_initialized(zone));
881         VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
882 
883         VM_BUG_ON(migratetype == -1);
884         if (likely(!is_migrate_isolate(migratetype)))
885                 __mod_zone_freepage_state(zone, 1 << order, migratetype);
886 
887         VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
888         VM_BUG_ON_PAGE(bad_range(zone, page), page);
889 
890 continue_merging:
891         while (order < max_order - 1) {
892                 if (compaction_capture(capc, page, order, migratetype)) {
893                         __mod_zone_freepage_state(zone, -(1 << order),
894                                                                 migratetype);
895                         return;
896                 }
897                 buddy_pfn = __find_buddy_pfn(pfn, order);
898                 buddy = page + (buddy_pfn - pfn);
899 
900                 if (!pfn_valid_within(buddy_pfn))
901                         goto done_merging;
902                 if (!page_is_buddy(page, buddy, order))
903                         goto done_merging;
904                 /*
905                  * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
906                  * merge with it and move up one order.
907                  */
908                 if (page_is_guard(buddy)) {
909                         clear_page_guard(zone, buddy, order, migratetype);
910                 } else {
911                         list_del(&buddy->lru);
912                         zone->free_area[order].nr_free--;
913                         rmv_page_order(buddy);
914                 }
915                 combined_pfn = buddy_pfn & pfn;
916                 page = page + (combined_pfn - pfn);
917                 pfn = combined_pfn;
918                 order++;
919         }
920         if (max_order < MAX_ORDER) {
921                 /* If we are here, it means order is >= pageblock_order.
922                  * We want to prevent merge between freepages on isolate
923                  * pageblock and normal pageblock. Without this, pageblock
924                  * isolation could cause incorrect freepage or CMA accounting.
925                  *
926                  * We don't want to hit this code for the more frequent
927                  * low-order merging.
928                  */
929                 if (unlikely(has_isolate_pageblock(zone))) {
930                         int buddy_mt;
931 
932                         buddy_pfn = __find_buddy_pfn(pfn, order);
933                         buddy = page + (buddy_pfn - pfn);
934                         buddy_mt = get_pageblock_migratetype(buddy);
935 
936                         if (migratetype != buddy_mt
937                                         && (is_migrate_isolate(migratetype) ||
938                                                 is_migrate_isolate(buddy_mt)))
939                                 goto done_merging;
940                 }
941                 max_order++;
942                 goto continue_merging;
943         }
944 
945 done_merging:
946         set_page_order(page, order);
947 
948         /*
949          * If this is not the largest possible page, check if the buddy
950          * of the next-highest order is free. If it is, it's possible
951          * that pages are being freed that will coalesce soon. In case,
952          * that is happening, add the free page to the tail of the list
953          * so it's less likely to be used soon and more likely to be merged
954          * as a higher order page
955          */
956         if ((order < MAX_ORDER-2) && pfn_valid_within(buddy_pfn)) {
957                 struct page *higher_page, *higher_buddy;
958                 combined_pfn = buddy_pfn & pfn;
959                 higher_page = page + (combined_pfn - pfn);
960                 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1);
961                 higher_buddy = higher_page + (buddy_pfn - combined_pfn);
962                 if (pfn_valid_within(buddy_pfn) &&
963                     page_is_buddy(higher_page, higher_buddy, order + 1)) {
964                         list_add_tail(&page->lru,
965                                 &zone->free_area[order].free_list[migratetype]);
966                         goto out;
967                 }
968         }
969 
970         list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
971 out:
972         zone->free_area[order].nr_free++;
973 }
974 
975 /*
976  * A bad page could be due to a number of fields. Instead of multiple branches,
977  * try and check multiple fields with one check. The caller must do a detailed
978  * check if necessary.
979  */
980 static inline bool page_expected_state(struct page *page,
981                                         unsigned long check_flags)
982 {
983         if (unlikely(atomic_read(&page->_mapcount) != -1))
984                 return false;
985 
986         if (unlikely((unsigned long)page->mapping |
987                         page_ref_count(page) |
988 #ifdef CONFIG_MEMCG
989                         (unsigned long)page->mem_cgroup |
990 #endif
991                         (page->flags & check_flags)))
992                 return false;
993 
994         return true;
995 }
996 
997 static void free_pages_check_bad(struct page *page)
998 {
999         const char *bad_reason;
1000         unsigned long bad_flags;
1001 
1002         bad_reason = NULL;
1003         bad_flags = 0;
1004 
1005         if (unlikely(atomic_read(&page->_mapcount) != -1))
1006                 bad_reason = "nonzero mapcount";
1007         if (unlikely(page->mapping != NULL))
1008                 bad_reason = "non-NULL mapping";
1009         if (unlikely(page_ref_count(page) != 0))
1010                 bad_reason = "nonzero _refcount";
1011         if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
1012                 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
1013                 bad_flags = PAGE_FLAGS_CHECK_AT_FREE;
1014         }
1015 #ifdef CONFIG_MEMCG
1016         if (unlikely(page->mem_cgroup))
1017                 bad_reason = "page still charged to cgroup";
1018 #endif
1019         bad_page(page, bad_reason, bad_flags);
1020 }
1021 
1022 static inline int free_pages_check(struct page *page)
1023 {
1024         if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
1025                 return 0;
1026 
1027         /* Something has gone sideways, find it */
1028         free_pages_check_bad(page);
1029         return 1;
1030 }
1031 
1032 static int free_tail_pages_check(struct page *head_page, struct page *page)
1033 {
1034         int ret = 1;
1035 
1036         /*
1037          * We rely page->lru.next never has bit 0 set, unless the page
1038          * is PageTail(). Let's make sure that's true even for poisoned ->lru.
1039          */
1040         BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1);
1041 
1042         if (!IS_ENABLED(CONFIG_DEBUG_VM)) {
1043                 ret = 0;
1044                 goto out;
1045         }
1046         switch (page - head_page) {
1047         case 1:
1048                 /* the first tail page: ->mapping may be compound_mapcount() */
1049                 if (unlikely(compound_mapcount(page))) {
1050                         bad_page(page, "nonzero compound_mapcount", 0);
1051                         goto out;
1052                 }
1053                 break;
1054         case 2:
1055                 /*
1056                  * the second tail page: ->mapping is
1057                  * deferred_list.next -- ignore value.
1058                  */
1059                 break;
1060         default:
1061                 if (page->mapping != TAIL_MAPPING) {
1062                         bad_page(page, "corrupted mapping in tail page", 0);
1063                         goto out;
1064                 }
1065                 break;
1066         }
1067         if (unlikely(!PageTail(page))) {
1068                 bad_page(page, "PageTail not set", 0);
1069                 goto out;
1070         }
1071         if (unlikely(compound_head(page) != head_page)) {
1072                 bad_page(page, "compound_head not consistent", 0);
1073                 goto out;
1074         }
1075         ret = 0;
1076 out:
1077         page->mapping = NULL;
1078         clear_compound_head(page);
1079         return ret;
1080 }
1081 
1082 static __always_inline bool free_pages_prepare(struct page *page,
1083                                         unsigned int order, bool check_free)
1084 {
1085         int bad = 0;
1086 
1087         VM_BUG_ON_PAGE(PageTail(page), page);
1088 
1089         trace_mm_page_free(page, order);
1090 
1091         /*
1092          * Check tail pages before head page information is cleared to
1093          * avoid checking PageCompound for order-0 pages.
1094          */
1095         if (unlikely(order)) {
1096                 bool compound = PageCompound(page);
1097                 int i;
1098 
1099                 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
1100 
1101                 if (compound)
1102                         ClearPageDoubleMap(page);
1103                 for (i = 1; i < (1 << order); i++) {
1104                         if (compound)
1105                                 bad += free_tail_pages_check(page, page + i);
1106                         if (unlikely(free_pages_check(page + i))) {
1107                                 bad++;
1108                                 continue;
1109                         }
1110                         (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1111                 }
1112         }
1113         if (PageMappingFlags(page))
1114                 page->mapping = NULL;
1115         if (memcg_kmem_enabled() && PageKmemcg(page))
1116                 __memcg_kmem_uncharge(page, order);
1117         if (check_free)
1118                 bad += free_pages_check(page);
1119         if (bad)
1120                 return false;
1121 
1122         page_cpupid_reset_last(page);
1123         page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1124         reset_page_owner(page, order);
1125 
1126         if (!PageHighMem(page)) {
1127                 debug_check_no_locks_freed(page_address(page),
1128                                            PAGE_SIZE << order);
1129                 debug_check_no_obj_freed(page_address(page),
1130                                            PAGE_SIZE << order);
1131         }
1132         arch_free_page(page, order);
1133         kernel_poison_pages(page, 1 << order, 0);
1134         kernel_map_pages(page, 1 << order, 0);
1135         kasan_free_nondeferred_pages(page, order);
1136 
1137         return true;
1138 }
1139 
1140 #ifdef CONFIG_DEBUG_VM
1141 static inline bool free_pcp_prepare(struct page *page)
1142 {
1143         return free_pages_prepare(page, 0, true);
1144 }
1145 
1146 static inline bool bulkfree_pcp_prepare(struct page *page)
1147 {
1148         return false;
1149 }
1150 #else
1151 static bool free_pcp_prepare(struct page *page)
1152 {
1153         return free_pages_prepare(page, 0, false);
1154 }
1155 
1156 static bool bulkfree_pcp_prepare(struct page *page)
1157 {
1158         return free_pages_check(page);
1159 }
1160 #endif /* CONFIG_DEBUG_VM */
1161 
1162 static inline void prefetch_buddy(struct page *page)
1163 {
1164         unsigned long pfn = page_to_pfn(page);
1165         unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0);
1166         struct page *buddy = page + (buddy_pfn - pfn);
1167 
1168         prefetch(buddy);
1169 }
1170 
1171 /*
1172  * Frees a number of pages from the PCP lists
1173  * Assumes all pages on list are in same zone, and of same order.
1174  * count is the number of pages to free.
1175  *
1176  * If the zone was previously in an "all pages pinned" state then look to
1177  * see if this freeing clears that state.
1178  *
1179  * And clear the zone's pages_scanned counter, to hold off the "all pages are
1180  * pinned" detection logic.
1181  */
1182 static void free_pcppages_bulk(struct zone *zone, int count,
1183                                         struct per_cpu_pages *pcp)
1184 {
1185         int migratetype = 0;
1186         int batch_free = 0;
1187         int prefetch_nr = 0;
1188         bool isolated_pageblocks;
1189         struct page *page, *tmp;
1190         LIST_HEAD(head);
1191 
1192         while (count) {
1193                 struct list_head *list;
1194 
1195                 /*
1196                  * Remove pages from lists in a round-robin fashion. A
1197                  * batch_free count is maintained that is incremented when an
1198                  * empty list is encountered.  This is so more pages are freed
1199                  * off fuller lists instead of spinning excessively around empty
1200                  * lists
1201                  */
1202                 do {
1203                         batch_free++;
1204                         if (++migratetype == MIGRATE_PCPTYPES)
1205                                 migratetype = 0;
1206                         list = &pcp->lists[migratetype];
1207                 } while (list_empty(list));
1208 
1209                 /* This is the only non-empty list. Free them all. */
1210                 if (batch_free == MIGRATE_PCPTYPES)
1211                         batch_free = count;
1212 
1213                 do {
1214                         page = list_last_entry(list, struct page, lru);
1215                         /* must delete to avoid corrupting pcp list */
1216                         list_del(&page->lru);
1217                         pcp->count--;
1218 
1219                         if (bulkfree_pcp_prepare(page))
1220                                 continue;
1221 
1222                         list_add_tail(&page->lru, &head);
1223 
1224                         /*
1225                          * We are going to put the page back to the global
1226                          * pool, prefetch its buddy to speed up later access
1227                          * under zone->lock. It is believed the overhead of
1228                          * an additional test and calculating buddy_pfn here
1229                          * can be offset by reduced memory latency later. To
1230                          * avoid excessive prefetching due to large count, only
1231                          * prefetch buddy for the first pcp->batch nr of pages.
1232                          */
1233                         if (prefetch_nr++ < pcp->batch)
1234                                 prefetch_buddy(page);
1235                 } while (--count && --batch_free && !list_empty(list));
1236         }
1237 
1238         spin_lock(&zone->lock);
1239         isolated_pageblocks = has_isolate_pageblock(zone);
1240 
1241         /*
1242          * Use safe version since after __free_one_page(),
1243          * page->lru.next will not point to original list.
1244          */
1245         list_for_each_entry_safe(page, tmp, &head, lru) {
1246                 int mt = get_pcppage_migratetype(page);
1247                 /* MIGRATE_ISOLATE page should not go to pcplists */
1248                 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
1249                 /* Pageblock could have been isolated meanwhile */
1250                 if (unlikely(isolated_pageblocks))
1251                         mt = get_pageblock_migratetype(page);
1252 
1253                 __free_one_page(page, page_to_pfn(page), zone, 0, mt);
1254                 trace_mm_page_pcpu_drain(page, 0, mt);
1255         }
1256         spin_unlock(&zone->lock);
1257 }
1258 
1259 static void free_one_page(struct zone *zone,
1260                                 struct page *page, unsigned long pfn,
1261                                 unsigned int order,
1262                                 int migratetype)
1263 {
1264         spin_lock(&zone->lock);
1265         if (unlikely(has_isolate_pageblock(zone) ||
1266                 is_migrate_isolate(migratetype))) {
1267                 migratetype = get_pfnblock_migratetype(page, pfn);
1268         }
1269         __free_one_page(page, pfn, zone, order, migratetype);
1270         spin_unlock(&zone->lock);
1271 }
1272 
1273 static void __meminit __init_single_page(struct page *page, unsigned long pfn,
1274                                 unsigned long zone, int nid)
1275 {
1276         mm_zero_struct_page(page);
1277         set_page_links(page, zone, nid, pfn);
1278         init_page_count(page);
1279         page_mapcount_reset(page);
1280         page_cpupid_reset_last(page);
1281         page_kasan_tag_reset(page);
1282 
1283         INIT_LIST_HEAD(&page->lru);
1284 #ifdef WANT_PAGE_VIRTUAL
1285         /* The shift won't overflow because ZONE_NORMAL is below 4G. */
1286         if (!is_highmem_idx(zone))
1287                 set_page_address(page, __va(pfn << PAGE_SHIFT));
1288 #endif
1289 }
1290 
1291 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1292 static void __meminit init_reserved_page(unsigned long pfn)
1293 {
1294         pg_data_t *pgdat;
1295         int nid, zid;
1296 
1297         if (!early_page_uninitialised(pfn))
1298                 return;
1299 
1300         nid = early_pfn_to_nid(pfn);
1301         pgdat = NODE_DATA(nid);
1302 
1303         for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1304                 struct zone *zone = &pgdat->node_zones[zid];
1305 
1306                 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone))
1307                         break;
1308         }
1309         __init_single_page(pfn_to_page(pfn), pfn, zid, nid);
1310 }
1311 #else
1312 static inline void init_reserved_page(unsigned long pfn)
1313 {
1314 }
1315 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1316 
1317 /*
1318  * Initialised pages do not have PageReserved set. This function is
1319  * called for each range allocated by the bootmem allocator and
1320  * marks the pages PageReserved. The remaining valid pages are later
1321  * sent to the buddy page allocator.
1322  */
1323 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
1324 {
1325         unsigned long start_pfn = PFN_DOWN(start);
1326         unsigned long end_pfn = PFN_UP(end);
1327 
1328         for (; start_pfn < end_pfn; start_pfn++) {
1329                 if (pfn_valid(start_pfn)) {
1330                         struct page *page = pfn_to_page(start_pfn);
1331 
1332                         init_reserved_page(start_pfn);
1333 
1334                         /* Avoid false-positive PageTail() */
1335                         INIT_LIST_HEAD(&page->lru);
1336 
1337                         /*
1338                          * no need for atomic set_bit because the struct
1339                          * page is not visible yet so nobody should
1340                          * access it yet.
1341                          */
1342                         __SetPageReserved(page);
1343                 }
1344         }
1345 }
1346 
1347 static void __free_pages_ok(struct page *page, unsigned int order)
1348 {
1349         unsigned long flags;
1350         int migratetype;
1351         unsigned long pfn = page_to_pfn(page);
1352 
1353         if (!free_pages_prepare(page, order, true))
1354                 return;
1355 
1356         migratetype = get_pfnblock_migratetype(page, pfn);
1357         local_irq_save(flags);
1358         __count_vm_events(PGFREE, 1 << order);
1359         free_one_page(page_zone(page), page, pfn, order, migratetype);
1360         local_irq_restore(flags);
1361 }
1362 
1363 void __free_pages_core(struct page *page, unsigned int order)
1364 {
1365         unsigned int nr_pages = 1 << order;
1366         struct page *p = page;
1367         unsigned int loop;
1368 
1369         prefetchw(p);
1370         for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
1371                 prefetchw(p + 1);
1372                 __ClearPageReserved(p);
1373                 set_page_count(p, 0);
1374         }
1375         __ClearPageReserved(p);
1376         set_page_count(p, 0);
1377 
1378         atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
1379         set_page_refcounted(page);
1380         __free_pages(page, order);
1381 }
1382 
1383 #if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \
1384         defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP)
1385 
1386 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;
1387 
1388 int __meminit early_pfn_to_nid(unsigned long pfn)
1389 {
1390         static DEFINE_SPINLOCK(early_pfn_lock);
1391         int nid;
1392 
1393         spin_lock(&early_pfn_lock);
1394         nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
1395         if (nid < 0)
1396                 nid = first_online_node;
1397         spin_unlock(&early_pfn_lock);
1398 
1399         return nid;
1400 }
1401 #endif
1402 
1403 #ifdef CONFIG_NODES_SPAN_OTHER_NODES
1404 static inline bool __meminit __maybe_unused
1405 meminit_pfn_in_nid(unsigned long pfn, int node,
1406                    struct mminit_pfnnid_cache *state)
1407 {
1408         int nid;
1409 
1410         nid = __early_pfn_to_nid(pfn, state);
1411         if (nid >= 0 && nid != node)
1412                 return false;
1413         return true;
1414 }
1415 
1416 /* Only safe to use early in boot when initialisation is single-threaded */
1417 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
1418 {
1419         return meminit_pfn_in_nid(pfn, node, &early_pfnnid_cache);
1420 }
1421 
1422 #else
1423 
1424 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
1425 {
1426         return true;
1427 }
1428 static inline bool __meminit  __maybe_unused
1429 meminit_pfn_in_nid(unsigned long pfn, int node,
1430                    struct mminit_pfnnid_cache *state)
1431 {
1432         return true;
1433 }
1434 #endif
1435 
1436 
1437 void __init memblock_free_pages(struct page *page, unsigned long pfn,
1438                                                         unsigned int order)
1439 {
1440         if (early_page_uninitialised(pfn))
1441                 return;
1442         __free_pages_core(page, order);
1443 }
1444 
1445 /*
1446  * Check that the whole (or subset of) a pageblock given by the interval of
1447  * [start_pfn, end_pfn) is valid and within the same zone, before scanning it
1448  * with the migration of free compaction scanner. The scanners then need to
1449  * use only pfn_valid_within() check for arches that allow holes within
1450  * pageblocks.
1451  *
1452  * Return struct page pointer of start_pfn, or NULL if checks were not passed.
1453  *
1454  * It's possible on some configurations to have a setup like node0 node1 node0
1455  * i.e. it's possible that all pages within a zones range of pages do not
1456  * belong to a single zone. We assume that a border between node0 and node1
1457  * can occur within a single pageblock, but not a node0 node1 node0
1458  * interleaving within a single pageblock. It is therefore sufficient to check
1459  * the first and last page of a pageblock and avoid checking each individual
1460  * page in a pageblock.
1461  */
1462 struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
1463                                      unsigned long end_pfn, struct zone *zone)
1464 {
1465         struct page *start_page;
1466         struct page *end_page;
1467 
1468         /* end_pfn is one past the range we are checking */
1469         end_pfn--;
1470 
1471         if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn))
1472                 return NULL;
1473 
1474         start_page = pfn_to_online_page(start_pfn);
1475         if (!start_page)
1476                 return NULL;
1477 
1478         if (page_zone(start_page) != zone)
1479                 return NULL;
1480 
1481         end_page = pfn_to_page(end_pfn);
1482 
1483         /* This gives a shorter code than deriving page_zone(end_page) */
1484         if (page_zone_id(start_page) != page_zone_id(end_page))
1485                 return NULL;
1486 
1487         return start_page;
1488 }
1489 
1490 void set_zone_contiguous(struct zone *zone)
1491 {
1492         unsigned long block_start_pfn = zone->zone_start_pfn;
1493         unsigned long block_end_pfn;
1494 
1495         block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages);
1496         for (; block_start_pfn < zone_end_pfn(zone);
1497                         block_start_pfn = block_end_pfn,
1498                          block_end_pfn += pageblock_nr_pages) {
1499 
1500                 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
1501 
1502                 if (!__pageblock_pfn_to_page(block_start_pfn,
1503                                              block_end_pfn, zone))
1504                         return;
1505         }
1506 
1507         /* We confirm that there is no hole */
1508         zone->contiguous = true;
1509 }
1510 
1511 void clear_zone_contiguous(struct zone *zone)
1512 {
1513         zone->contiguous = false;
1514 }
1515 
1516 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1517 static void __init deferred_free_range(unsigned long pfn,
1518                                        unsigned long nr_pages)
1519 {
1520         struct page *page;
1521         unsigned long i;
1522 
1523         if (!nr_pages)
1524                 return;
1525 
1526         page = pfn_to_page(pfn);
1527 
1528         /* Free a large naturally-aligned chunk if possible */
1529         if (nr_pages == pageblock_nr_pages &&
1530             (pfn & (pageblock_nr_pages - 1)) == 0) {
1531                 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1532                 __free_pages_core(page, pageblock_order);
1533                 return;
1534         }
1535 
1536         for (i = 0; i < nr_pages; i++, page++, pfn++) {
1537                 if ((pfn & (pageblock_nr_pages - 1)) == 0)
1538                         set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1539                 __free_pages_core(page, 0);
1540         }
1541 }
1542 
1543 /* Completion tracking for deferred_init_memmap() threads */
1544 static atomic_t pgdat_init_n_undone __initdata;
1545 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
1546 
1547 static inline void __init pgdat_init_report_one_done(void)
1548 {
1549         if (atomic_dec_and_test(&pgdat_init_n_undone))
1550                 complete(&pgdat_init_all_done_comp);
1551 }
1552 
1553 /*
1554  * Returns true if page needs to be initialized or freed to buddy allocator.
1555  *
1556  * First we check if pfn is valid on architectures where it is possible to have
1557  * holes within pageblock_nr_pages. On systems where it is not possible, this
1558  * function is optimized out.
1559  *
1560  * Then, we check if a current large page is valid by only checking the validity
1561  * of the head pfn.
1562  *
1563  * Finally, meminit_pfn_in_nid is checked on systems where pfns can interleave
1564  * within a node: a pfn is between start and end of a node, but does not belong
1565  * to this memory node.
1566  */
1567 static inline bool __init
1568 deferred_pfn_valid(int nid, unsigned long pfn,
1569                    struct mminit_pfnnid_cache *nid_init_state)
1570 {
1571         if (!pfn_valid_within(pfn))
1572                 return false;
1573         if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn))
1574                 return false;
1575         if (!meminit_pfn_in_nid(pfn, nid, nid_init_state))
1576                 return false;
1577         return true;
1578 }
1579 
1580 /*
1581  * Free pages to buddy allocator. Try to free aligned pages in
1582  * pageblock_nr_pages sizes.
1583  */
1584 static void __init deferred_free_pages(int nid, int zid, unsigned long pfn,
1585                                        unsigned long end_pfn)
1586 {
1587         struct mminit_pfnnid_cache nid_init_state = { };
1588         unsigned long nr_pgmask = pageblock_nr_pages - 1;
1589         unsigned long nr_free = 0;
1590 
1591         for (; pfn < end_pfn; pfn++) {
1592                 if (!deferred_pfn_valid(nid, pfn, &nid_init_state)) {
1593                         deferred_free_range(pfn - nr_free, nr_free);
1594                         nr_free = 0;
1595                 } else if (!(pfn & nr_pgmask)) {
1596                         deferred_free_range(pfn - nr_free, nr_free);
1597                         nr_free = 1;
1598                         touch_nmi_watchdog();
1599                 } else {
1600                         nr_free++;
1601                 }
1602         }
1603         /* Free the last block of pages to allocator */
1604         deferred_free_range(pfn - nr_free, nr_free);
1605 }
1606 
1607 /*
1608  * Initialize struct pages.  We minimize pfn page lookups and scheduler checks
1609  * by performing it only once every pageblock_nr_pages.
1610  * Return number of pages initialized.
1611  */
1612 static unsigned long  __init deferred_init_pages(int nid, int zid,
1613                                                  unsigned long pfn,
1614                                                  unsigned long end_pfn)
1615 {
1616         struct mminit_pfnnid_cache nid_init_state = { };
1617         unsigned long nr_pgmask = pageblock_nr_pages - 1;
1618         unsigned long nr_pages = 0;
1619         struct page *page = NULL;
1620 
1621         for (; pfn < end_pfn; pfn++) {
1622                 if (!deferred_pfn_valid(nid, pfn, &nid_init_state)) {
1623                         page = NULL;
1624                         continue;
1625                 } else if (!page || !(pfn & nr_pgmask)) {
1626                         page = pfn_to_page(pfn);
1627                         touch_nmi_watchdog();
1628                 } else {
1629                         page++;
1630                 }
1631                 __init_single_page(page, pfn, zid, nid);
1632                 nr_pages++;
1633         }
1634         return (nr_pages);
1635 }
1636 
1637 /* Initialise remaining memory on a node */
1638 static int __init deferred_init_memmap(void *data)
1639 {
1640         pg_data_t *pgdat = data;
1641         int nid = pgdat->node_id;
1642         unsigned long start = jiffies;
1643         unsigned long nr_pages = 0;
1644         unsigned long spfn, epfn, first_init_pfn, flags;
1645         phys_addr_t spa, epa;
1646         int zid;
1647         struct zone *zone;
1648         const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
1649         u64 i;
1650 
1651         /* Bind memory initialisation thread to a local node if possible */
1652         if (!cpumask_empty(cpumask))
1653                 set_cpus_allowed_ptr(current, cpumask);
1654 
1655         pgdat_resize_lock(pgdat, &flags);
1656         first_init_pfn = pgdat->first_deferred_pfn;
1657         if (first_init_pfn == ULONG_MAX) {
1658                 pgdat_resize_unlock(pgdat, &flags);
1659                 pgdat_init_report_one_done();
1660                 return 0;
1661         }
1662 
1663         /* Sanity check boundaries */
1664         BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
1665         BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
1666         pgdat->first_deferred_pfn = ULONG_MAX;
1667 
1668         /* Only the highest zone is deferred so find it */
1669         for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1670                 zone = pgdat->node_zones + zid;
1671                 if (first_init_pfn < zone_end_pfn(zone))
1672                         break;
1673         }
1674         first_init_pfn = max(zone->zone_start_pfn, first_init_pfn);
1675 
1676         /*
1677          * Initialize and free pages. We do it in two loops: first we initialize
1678          * struct page, than free to buddy allocator, because while we are
1679          * freeing pages we can access pages that are ahead (computing buddy
1680          * page in __free_one_page()).
1681          */
1682         for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) {
1683                 spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa));
1684                 epfn = min_t(unsigned long, zone_end_pfn(zone), PFN_DOWN(epa));
1685                 nr_pages += deferred_init_pages(nid, zid, spfn, epfn);
1686         }
1687         for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) {
1688                 spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa));
1689                 epfn = min_t(unsigned long, zone_end_pfn(zone), PFN_DOWN(epa));
1690                 deferred_free_pages(nid, zid, spfn, epfn);
1691         }
1692         pgdat_resize_unlock(pgdat, &flags);
1693 
1694         /* Sanity check that the next zone really is unpopulated */
1695         WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
1696 
1697         pr_info("node %d initialised, %lu pages in %ums\n", nid, nr_pages,
1698                                         jiffies_to_msecs(jiffies - start));
1699 
1700         pgdat_init_report_one_done();
1701         return 0;
1702 }
1703 
1704 /*
1705  * If this zone has deferred pages, try to grow it by initializing enough
1706  * deferred pages to satisfy the allocation specified by order, rounded up to
1707  * the nearest PAGES_PER_SECTION boundary.  So we're adding memory in increments
1708  * of SECTION_SIZE bytes by initializing struct pages in increments of
1709  * PAGES_PER_SECTION * sizeof(struct page) bytes.
1710  *
1711  * Return true when zone was grown, otherwise return false. We return true even
1712  * when we grow less than requested, to let the caller decide if there are
1713  * enough pages to satisfy the allocation.
1714  *
1715  * Note: We use noinline because this function is needed only during boot, and
1716  * it is called from a __ref function _deferred_grow_zone. This way we are
1717  * making sure that it is not inlined into permanent text section.
1718  */
1719 static noinline bool __init
1720 deferred_grow_zone(struct zone *zone, unsigned int order)
1721 {
1722         int zid = zone_idx(zone);
1723         int nid = zone_to_nid(zone);
1724         pg_data_t *pgdat = NODE_DATA(nid);
1725         unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);
1726         unsigned long nr_pages = 0;
1727         unsigned long first_init_pfn, spfn, epfn, t, flags;
1728         unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
1729         phys_addr_t spa, epa;
1730         u64 i;
1731 
1732         /* Only the last zone may have deferred pages */
1733         if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))
1734                 return false;
1735 
1736         pgdat_resize_lock(pgdat, &flags);
1737 
1738         /*
1739          * If deferred pages have been initialized while we were waiting for
1740          * the lock, return true, as the zone was grown.  The caller will retry
1741          * this zone.  We won't return to this function since the caller also
1742          * has this static branch.
1743          */
1744         if (!static_branch_unlikely(&deferred_pages)) {
1745                 pgdat_resize_unlock(pgdat, &flags);
1746                 return true;
1747         }
1748 
1749         /*
1750          * If someone grew this zone while we were waiting for spinlock, return
1751          * true, as there might be enough pages already.
1752          */
1753         if (first_deferred_pfn != pgdat->first_deferred_pfn) {
1754                 pgdat_resize_unlock(pgdat, &flags);
1755                 return true;
1756         }
1757 
1758         first_init_pfn = max(zone->zone_start_pfn, first_deferred_pfn);
1759 
1760         if (first_init_pfn >= pgdat_end_pfn(pgdat)) {
1761                 pgdat_resize_unlock(pgdat, &flags);
1762                 return false;
1763         }
1764 
1765         for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) {
1766                 spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa));
1767                 epfn = min_t(unsigned long, zone_end_pfn(zone), PFN_DOWN(epa));
1768 
1769                 while (spfn < epfn && nr_pages < nr_pages_needed) {
1770                         t = ALIGN(spfn + PAGES_PER_SECTION, PAGES_PER_SECTION);
1771                         first_deferred_pfn = min(t, epfn);
1772                         nr_pages += deferred_init_pages(nid, zid, spfn,
1773                                                         first_deferred_pfn);
1774                         spfn = first_deferred_pfn;
1775                 }
1776 
1777                 if (nr_pages >= nr_pages_needed)
1778                         break;
1779         }
1780 
1781         for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) {
1782                 spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa));
1783                 epfn = min_t(unsigned long, first_deferred_pfn, PFN_DOWN(epa));
1784                 deferred_free_pages(nid, zid, spfn, epfn);
1785 
1786                 if (first_deferred_pfn == epfn)
1787                         break;
1788         }
1789         pgdat->first_deferred_pfn = first_deferred_pfn;
1790         pgdat_resize_unlock(pgdat, &flags);
1791 
1792         return nr_pages > 0;
1793 }
1794 
1795 /*
1796  * deferred_grow_zone() is __init, but it is called from
1797  * get_page_from_freelist() during early boot until deferred_pages permanently
1798  * disables this call. This is why we have refdata wrapper to avoid warning,
1799  * and to ensure that the function body gets unloaded.
1800  */
1801 static bool __ref
1802 _deferred_grow_zone(struct zone *zone, unsigned int order)
1803 {
1804         return deferred_grow_zone(zone, order);
1805 }
1806 
1807 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1808 
1809 void __init page_alloc_init_late(void)
1810 {
1811         struct zone *zone;
1812 
1813 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1814         int nid;
1815 
1816         /* There will be num_node_state(N_MEMORY) threads */
1817         atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
1818         for_each_node_state(nid, N_MEMORY) {
1819                 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
1820         }
1821 
1822         /* Block until all are initialised */
1823         wait_for_completion(&pgdat_init_all_done_comp);
1824 
1825         /*
1826          * We initialized the rest of the deferred pages.  Permanently disable
1827          * on-demand struct page initialization.
1828          */
1829         static_branch_disable(&deferred_pages);
1830 
1831         /* Reinit limits that are based on free pages after the kernel is up */
1832         files_maxfiles_init();
1833 #endif
1834 #ifdef CONFIG_ARCH_DISCARD_MEMBLOCK
1835         /* Discard memblock private memory */
1836         memblock_discard();
1837 #endif
1838 
1839         for_each_populated_zone(zone)
1840                 set_zone_contiguous(zone);
1841 }
1842 
1843 #ifdef CONFIG_CMA
1844 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */
1845 void __init init_cma_reserved_pageblock(struct page *page)
1846 {
1847         unsigned i = pageblock_nr_pages;
1848         struct page *p = page;
1849 
1850         do {
1851                 __ClearPageReserved(p);
1852                 set_page_count(p, 0);
1853         } while (++p, --i);
1854 
1855         set_pageblock_migratetype(page, MIGRATE_CMA);
1856 
1857         if (pageblock_order >= MAX_ORDER) {
1858                 i = pageblock_nr_pages;
1859                 p = page;
1860                 do {
1861                         set_page_refcounted(p);
1862                         __free_pages(p, MAX_ORDER - 1);
1863                         p += MAX_ORDER_NR_PAGES;
1864                 } while (i -= MAX_ORDER_NR_PAGES);
1865         } else {
1866                 set_page_refcounted(page);
1867                 __free_pages(page, pageblock_order);
1868         }
1869 
1870         adjust_managed_page_count(page, pageblock_nr_pages);
1871 }
1872 #endif
1873 
1874 /*
1875  * The order of subdivision here is critical for the IO subsystem.
1876  * Please do not alter this order without good reasons and regression
1877  * testing. Specifically, as large blocks of memory are subdivided,
1878  * the order in which smaller blocks are delivered depends on the order
1879  * they're subdivided in this function. This is the primary factor
1880  * influencing the order in which pages are delivered to the IO
1881  * subsystem according to empirical testing, and this is also justified
1882  * by considering the behavior of a buddy system containing a single
1883  * large block of memory acted on by a series of small allocations.
1884  * This behavior is a critical factor in sglist merging's success.
1885  *
1886  * -- nyc
1887  */
1888 static inline void expand(struct zone *zone, struct page *page,
1889         int low, int high, struct free_area *area,
1890         int migratetype)
1891 {
1892         unsigned long size = 1 << high;
1893 
1894         while (high > low) {
1895                 area--;
1896                 high--;
1897                 size >>= 1;
1898                 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
1899 
1900                 /*
1901                  * Mark as guard pages (or page), that will allow to
1902                  * merge back to allocator when buddy will be freed.
1903                  * Corresponding page table entries will not be touched,
1904                  * pages will stay not present in virtual address space
1905                  */
1906                 if (set_page_guard(zone, &page[size], high, migratetype))
1907                         continue;
1908 
1909                 list_add(&page[size].lru, &area->free_list[migratetype]);
1910                 area->nr_free++;
1911                 set_page_order(&page[size], high);
1912         }
1913 }
1914 
1915 static void check_new_page_bad(struct page *page)
1916 {
1917         const char *bad_reason = NULL;
1918         unsigned long bad_flags = 0;
1919 
1920         if (unlikely(atomic_read(&page->_mapcount) != -1))
1921                 bad_reason = "nonzero mapcount";
1922         if (unlikely(page->mapping != NULL))
1923                 bad_reason = "non-NULL mapping";
1924         if (unlikely(page_ref_count(page) != 0))
1925                 bad_reason = "nonzero _count";
1926         if (unlikely(page->flags & __PG_HWPOISON)) {
1927                 bad_reason = "HWPoisoned (hardware-corrupted)";
1928                 bad_flags = __PG_HWPOISON;
1929                 /* Don't complain about hwpoisoned pages */
1930                 page_mapcount_reset(page); /* remove PageBuddy */
1931                 return;
1932         }
1933         if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) {
1934                 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set";
1935                 bad_flags = PAGE_FLAGS_CHECK_AT_PREP;
1936         }
1937 #ifdef CONFIG_MEMCG
1938         if (unlikely(page->mem_cgroup))
1939                 bad_reason = "page still charged to cgroup";
1940 #endif
1941         bad_page(page, bad_reason, bad_flags);
1942 }
1943 
1944 /*
1945  * This page is about to be returned from the page allocator
1946  */
1947 static inline int check_new_page(struct page *page)
1948 {
1949         if (likely(page_expected_state(page,
1950                                 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON)))
1951                 return 0;
1952 
1953         check_new_page_bad(page);
1954         return 1;
1955 }
1956 
1957 static inline bool free_pages_prezeroed(void)
1958 {
1959         return IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) &&
1960                 page_poisoning_enabled();
1961 }
1962 
1963 #ifdef CONFIG_DEBUG_VM
1964 static bool check_pcp_refill(struct page *page)
1965 {
1966         return false;
1967 }
1968 
1969 static bool check_new_pcp(struct page *page)
1970 {
1971         return check_new_page(page);
1972 }
1973 #else
1974 static bool check_pcp_refill(struct page *page)
1975 {
1976         return check_new_page(page);
1977 }
1978 static bool check_new_pcp(struct page *page)
1979 {
1980         return false;
1981 }
1982 #endif /* CONFIG_DEBUG_VM */
1983 
1984 static bool check_new_pages(struct page *page, unsigned int order)
1985 {
1986         int i;
1987         for (i = 0; i < (1 << order); i++) {
1988                 struct page *p = page + i;
1989 
1990                 if (unlikely(check_new_page(p)))
1991                         return true;
1992         }
1993 
1994         return false;
1995 }
1996 
1997 inline void post_alloc_hook(struct page *page, unsigned int order,
1998                                 gfp_t gfp_flags)
1999 {
2000         set_page_private(page, 0);
2001         set_page_refcounted(page);
2002 
2003         arch_alloc_page(page, order);
2004         kernel_map_pages(page, 1 << order, 1);
2005         kasan_alloc_pages(page, order);
2006         kernel_poison_pages(page, 1 << order, 1);
2007         set_page_owner(page, order, gfp_flags);
2008 }
2009 
2010 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
2011                                                         unsigned int alloc_flags)
2012 {
2013         int i;
2014 
2015         post_alloc_hook(page, order, gfp_flags);
2016 
2017         if (!free_pages_prezeroed() && (gfp_flags & __GFP_ZERO))
2018                 for (i = 0; i < (1 << order); i++)
2019                         clear_highpage(page + i);
2020 
2021         if (order && (gfp_flags & __GFP_COMP))
2022                 prep_compound_page(page, order);
2023 
2024         /*
2025          * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
2026          * allocate the page. The expectation is that the caller is taking
2027          * steps that will free more memory. The caller should avoid the page
2028          * being used for !PFMEMALLOC purposes.
2029          */
2030         if (alloc_flags & ALLOC_NO_WATERMARKS)
2031                 set_page_pfmemalloc(page);
2032         else
2033                 clear_page_pfmemalloc(page);
2034 }
2035 
2036 /*
2037  * Go through the free lists for the given migratetype and remove
2038  * the smallest available page from the freelists
2039  */
2040 static __always_inline
2041 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
2042                                                 int migratetype)
2043 {
2044         unsigned int current_order;
2045         struct free_area *area;
2046         struct page *page;
2047 
2048         /* Find a page of the appropriate size in the preferred list */
2049         for (current_order = order; current_order < MAX_ORDER; ++current_order) {
2050                 area = &(zone->free_area[current_order]);
2051                 page = list_first_entry_or_null(&area->free_list[migratetype],
2052                                                         struct page, lru);
2053                 if (!page)
2054                         continue;
2055                 list_del(&page->lru);
2056                 rmv_page_order(page);
2057                 area->nr_free--;
2058                 expand(zone, page, order, current_order, area, migratetype);
2059                 set_pcppage_migratetype(page, migratetype);
2060                 return page;
2061         }
2062 
2063         return NULL;
2064 }
2065 
2066 
2067 /*
2068  * This array describes the order lists are fallen back to when
2069  * the free lists for the desirable migrate type are depleted
2070  */
2071 static int fallbacks[MIGRATE_TYPES][4] = {
2072         [MIGRATE_UNMOVABLE]   = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE,   MIGRATE_TYPES },
2073         [MIGRATE_MOVABLE]     = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES },
2074         [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE,   MIGRATE_MOVABLE,   MIGRATE_TYPES },
2075 #ifdef CONFIG_CMA
2076         [MIGRATE_CMA]         = { MIGRATE_TYPES }, /* Never used */
2077 #endif
2078 #ifdef CONFIG_MEMORY_ISOLATION
2079         [MIGRATE_ISOLATE]     = { MIGRATE_TYPES }, /* Never used */
2080 #endif
2081 };
2082 
2083 #ifdef CONFIG_CMA
2084 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone,
2085                                         unsigned int order)
2086 {
2087         return __rmqueue_smallest(zone, order, MIGRATE_CMA);
2088 }
2089 #else
2090 static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
2091                                         unsigned int order) { return NULL; }
2092 #endif
2093 
2094 /*
2095  * Move the free pages in a range to the free lists of the requested type.
2096  * Note that start_page and end_pages are not aligned on a pageblock
2097  * boundary. If alignment is required, use move_freepages_block()
2098  */
2099 static int move_freepages(struct zone *zone,
2100                           struct page *start_page, struct page *end_page,
2101                           int migratetype, int *num_movable)
2102 {
2103         struct page *page;
2104         unsigned int order;
2105         int pages_moved = 0;
2106 
2107 #ifndef CONFIG_HOLES_IN_ZONE
2108         /*
2109          * page_zone is not safe to call in this context when
2110          * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
2111          * anyway as we check zone boundaries in move_freepages_block().
2112          * Remove at a later date when no bug reports exist related to
2113          * grouping pages by mobility
2114          */
2115         VM_BUG_ON(pfn_valid(page_to_pfn(start_page)) &&
2116                   pfn_valid(page_to_pfn(end_page)) &&
2117                   page_zone(start_page) != page_zone(end_page));
2118 #endif
2119         for (page = start_page; page <= end_page;) {
2120                 if (!pfn_valid_within(page_to_pfn(page))) {
2121                         page++;
2122                         continue;
2123                 }
2124 
2125                 /* Make sure we are not inadvertently changing nodes */
2126                 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
2127 
2128                 if (!PageBuddy(page)) {
2129                         /*
2130                          * We assume that pages that could be isolated for
2131                          * migration are movable. But we don't actually try
2132                          * isolating, as that would be expensive.
2133                          */
2134                         if (num_movable &&
2135                                         (PageLRU(page) || __PageMovable(page)))
2136                                 (*num_movable)++;
2137 
2138                         page++;
2139                         continue;
2140                 }
2141 
2142                 order = page_order(page);
2143                 list_move(&page->lru,
2144                           &zone->free_area[order].free_list[migratetype]);
2145                 page += 1 << order;
2146                 pages_moved += 1 << order;
2147         }
2148 
2149         return pages_moved;
2150 }
2151 
2152 int move_freepages_block(struct zone *zone, struct page *page,
2153                                 int migratetype, int *num_movable)
2154 {
2155         unsigned long start_pfn, end_pfn;
2156         struct page *start_page, *end_page;
2157 
2158         if (num_movable)
2159                 *num_movable = 0;
2160 
2161         start_pfn = page_to_pfn(page);
2162         start_pfn = start_pfn & ~(pageblock_nr_pages-1);
2163         start_page = pfn_to_page(start_pfn);
2164         end_page = start_page + pageblock_nr_pages - 1;
2165         end_pfn = start_pfn + pageblock_nr_pages - 1;
2166 
2167         /* Do not cross zone boundaries */
2168         if (!zone_spans_pfn(zone, start_pfn))
2169                 start_page = page;
2170         if (!zone_spans_pfn(zone, end_pfn))
2171                 return 0;
2172 
2173         return move_freepages(zone, start_page, end_page, migratetype,
2174                                                                 num_movable);
2175 }
2176 
2177 static void change_pageblock_range(struct page *pageblock_page,
2178                                         int start_order, int migratetype)
2179 {
2180         int nr_pageblocks = 1 << (start_order - pageblock_order);
2181 
2182         while (nr_pageblocks--) {
2183                 set_pageblock_migratetype(pageblock_page, migratetype);
2184                 pageblock_page += pageblock_nr_pages;
2185         }
2186 }
2187 
2188 /*
2189  * When we are falling back to another migratetype during allocation, try to
2190  * steal extra free pages from the same pageblocks to satisfy further
2191  * allocations, instead of polluting multiple pageblocks.
2192  *
2193  * If we are stealing a relatively large buddy page, it is likely there will
2194  * be more free pages in the pageblock, so try to steal them all. For
2195  * reclaimable and unmovable allocations, we steal regardless of page size,
2196  * as fragmentation caused by those allocations polluting movable pageblocks
2197  * is worse than movable allocations stealing from unmovable and reclaimable
2198  * pageblocks.
2199  */
2200 static bool can_steal_fallback(unsigned int order, int start_mt)
2201 {
2202         /*
2203          * Leaving this order check is intended, although there is
2204          * relaxed order check in next check. The reason is that
2205          * we can actually steal whole pageblock if this condition met,
2206          * but, below check doesn't guarantee it and that is just heuristic
2207          * so could be changed anytime.
2208          */
2209         if (order >= pageblock_order)
2210                 return true;
2211 
2212         if (order >= pageblock_order / 2 ||
2213                 start_mt == MIGRATE_RECLAIMABLE ||
2214                 start_mt == MIGRATE_UNMOVABLE ||
2215                 page_group_by_mobility_disabled)
2216                 return true;
2217 
2218         return false;
2219 }
2220 
2221 static inline void boost_watermark(struct zone *zone)
2222 {
2223         unsigned long max_boost;
2224 
2225         if (!watermark_boost_factor)
2226                 return;
2227 
2228         max_boost = mult_frac(zone->_watermark[WMARK_HIGH],
2229                         watermark_boost_factor, 10000);
2230 
2231         /*
2232          * high watermark may be uninitialised if fragmentation occurs
2233          * very early in boot so do not boost. We do not fall
2234          * through and boost by pageblock_nr_pages as failing
2235          * allocations that early means that reclaim is not going
2236          * to help and it may even be impossible to reclaim the
2237          * boosted watermark resulting in a hang.
2238          */
2239         if (!max_boost)
2240                 return;
2241 
2242         max_boost = max(pageblock_nr_pages, max_boost);
2243 
2244         zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages,
2245                 max_boost);
2246 }
2247 
2248 /*
2249  * This function implements actual steal behaviour. If order is large enough,
2250  * we can steal whole pageblock. If not, we first move freepages in this
2251  * pageblock to our migratetype and determine how many already-allocated pages
2252  * are there in the pageblock with a compatible migratetype. If at least half
2253  * of pages are free or compatible, we can change migratetype of the pageblock
2254  * itself, so pages freed in the future will be put on the correct free list.
2255  */
2256 static void steal_suitable_fallback(struct zone *zone, struct page *page,
2257                 unsigned int alloc_flags, int start_type, bool whole_block)
2258 {
2259         unsigned int current_order = page_order(page);
2260         struct free_area *area;
2261         int free_pages, movable_pages, alike_pages;
2262         int old_block_type;
2263 
2264         old_block_type = get_pageblock_migratetype(page);
2265 
2266         /*
2267          * This can happen due to races and we want to prevent broken
2268          * highatomic accounting.
2269          */
2270         if (is_migrate_highatomic(old_block_type))
2271                 goto single_page;
2272 
2273         /* Take ownership for orders >= pageblock_order */
2274         if (current_order >= pageblock_order) {
2275                 change_pageblock_range(page, current_order, start_type);
2276                 goto single_page;
2277         }
2278 
2279         /*
2280          * Boost watermarks to increase reclaim pressure to reduce the
2281          * likelihood of future fallbacks. Wake kswapd now as the node
2282          * may be balanced overall and kswapd will not wake naturally.
2283          */
2284         boost_watermark(zone);
2285         if (alloc_flags & ALLOC_KSWAPD)
2286                 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
2287 
2288         /* We are not allowed to try stealing from the whole block */
2289         if (!whole_block)
2290                 goto single_page;
2291 
2292         free_pages = move_freepages_block(zone, page, start_type,
2293                                                 &movable_pages);
2294         /*
2295          * Determine how many pages are compatible with our allocation.
2296          * For movable allocation, it's the number of movable pages which
2297          * we just obtained. For other types it's a bit more tricky.
2298          */
2299         if (start_type == MIGRATE_MOVABLE) {
2300                 alike_pages = movable_pages;
2301         } else {
2302                 /*
2303                  * If we are falling back a RECLAIMABLE or UNMOVABLE allocation
2304                  * to MOVABLE pageblock, consider all non-movable pages as
2305                  * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or
2306                  * vice versa, be conservative since we can't distinguish the
2307                  * exact migratetype of non-movable pages.
2308                  */
2309                 if (old_block_type == MIGRATE_MOVABLE)
2310                         alike_pages = pageblock_nr_pages
2311                                                 - (free_pages + movable_pages);
2312                 else
2313                         alike_pages = 0;
2314         }
2315 
2316         /* moving whole block can fail due to zone boundary conditions */
2317         if (!free_pages)
2318                 goto single_page;
2319 
2320         /*
2321          * If a sufficient number of pages in the block are either free or of
2322          * comparable migratability as our allocation, claim the whole block.
2323          */
2324         if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||
2325                         page_group_by_mobility_disabled)
2326                 set_pageblock_migratetype(page, start_type);
2327 
2328         return;
2329 
2330 single_page:
2331         area = &zone->free_area[current_order];
2332         list_move(&page->lru, &area->free_list[start_type]);
2333 }
2334 
2335 /*
2336  * Check whether there is a suitable fallback freepage with requested order.
2337  * If only_stealable is true, this function returns fallback_mt only if
2338  * we can steal other freepages all together. This would help to reduce
2339  * fragmentation due to mixed migratetype pages in one pageblock.
2340  */
2341 int find_suitable_fallback(struct free_area *area, unsigned int order,
2342                         int migratetype, bool only_stealable, bool *can_steal)
2343 {
2344         int i;
2345         int fallback_mt;
2346 
2347         if (area->nr_free == 0)
2348                 return -1;
2349 
2350         *can_steal = false;
2351         for (i = 0;; i++) {
2352                 fallback_mt = fallbacks[migratetype][i];
2353                 if (fallback_mt == MIGRATE_TYPES)
2354                         break;
2355 
2356                 if (list_empty(&area->free_list[fallback_mt]))
2357                         continue;
2358 
2359                 if (can_steal_fallback(order, migratetype))
2360                         *can_steal = true;
2361 
2362                 if (!only_stealable)
2363                         return fallback_mt;
2364 
2365                 if (*can_steal)
2366                         return fallback_mt;
2367         }
2368 
2369         return -1;
2370 }
2371 
2372 /*
2373  * Reserve a pageblock for exclusive use of high-order atomic allocations if
2374  * there are no empty page blocks that contain a page with a suitable order
2375  */
2376 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
2377                                 unsigned int alloc_order)
2378 {
2379         int mt;
2380         unsigned long max_managed, flags;
2381 
2382         /*
2383          * Limit the number reserved to 1 pageblock or roughly 1% of a zone.
2384          * Check is race-prone but harmless.
2385          */
2386         max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages;
2387         if (zone->nr_reserved_highatomic >= max_managed)
2388                 return;
2389 
2390         spin_lock_irqsave(&zone->lock, flags);
2391 
2392         /* Recheck the nr_reserved_highatomic limit under the lock */
2393         if (zone->nr_reserved_highatomic >= max_managed)
2394                 goto out_unlock;
2395 
2396         /* Yoink! */
2397         mt = get_pageblock_migratetype(page);
2398         if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt)
2399             && !is_migrate_cma(mt)) {
2400                 zone->nr_reserved_highatomic += pageblock_nr_pages;
2401                 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
2402                 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL);
2403         }
2404 
2405 out_unlock:
2406         spin_unlock_irqrestore(&zone->lock, flags);
2407 }
2408 
2409 /*
2410  * Used when an allocation is about to fail under memory pressure. This
2411  * potentially hurts the reliability of high-order allocations when under
2412  * intense memory pressure but failed atomic allocations should be easier
2413  * to recover from than an OOM.
2414  *
2415  * If @force is true, try to unreserve a pageblock even though highatomic
2416  * pageblock is exhausted.
2417  */
2418 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
2419                                                 bool force)
2420 {
2421         struct zonelist *zonelist = ac->zonelist;
2422         unsigned long flags;
2423         struct zoneref *z;
2424         struct zone *zone;
2425         struct page *page;
2426         int order;
2427         bool ret;
2428 
2429         for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx,
2430                                                                 ac->nodemask) {
2431                 /*
2432                  * Preserve at least one pageblock unless memory pressure
2433                  * is really high.
2434                  */
2435                 if (!force && zone->nr_reserved_highatomic <=
2436                                         pageblock_nr_pages)
2437                         continue;
2438 
2439                 spin_lock_irqsave(&zone->lock, flags);
2440                 for (order = 0; order < MAX_ORDER; order++) {
2441                         struct free_area *area = &(zone->free_area[order]);
2442 
2443                         page = list_first_entry_or_null(
2444                                         &area->free_list[MIGRATE_HIGHATOMIC],
2445                                         struct page, lru);
2446                         if (!page)
2447                                 continue;
2448 
2449                         /*
2450                          * In page freeing path, migratetype change is racy so
2451                          * we can counter several free pages in a pageblock
2452                          * in this loop althoug we changed the pageblock type
2453                          * from highatomic to ac->migratetype. So we should
2454                          * adjust the count once.
2455                          */
2456                         if (is_migrate_highatomic_page(page)) {
2457                                 /*
2458                                  * It should never happen but changes to
2459                                  * locking could inadvertently allow a per-cpu
2460                                  * drain to add pages to MIGRATE_HIGHATOMIC
2461                                  * while unreserving so be safe and watch for
2462                                  * underflows.
2463                                  */
2464                                 zone->nr_reserved_highatomic -= min(
2465                                                 pageblock_nr_pages,
2466                                                 zone->nr_reserved_highatomic);
2467                         }
2468 
2469                         /*
2470                          * Convert to ac->migratetype and avoid the normal
2471                          * pageblock stealing heuristics. Minimally, the caller
2472                          * is doing the work and needs the pages. More
2473                          * importantly, if the block was always converted to
2474                          * MIGRATE_UNMOVABLE or another type then the number
2475                          * of pageblocks that cannot be completely freed
2476                          * may increase.
2477                          */
2478                         set_pageblock_migratetype(page, ac->migratetype);
2479                         ret = move_freepages_block(zone, page, ac->migratetype,
2480                                                                         NULL);
2481                         if (ret) {
2482                                 spin_unlock_irqrestore(&zone->lock, flags);
2483                                 return ret;
2484                         }
2485                 }
2486                 spin_unlock_irqrestore(&zone->lock, flags);
2487         }
2488 
2489         return false;
2490 }
2491 
2492 /*
2493  * Try finding a free buddy page on the fallback list and put it on the free
2494  * list of requested migratetype, possibly along with other pages from the same
2495  * block, depending on fragmentation avoidance heuristics. Returns true if
2496  * fallback was found so that __rmqueue_smallest() can grab it.
2497  *
2498  * The use of signed ints for order and current_order is a deliberate
2499  * deviation from the rest of this file, to make the for loop
2500  * condition simpler.
2501  */
2502 static __always_inline bool
2503 __rmqueue_fallback(struct zone *zone, int order, int start_migratetype,
2504                                                 unsigned int alloc_flags)
2505 {
2506         struct free_area *area;
2507         int current_order;
2508         int min_order = order;
2509         struct page *page;
2510         int fallback_mt;
2511         bool can_steal;
2512 
2513         /*
2514          * Do not steal pages from freelists belonging to other pageblocks
2515          * i.e. orders < pageblock_order. If there are no local zones free,
2516          * the zonelists will be reiterated without ALLOC_NOFRAGMENT.
2517          */
2518         if (alloc_flags & ALLOC_NOFRAGMENT)
2519                 min_order = pageblock_order;
2520 
2521         /*
2522          * Find the largest available free page in the other list. This roughly
2523          * approximates finding the pageblock with the most free pages, which
2524          * would be too costly to do exactly.
2525          */
2526         for (current_order = MAX_ORDER - 1; current_order >= min_order;
2527                                 --current_order) {
2528                 area = &(zone->free_area[current_order]);
2529                 fallback_mt = find_suitable_fallback(area, current_order,
2530                                 start_migratetype, false, &can_steal);
2531                 if (fallback_mt == -1)
2532                         continue;
2533 
2534                 /*
2535                  * We cannot steal all free pages from the pageblock and the
2536                  * requested migratetype is movable. In that case it's better to
2537                  * steal and split the smallest available page instead of the
2538                  * largest available page, because even if the next movable
2539                  * allocation falls back into a different pageblock than this
2540                  * one, it won't cause permanent fragmentation.
2541                  */
2542                 if (!can_steal && start_migratetype == MIGRATE_MOVABLE
2543                                         && current_order > order)
2544                         goto find_smallest;
2545 
2546                 goto do_steal;
2547         }
2548 
2549         return false;
2550 
2551 find_smallest:
2552         for (current_order = order; current_order < MAX_ORDER;
2553                                                         current_order++) {
2554                 area = &(zone->free_area[current_order]);
2555                 fallback_mt = find_suitable_fallback(area, current_order,
2556                                 start_migratetype, false, &can_steal);
2557                 if (fallback_mt != -1)
2558                         break;
2559         }
2560 
2561         /*
2562          * This should not happen - we already found a suitable fallback
2563          * when looking for the largest page.
2564          */
2565         VM_BUG_ON(current_order == MAX_ORDER);
2566 
2567 do_steal:
2568         page = list_first_entry(&area->free_list[fallback_mt],
2569                                                         struct page, lru);
2570 
2571         steal_suitable_fallback(zone, page, alloc_flags, start_migratetype,
2572                                                                 can_steal);
2573 
2574         trace_mm_page_alloc_extfrag(page, order, current_order,
2575                 start_migratetype, fallback_mt);
2576 
2577         return true;
2578 
2579 }
2580 
2581 /*
2582  * Do the hard work of removing an element from the buddy allocator.
2583  * Call me with the zone->lock already held.
2584  */
2585 static __always_inline struct page *
2586 __rmqueue(struct zone *zone, unsigned int order, int migratetype,
2587                                                 unsigned int alloc_flags)
2588 {
2589         struct page *page;
2590 
2591 retry:
2592         page = __rmqueue_smallest(zone, order, migratetype);
2593         if (unlikely(!page)) {
2594                 if (migratetype == MIGRATE_MOVABLE)
2595                         page = __rmqueue_cma_fallback(zone, order);
2596 
2597                 if (!page && __rmqueue_fallback(zone, order, migratetype,
2598                                                                 alloc_flags))
2599                         goto retry;
2600         }
2601 
2602         trace_mm_page_alloc_zone_locked(page, order, migratetype);
2603         return page;
2604 }
2605 
2606 /*
2607  * Obtain a specified number of elements from the buddy allocator, all under
2608  * a single hold of the lock, for efficiency.  Add them to the supplied list.
2609  * Returns the number of new pages which were placed at *list.
2610  */
2611 static int rmqueue_bulk(struct zone *zone, unsigned int order,
2612                         unsigned long count, struct list_head *list,
2613                         int migratetype, unsigned int alloc_flags)
2614 {
2615         int i, alloced = 0;
2616 
2617         spin_lock(&zone->lock);
2618         for (i = 0; i < count; ++i) {
2619                 struct page *page = __rmqueue(zone, order, migratetype,
2620                                                                 alloc_flags);
2621                 if (unlikely(page == NULL))
2622                         break;
2623 
2624                 if (unlikely(check_pcp_refill(page)))
2625                         continue;
2626 
2627                 /*
2628                  * Split buddy pages returned by expand() are received here in
2629                  * physical page order. The page is added to the tail of
2630                  * caller's list. From the callers perspective, the linked list
2631                  * is ordered by page number under some conditions. This is
2632                  * useful for IO devices that can forward direction from the
2633                  * head, thus also in the physical page order. This is useful
2634                  * for IO devices that can merge IO requests if the physical
2635                  * pages are ordered properly.
2636                  */
2637                 list_add_tail(&page->lru, list);
2638                 alloced++;
2639                 if (is_migrate_cma(get_pcppage_migratetype(page)))
2640                         __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
2641                                               -(1 << order));
2642         }
2643 
2644         /*
2645          * i pages were removed from the buddy list even if some leak due
2646          * to check_pcp_refill failing so adjust NR_FREE_PAGES based
2647          * on i. Do not confuse with 'alloced' which is the number of
2648          * pages added to the pcp list.
2649          */
2650         __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
2651         spin_unlock(&zone->lock);
2652         return alloced;
2653 }
2654 
2655 #ifdef CONFIG_NUMA
2656 /*
2657  * Called from the vmstat counter updater to drain pagesets of this
2658  * currently executing processor on remote nodes after they have
2659  * expired.
2660  *
2661  * Note that this function must be called with the thread pinned to
2662  * a single processor.
2663  */
2664 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
2665 {
2666         unsigned long flags;
2667         int to_drain, batch;
2668 
2669         local_irq_save(flags);
2670         batch = READ_ONCE(pcp->batch);
2671         to_drain = min(pcp->count, batch);
2672         if (to_drain > 0)
2673                 free_pcppages_bulk(zone, to_drain, pcp);
2674         local_irq_restore(flags);
2675 }
2676 #endif
2677 
2678 /*
2679  * Drain pcplists of the indicated processor and zone.
2680  *
2681  * The processor must either be the current processor and the
2682  * thread pinned to the current processor or a processor that
2683  * is not online.
2684  */
2685 static void drain_pages_zone(unsigned int cpu, struct zone *zone)
2686 {
2687         unsigned long flags;
2688         struct per_cpu_pageset *pset;
2689         struct per_cpu_pages *pcp;
2690 
2691         local_irq_save(flags);
2692         pset = per_cpu_ptr(zone->pageset, cpu);
2693 
2694         pcp = &pset->pcp;
2695         if (pcp->count)
2696                 free_pcppages_bulk(zone, pcp->count, pcp);
2697         local_irq_restore(flags);
2698 }
2699 
2700 /*
2701  * Drain pcplists of all zones on the indicated processor.
2702  *
2703  * The processor must either be the current processor and the
2704  * thread pinned to the current processor or a processor that
2705  * is not online.
2706  */
2707 static void drain_pages(unsigned int cpu)
2708 {
2709         struct zone *zone;
2710 
2711         for_each_populated_zone(zone) {
2712                 drain_pages_zone(cpu, zone);
2713         }
2714 }
2715 
2716 /*
2717  * Spill all of this CPU's per-cpu pages back into the buddy allocator.
2718  *
2719  * The CPU has to be pinned. When zone parameter is non-NULL, spill just
2720  * the single zone's pages.
2721  */
2722 void drain_local_pages(struct zone *zone)
2723 {
2724         int cpu = smp_processor_id();
2725 
2726         if (zone)
2727                 drain_pages_zone(cpu, zone);
2728         else
2729                 drain_pages(cpu);
2730 }
2731 
2732 static void drain_local_pages_wq(struct work_struct *work)
2733 {
2734         struct pcpu_drain *drain;
2735 
2736         drain = container_of(work, struct pcpu_drain, work);
2737 
2738         /*
2739          * drain_all_pages doesn't use proper cpu hotplug protection so
2740          * we can race with cpu offline when the WQ can move this from
2741          * a cpu pinned worker to an unbound one. We can operate on a different
2742          * cpu which is allright but we also have to make sure to not move to
2743          * a different one.
2744          */
2745         preempt_disable();
2746         drain_local_pages(drain->zone);
2747         preempt_enable();
2748 }
2749 
2750 /*
2751  * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
2752  *
2753  * When zone parameter is non-NULL, spill just the single zone's pages.
2754  *
2755  * Note that this can be extremely slow as the draining happens in a workqueue.
2756  */
2757 void drain_all_pages(struct zone *zone)
2758 {
2759         int cpu;
2760 
2761         /*
2762          * Allocate in the BSS so we wont require allocation in
2763          * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
2764          */
2765         static cpumask_t cpus_with_pcps;
2766 
2767         /*
2768          * Make sure nobody triggers this path before mm_percpu_wq is fully
2769          * initialized.
2770          */
2771         if (WARN_ON_ONCE(!mm_percpu_wq))
2772                 return;
2773 
2774         /*
2775          * Do not drain if one is already in progress unless it's specific to
2776          * a zone. Such callers are primarily CMA and memory hotplug and need
2777          * the drain to be complete when the call returns.
2778          */
2779         if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) {
2780                 if (!zone)
2781                         return;
2782                 mutex_lock(&pcpu_drain_mutex);
2783         }
2784 
2785         /*
2786          * We don't care about racing with CPU hotplug event
2787          * as offline notification will cause the notified
2788          * cpu to drain that CPU pcps and on_each_cpu_mask
2789          * disables preemption as part of its processing
2790          */
2791         for_each_online_cpu(cpu) {
2792                 struct per_cpu_pageset *pcp;
2793                 struct zone *z;
2794                 bool has_pcps = false;
2795 
2796                 if (zone) {
2797                         pcp = per_cpu_ptr(zone->pageset, cpu);
2798                         if (pcp->pcp.count)
2799                                 has_pcps = true;
2800                 } else {
2801                         for_each_populated_zone(z) {
2802                                 pcp = per_cpu_ptr(z->pageset, cpu);
2803                                 if (pcp->pcp.count) {
2804                                         has_pcps = true;
2805                                         break;
2806                                 }
2807                         }
2808                 }
2809 
2810                 if (has_pcps)
2811                         cpumask_set_cpu(cpu, &cpus_with_pcps);
2812                 else
2813                         cpumask_clear_cpu(cpu, &cpus_with_pcps);
2814         }
2815 
2816         for_each_cpu(cpu, &cpus_with_pcps) {
2817                 struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu);
2818 
2819                 drain->zone = zone;
2820                 INIT_WORK(&drain->work, drain_local_pages_wq);
2821                 queue_work_on(cpu, mm_percpu_wq, &drain->work);
2822         }
2823         for_each_cpu(cpu, &cpus_with_pcps)
2824                 flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work);
2825 
2826         mutex_unlock(&pcpu_drain_mutex);
2827 }
2828 
2829 #ifdef CONFIG_HIBERNATION
2830 
2831 /*
2832  * Touch the watchdog for every WD_PAGE_COUNT pages.
2833  */
2834 #define WD_PAGE_COUNT   (128*1024)
2835 
2836 void mark_free_pages(struct zone *zone)
2837 {
2838         unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT;
2839         unsigned long flags;
2840         unsigned int order, t;
2841         struct page *page;
2842 
2843         if (zone_is_empty(zone))
2844                 return;
2845 
2846         spin_lock_irqsave(&zone->lock, flags);
2847 
2848         max_zone_pfn = zone_end_pfn(zone);
2849         for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
2850                 if (pfn_valid(pfn)) {
2851                         page = pfn_to_page(pfn);
2852 
2853                         if (!--page_count) {
2854                                 touch_nmi_watchdog();
2855                                 page_count = WD_PAGE_COUNT;
2856                         }
2857 
2858                         if (page_zone(page) != zone)
2859                                 continue;
2860 
2861                         if (!swsusp_page_is_forbidden(page))
2862                                 swsusp_unset_page_free(page);
2863                 }
2864 
2865         for_each_migratetype_order(order, t) {
2866                 list_for_each_entry(page,
2867                                 &zone->free_area[order].free_list[t], lru) {
2868                         unsigned long i;
2869 
2870                         pfn = page_to_pfn(page);
2871                         for (i = 0; i < (1UL << order); i++) {
2872                                 if (!--page_count) {
2873                                         touch_nmi_watchdog();
2874                                         page_count = WD_PAGE_COUNT;
2875                                 }
2876                                 swsusp_set_page_free(pfn_to_page(pfn + i));
2877                         }
2878                 }
2879         }
2880         spin_unlock_irqrestore(&zone->lock, flags);
2881 }
2882 #endif /* CONFIG_PM */
2883 
2884 static bool free_unref_page_prepare(struct page *page, unsigned long pfn)
2885 {
2886         int migratetype;
2887 
2888         if (!free_pcp_prepare(page))
2889                 return false;
2890 
2891         migratetype = get_pfnblock_migratetype(page, pfn);
2892         set_pcppage_migratetype(page, migratetype);
2893         return true;
2894 }
2895 
2896 static void free_unref_page_commit(struct page *page, unsigned long pfn)
2897 {
2898         struct zone *zone = page_zone(page);
2899         struct per_cpu_pages *pcp;
2900         int migratetype;
2901 
2902         migratetype = get_pcppage_migratetype(page);
2903         __count_vm_event(PGFREE);
2904 
2905         /*
2906          * We only track unmovable, reclaimable and movable on pcp lists.
2907          * Free ISOLATE pages back to the allocator because they are being
2908          * offlined but treat HIGHATOMIC as movable pages so we can get those
2909          * areas back if necessary. Otherwise, we may have to free
2910          * excessively into the page allocator
2911          */
2912         if (migratetype >= MIGRATE_PCPTYPES) {
2913                 if (unlikely(is_migrate_isolate(migratetype))) {
2914                         free_one_page(zone, page, pfn, 0, migratetype);
2915                         return;
2916                 }
2917                 migratetype = MIGRATE_MOVABLE;
2918         }
2919 
2920         pcp = &this_cpu_ptr(zone->pageset)->pcp;
2921         list_add(&page->lru, &pcp->lists[migratetype]);
2922         pcp->count++;
2923         if (pcp->count >= pcp->high) {
2924                 unsigned long batch = READ_ONCE(pcp->batch);
2925                 free_pcppages_bulk(zone, batch, pcp);
2926         }
2927 }
2928 
2929 /*
2930  * Free a 0-order page
2931  */
2932 void free_unref_page(struct page *page)
2933 {
2934         unsigned long flags;
2935         unsigned long pfn = page_to_pfn(page);
2936 
2937         if (!free_unref_page_prepare(page, pfn))
2938                 return;
2939 
2940         local_irq_save(flags);
2941         free_unref_page_commit(page, pfn);
2942         local_irq_restore(flags);
2943 }
2944 
2945 /*
2946  * Free a list of 0-order pages
2947  */
2948 void free_unref_page_list(struct list_head *list)
2949 {
2950         struct page *page, *next;
2951         unsigned long flags, pfn;
2952         int batch_count = 0;
2953 
2954         /* Prepare pages for freeing */
2955         list_for_each_entry_safe(page, next, list, lru) {
2956                 pfn = page_to_pfn(page);
2957                 if (!free_unref_page_prepare(page, pfn))
2958                         list_del(&page->lru);
2959                 set_page_private(page, pfn);
2960         }
2961 
2962         local_irq_save(flags);
2963         list_for_each_entry_safe(page, next, list, lru) {
2964                 unsigned long pfn = page_private(page);
2965 
2966                 set_page_private(page, 0);
2967                 trace_mm_page_free_batched(page);
2968                 free_unref_page_commit(page, pfn);
2969 
2970                 /*
2971                  * Guard against excessive IRQ disabled times when we get
2972                  * a large list of pages to free.
2973                  */
2974                 if (++batch_count == SWAP_CLUSTER_MAX) {
2975                         local_irq_restore(flags);
2976                         batch_count = 0;
2977                         local_irq_save(flags);
2978                 }
2979         }
2980         local_irq_restore(flags);
2981 }
2982 
2983 /*
2984  * split_page takes a non-compound higher-order page, and splits it into
2985  * n (1<<order) sub-pages: page[0..n]
2986  * Each sub-page must be freed individually.
2987  *
2988  * Note: this is probably too low level an operation for use in drivers.
2989  * Please consult with lkml before using this in your driver.
2990  */
2991 void split_page(struct page *page, unsigned int order)
2992 {
2993         int i;
2994 
2995         VM_BUG_ON_PAGE(PageCompound(page), page);
2996         VM_BUG_ON_PAGE(!page_count(page), page);
2997 
2998         for (i = 1; i < (1 << order); i++)
2999                 set_page_refcounted(page + i);
3000         split_page_owner(page, order);
3001 }
3002 EXPORT_SYMBOL_GPL(split_page);
3003 
3004 int __isolate_free_page(struct page *page, unsigned int order)
3005 {
3006         unsigned long watermark;
3007         struct zone *zone;
3008         int mt;
3009 
3010         BUG_ON(!PageBuddy(page));
3011 
3012         zone = page_zone(page);
3013         mt = get_pageblock_migratetype(page);
3014 
3015         if (!is_migrate_isolate(mt)) {
3016                 /*
3017                  * Obey watermarks as if the page was being allocated. We can
3018                  * emulate a high-order watermark check with a raised order-0
3019                  * watermark, because we already know our high-order page
3020                  * exists.
3021                  */
3022                 watermark = zone->_watermark[WMARK_MIN] + (1UL << order);
3023                 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))
3024                         return 0;
3025 
3026                 __mod_zone_freepage_state(zone, -(1UL << order), mt);
3027         }
3028 
3029         /* Remove page from free list */
3030         list_del(&page->lru);
3031         zone->free_area[order].nr_free--;
3032         rmv_page_order(page);
3033 
3034         /*
3035          * Set the pageblock if the isolated page is at least half of a
3036          * pageblock
3037          */
3038         if (order >= pageblock_order - 1) {
3039                 struct page *endpage = page + (1 << order) - 1;
3040                 for (; page < endpage; page += pageblock_nr_pages) {
3041                         int mt = get_pageblock_migratetype(page);
3042                         if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)
3043                             && !is_migrate_highatomic(mt))
3044                                 set_pageblock_migratetype(page,
3045                                                           MIGRATE_MOVABLE);
3046                 }
3047         }
3048 
3049 
3050         return 1UL << order;
3051 }
3052 
3053 /*
3054  * Update NUMA hit/miss statistics
3055  *
3056  * Must be called with interrupts disabled.
3057  */
3058 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z)
3059 {
3060 #ifdef CONFIG_NUMA
3061         enum numa_stat_item local_stat = NUMA_LOCAL;
3062 
3063         /* skip numa counters update if numa stats is disabled */
3064         if (!static_branch_likely(&vm_numa_stat_key))
3065                 return;
3066 
3067         if (zone_to_nid(z) != numa_node_id())
3068                 local_stat = NUMA_OTHER;
3069 
3070         if (zone_to_nid(z) == zone_to_nid(preferred_zone))
3071                 __inc_numa_state(z, NUMA_HIT);
3072         else {
3073                 __inc_numa_state(z, NUMA_MISS);
3074                 __inc_numa_state(preferred_zone, NUMA_FOREIGN);
3075         }
3076         __inc_numa_state(z, local_stat);
3077 #endif
3078 }
3079 
3080 /* Remove page from the per-cpu list, caller must protect the list */
3081 static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype,
3082                         unsigned int alloc_flags,
3083                         struct per_cpu_pages *pcp,
3084                         struct list_head *list)
3085 {
3086         struct page *page;
3087 
3088         do {
3089                 if (list_empty(list)) {
3090                         pcp->count += rmqueue_bulk(zone, 0,
3091                                         pcp->batch, list,
3092                                         migratetype, alloc_flags);
3093                         if (unlikely(list_empty(list)))
3094                                 return NULL;
3095                 }
3096 
3097                 page = list_first_entry(list, struct page, lru);
3098                 list_del(&page->lru);
3099                 pcp->count--;
3100         } while (check_new_pcp(page));
3101 
3102         return page;
3103 }
3104 
3105 /* Lock and remove page from the per-cpu list */
3106 static struct page *rmqueue_pcplist(struct zone *preferred_zone,
3107                         struct zone *zone, unsigned int order,
3108                         gfp_t gfp_flags, int migratetype,
3109                         unsigned int alloc_flags)
3110 {
3111         struct per_cpu_pages *pcp;
3112         struct list_head *list;
3113         struct page *page;
3114         unsigned long flags;
3115 
3116         local_irq_save(flags);
3117         pcp = &this_cpu_ptr(zone->pageset)->pcp;
3118         list = &pcp->lists[migratetype];
3119         page = __rmqueue_pcplist(zone,  migratetype, alloc_flags, pcp, list);
3120         if (page) {
3121                 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
3122                 zone_statistics(preferred_zone, zone);
3123         }
3124         local_irq_restore(flags);
3125         return page;
3126 }
3127 
3128 /*
3129  * Allocate a page from the given zone. Use pcplists for order-0 allocations.
3130  */
3131 static inline
3132 struct page *rmqueue(struct zone *preferred_zone,
3133                         struct zone *zone, unsigned int order,
3134                         gfp_t gfp_flags, unsigned int alloc_flags,
3135                         int migratetype)
3136 {
3137         unsigned long flags;
3138         struct page *page;
3139 
3140         if (likely(order == 0)) {
3141                 page = rmqueue_pcplist(preferred_zone, zone, order,
3142                                 gfp_flags, migratetype, alloc_flags);
3143                 goto out;
3144         }
3145 
3146         /*
3147          * We most definitely don't want callers attempting to
3148          * allocate greater than order-1 page units with __GFP_NOFAIL.
3149          */
3150         WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
3151         spin_lock_irqsave(&zone->lock, flags);
3152 
3153         do {
3154                 page = NULL;
3155                 if (alloc_flags & ALLOC_HARDER) {
3156                         page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
3157                         if (page)
3158                                 trace_mm_page_alloc_zone_locked(page, order, migratetype);
3159                 }
3160                 if (!page)
3161                         page = __rmqueue(zone, order, migratetype, alloc_flags);
3162         } while (page && check_new_pages(page, order));
3163         spin_unlock(&zone->lock);
3164         if (!page)
3165                 goto failed;
3166         __mod_zone_freepage_state(zone, -(1 << order),
3167                                   get_pcppage_migratetype(page));
3168 
3169         __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
3170         zone_statistics(preferred_zone, zone);
3171         local_irq_restore(flags);
3172 
3173 out:
3174         /* Separate test+clear to avoid unnecessary atomics */
3175         if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) {
3176                 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
3177                 wakeup_kswapd(zone, 0, 0, zone_idx(zone));
3178         }
3179 
3180         VM_BUG_ON_PAGE(page && bad_range(zone, page), page);
3181         return page;
3182 
3183 failed:
3184         local_irq_restore(flags);
3185         return NULL;
3186 }
3187 
3188 #ifdef CONFIG_FAIL_PAGE_ALLOC
3189 
3190 static struct {
3191         struct fault_attr attr;
3192 
3193         bool ignore_gfp_highmem;
3194         bool ignore_gfp_reclaim;
3195         u32 min_order;
3196 } fail_page_alloc = {
3197         .attr = FAULT_ATTR_INITIALIZER,
3198         .ignore_gfp_reclaim = true,
3199         .ignore_gfp_highmem = true,
3200         .min_order = 1,
3201 };
3202 
3203 static int __init setup_fail_page_alloc(char *str)
3204 {
3205         return setup_fault_attr(&fail_page_alloc.attr, str);
3206 }
3207 __setup("fail_page_alloc=", setup_fail_page_alloc);
3208 
3209 static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3210 {
3211         if (order < fail_page_alloc.min_order)
3212                 return false;
3213         if (gfp_mask & __GFP_NOFAIL)
3214                 return false;
3215         if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
3216                 return false;
3217         if (fail_page_alloc.ignore_gfp_reclaim &&
3218                         (gfp_mask & __GFP_DIRECT_RECLAIM))
3219                 return false;
3220 
3221         return should_fail(&fail_page_alloc.attr, 1 << order);
3222 }
3223 
3224 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
3225 
3226 static int __init fail_page_alloc_debugfs(void)
3227 {
3228         umode_t mode = S_IFREG | 0600;
3229         struct dentry *dir;
3230 
3231         dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
3232                                         &fail_page_alloc.attr);
3233 
3234         debugfs_create_bool("ignore-gfp-wait", mode, dir,
3235                             &fail_page_alloc.ignore_gfp_reclaim);
3236         debugfs_create_bool("ignore-gfp-highmem", mode, dir,
3237                             &fail_page_alloc.ignore_gfp_highmem);
3238         debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order);
3239 
3240         return 0;
3241 }
3242 
3243 late_initcall(fail_page_alloc_debugfs);
3244 
3245 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
3246 
3247 #else /* CONFIG_FAIL_PAGE_ALLOC */
3248 
3249 static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3250 {
3251         return false;
3252 }
3253 
3254 #endif /* CONFIG_FAIL_PAGE_ALLOC */
3255 
3256 static noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3257 {
3258         return __should_fail_alloc_page(gfp_mask, order);
3259 }
3260 ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE);
3261 
3262 /*
3263  * Return true if free base pages are above 'mark'. For high-order checks it
3264  * will return true of the order-0 watermark is reached and there is at least
3265  * one free page of a suitable size. Checking now avoids taking the zone lock
3266  * to check in the allocation paths if no pages are free.
3267  */
3268 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
3269                          int classzone_idx, unsigned int alloc_flags,
3270                          long free_pages)
3271 {
3272         long min = mark;
3273         int o;
3274         const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
3275 
3276         /* free_pages may go negative - that's OK */
3277         free_pages -= (1 << order) - 1;
3278 
3279         if (alloc_flags & ALLOC_HIGH)
3280                 min -= min / 2;
3281 
3282         /*
3283          * If the caller does not have rights to ALLOC_HARDER then subtract
3284          * the high-atomic reserves. This will over-estimate the size of the
3285          * atomic reserve but it avoids a search.
3286          */
3287         if (likely(!alloc_harder)) {
3288                 free_pages -= z->nr_reserved_highatomic;
3289         } else {
3290                 /*
3291                  * OOM victims can try even harder than normal ALLOC_HARDER
3292                  * users on the grounds that it's definitely going to be in
3293                  * the exit path shortly and free memory. Any allocation it
3294                  * makes during the free path will be small and short-lived.
3295                  */
3296                 if (alloc_flags & ALLOC_OOM)
3297                         min -= min / 2;
3298                 else
3299                         min -= min / 4;
3300         }
3301 
3302 
3303 #ifdef CONFIG_CMA
3304         /* If allocation can't use CMA areas don't use free CMA pages */
3305         if (!(alloc_flags & ALLOC_CMA))
3306                 free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES);
3307 #endif
3308 
3309         /*
3310          * Check watermarks for an order-0 allocation request. If these
3311          * are not met, then a high-order request also cannot go ahead
3312          * even if a suitable page happened to be free.
3313          */
3314         if (free_pages <= min + z->lowmem_reserve[classzone_idx])
3315                 return false;
3316 
3317         /* If this is an order-0 request then the watermark is fine */
3318         if (!order)
3319                 return true;
3320 
3321         /* For a high-order request, check at least one suitable page is free */
3322         for (o = order; o < MAX_ORDER; o++) {
3323                 struct free_area *area = &z->free_area[o];
3324                 int mt;
3325 
3326                 if (!area->nr_free)
3327                         continue;
3328 
3329                 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
3330                         if (!list_empty(&area->free_list[mt]))
3331                                 return true;
3332                 }
3333 
3334 #ifdef CONFIG_CMA
3335                 if ((alloc_flags & ALLOC_CMA) &&
3336                     !list_empty(&area->free_list[MIGRATE_CMA])) {
3337                         return true;
3338                 }
3339 #endif
3340                 if (alloc_harder &&
3341                         !list_empty(&area->free_list[MIGRATE_HIGHATOMIC]))
3342                         return true;
3343         }
3344         return false;
3345 }
3346 
3347 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
3348                       int classzone_idx, unsigned int alloc_flags)
3349 {
3350         return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
3351                                         zone_page_state(z, NR_FREE_PAGES));
3352 }
3353 
3354 static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
3355                 unsigned long mark, int classzone_idx, unsigned int alloc_flags)
3356 {
3357         long free_pages = zone_page_state(z, NR_FREE_PAGES);
3358         long cma_pages = 0;
3359 
3360 #ifdef CONFIG_CMA
3361         /* If allocation can't use CMA areas don't use free CMA pages */
3362         if (!(alloc_flags & ALLOC_CMA))
3363                 cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES);
3364 #endif
3365 
3366         /*
3367          * Fast check for order-0 only. If this fails then the reserves
3368          * need to be calculated. There is a corner case where the check
3369          * passes but only the high-order atomic reserve are free. If
3370          * the caller is !atomic then it'll uselessly search the free
3371          * list. That corner case is then slower but it is harmless.
3372          */
3373         if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx])
3374                 return true;
3375 
3376         return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
3377                                         free_pages);
3378 }
3379 
3380 bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
3381                         unsigned long mark, int classzone_idx)
3382 {
3383         long free_pages = zone_page_state(z, NR_FREE_PAGES);
3384 
3385         if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
3386                 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
3387 
3388         return __zone_watermark_ok(z, order, mark, classzone_idx, 0,
3389                                                                 free_pages);
3390 }
3391 
3392 #ifdef CONFIG_NUMA
3393 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3394 {
3395         return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <=
3396                                 RECLAIM_DISTANCE;
3397 }
3398 #else   /* CONFIG_NUMA */
3399 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3400 {
3401         return true;
3402 }
3403 #endif  /* CONFIG_NUMA */
3404 
3405 /*
3406  * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid
3407  * fragmentation is subtle. If the preferred zone was HIGHMEM then
3408  * premature use of a lower zone may cause lowmem pressure problems that
3409  * are worse than fragmentation. If the next zone is ZONE_DMA then it is
3410  * probably too small. It only makes sense to spread allocations to avoid
3411  * fragmentation between the Normal and DMA32 zones.
3412  */
3413 static inline unsigned int
3414 alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)
3415 {
3416         unsigned int alloc_flags = 0;
3417 
3418         if (gfp_mask & __GFP_KSWAPD_RECLAIM)
3419                 alloc_flags |= ALLOC_KSWAPD;
3420 
3421 #ifdef CONFIG_ZONE_DMA32
3422         if (zone_idx(zone) != ZONE_NORMAL)
3423                 goto out;
3424 
3425         /*
3426          * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and
3427          * the pointer is within zone->zone_pgdat->node_zones[]. Also assume
3428          * on UMA that if Normal is populated then so is DMA32.
3429          */
3430         BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1);
3431         if (nr_online_nodes > 1 && !populated_zone(--zone))
3432                 goto out;
3433 
3434 out:
3435 #endif /* CONFIG_ZONE_DMA32 */
3436         return alloc_flags;
3437 }
3438 
3439 /*
3440  * get_page_from_freelist goes through the zonelist trying to allocate
3441  * a page.
3442  */
3443 static struct page *
3444 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
3445                                                 const struct alloc_context *ac)
3446 {
3447         struct zoneref *z;
3448         struct zone *zone;
3449         struct pglist_data *last_pgdat_dirty_limit = NULL;
3450         bool no_fallback;
3451 
3452 retry:
3453         /*
3454          * Scan zonelist, looking for a zone with enough free.
3455          * See also __cpuset_node_allowed() comment in kernel/cpuset.c.
3456          */
3457         no_fallback = alloc_flags & ALLOC_NOFRAGMENT;
3458         z = ac->preferred_zoneref;
3459         for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
3460                                                                 ac->nodemask) {
3461                 struct page *page;
3462                 unsigned long mark;
3463 
3464                 if (cpusets_enabled() &&
3465                         (alloc_flags & ALLOC_CPUSET) &&
3466                         !__cpuset_zone_allowed(zone, gfp_mask))
3467                                 continue;
3468                 /*
3469                  * When allocating a page cache page for writing, we
3470                  * want to get it from a node that is within its dirty
3471                  * limit, such that no single node holds more than its
3472                  * proportional share of globally allowed dirty pages.
3473                  * The dirty limits take into account the node's
3474                  * lowmem reserves and high watermark so that kswapd
3475                  * should be able to balance it without having to
3476                  * write pages from its LRU list.
3477                  *
3478                  * XXX: For now, allow allocations to potentially
3479                  * exceed the per-node dirty limit in the slowpath
3480                  * (spread_dirty_pages unset) before going into reclaim,
3481                  * which is important when on a NUMA setup the allowed
3482                  * nodes are together not big enough to reach the
3483                  * global limit.  The proper fix for these situations
3484                  * will require awareness of nodes in the
3485                  * dirty-throttling and the flusher threads.
3486                  */
3487                 if (ac->spread_dirty_pages) {
3488                         if (last_pgdat_dirty_limit == zone->zone_pgdat)
3489                                 continue;
3490 
3491                         if (!node_dirty_ok(zone->zone_pgdat)) {
3492                                 last_pgdat_dirty_limit = zone->zone_pgdat;
3493                                 continue;
3494                         }
3495                 }
3496 
3497                 if (no_fallback && nr_online_nodes > 1 &&
3498                     zone != ac->preferred_zoneref->zone) {
3499                         int local_nid;
3500 
3501                         /*
3502                          * If moving to a remote node, retry but allow
3503                          * fragmenting fallbacks. Locality is more important
3504                          * than fragmentation avoidance.
3505                          */
3506                         local_nid = zone_to_nid(ac->preferred_zoneref->zone);
3507                         if (zone_to_nid(zone) != local_nid) {
3508                                 alloc_flags &= ~ALLOC_NOFRAGMENT;
3509                                 goto retry;
3510                         }
3511                 }
3512 
3513                 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
3514                 if (!zone_watermark_fast(zone, order, mark,
3515                                        ac_classzone_idx(ac), alloc_flags)) {
3516                         int ret;
3517 
3518 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
3519                         /*
3520                          * Watermark failed for this zone, but see if we can
3521                          * grow this zone if it contains deferred pages.
3522                          */
3523                         if (static_branch_unlikely(&deferred_pages)) {
3524                                 if (_deferred_grow_zone(zone, order))
3525                                         goto try_this_zone;
3526                         }
3527 #endif
3528                         /* Checked here to keep the fast path fast */
3529                         BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
3530                         if (alloc_flags & ALLOC_NO_WATERMARKS)
3531                                 goto try_this_zone;
3532 
3533                         if (node_reclaim_mode == 0 ||
3534                             !zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
3535                                 continue;
3536 
3537                         ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
3538                         switch (ret) {
3539                         case NODE_RECLAIM_NOSCAN:
3540                                 /* did not scan */
3541                                 continue;
3542                         case NODE_RECLAIM_FULL:
3543                                 /* scanned but unreclaimable */
3544                                 continue;
3545                         default:
3546                                 /* did we reclaim enough */
3547                                 if (zone_watermark_ok(zone, order, mark,
3548                                                 ac_classzone_idx(ac), alloc_flags))
3549                                         goto try_this_zone;
3550 
3551                                 continue;
3552                         }
3553                 }
3554 
3555 try_this_zone:
3556                 page = rmqueue(ac->preferred_zoneref->zone, zone, order,
3557                                 gfp_mask, alloc_flags, ac->migratetype);
3558                 if (page) {
3559                         prep_new_page(page, order, gfp_mask, alloc_flags);
3560 
3561                         /*
3562                          * If this is a high-order atomic allocation then check
3563                          * if the pageblock should be reserved for the future
3564                          */
3565                         if (unlikely(order && (alloc_flags & ALLOC_HARDER)))
3566                                 reserve_highatomic_pageblock(page, zone, order);
3567 
3568                         return page;
3569                 } else {
3570 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
3571                         /* Try again if zone has deferred pages */
3572                         if (static_branch_unlikely(&deferred_pages)) {
3573                                 if (_deferred_grow_zone(zone, order))
3574                                         goto try_this_zone;
3575                         }
3576 #endif
3577                 }
3578         }
3579 
3580         /*
3581          * It's possible on a UMA machine to get through all zones that are
3582          * fragmented. If avoiding fragmentation, reset and try again.
3583          */
3584         if (no_fallback) {
3585                 alloc_flags &= ~ALLOC_NOFRAGMENT;
3586                 goto retry;
3587         }
3588 
3589         return NULL;
3590 }
3591 
3592 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask)
3593 {
3594         unsigned int filter = SHOW_MEM_FILTER_NODES;
3595         static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1);
3596 
3597         if (!__ratelimit(&show_mem_rs))
3598                 return;
3599 
3600         /*
3601          * This documents exceptions given to allocations in certain
3602          * contexts that are allowed to allocate outside current's set
3603          * of allowed nodes.
3604          */
3605         if (!(gfp_mask & __GFP_NOMEMALLOC))
3606                 if (tsk_is_oom_victim(current) ||
3607                     (current->flags & (PF_MEMALLOC | PF_EXITING)))
3608                         filter &= ~SHOW_MEM_FILTER_NODES;
3609         if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
3610                 filter &= ~SHOW_MEM_FILTER_NODES;
3611 
3612         show_mem(filter, nodemask);
3613 }
3614 
3615 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
3616 {
3617         struct va_format vaf;
3618         va_list args;
3619         static DEFINE_RATELIMIT_STATE(nopage_rs, DEFAULT_RATELIMIT_INTERVAL,
3620                                       DEFAULT_RATELIMIT_BURST);
3621 
3622         if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs))
3623                 return;
3624 
3625         va_start(args, fmt);
3626         vaf.fmt = fmt;
3627         vaf.va = &args;
3628         pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl",
3629                         current->comm, &vaf, gfp_mask, &gfp_mask,
3630                         nodemask_pr_args(nodemask));
3631         va_end(args);
3632 
3633         cpuset_print_current_mems_allowed();
3634         pr_cont("\n");
3635         dump_stack();
3636         warn_alloc_show_mem(gfp_mask, nodemask);
3637 }
3638 
3639 static inline struct page *
3640 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order,
3641                               unsigned int alloc_flags,
3642                               const struct alloc_context *ac)
3643 {
3644         struct page *page;
3645 
3646         page = get_page_from_freelist(gfp_mask, order,
3647                         alloc_flags|ALLOC_CPUSET, ac);
3648         /*
3649          * fallback to ignore cpuset restriction if our nodes
3650          * are depleted
3651          */
3652         if (!page)
3653                 page = get_page_from_freelist(gfp_mask, order,
3654                                 alloc_flags, ac);
3655 
3656         return page;
3657 }
3658 
3659 static inline struct page *
3660 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
3661         const struct alloc_context *ac, unsigned long *did_some_progress)
3662 {
3663         struct oom_control oc = {
3664                 .zonelist = ac->zonelist,
3665                 .nodemask = ac->nodemask,
3666                 .memcg = NULL,
3667                 .gfp_mask = gfp_mask,
3668                 .order = order,
3669         };
3670         struct page *page;
3671 
3672         *did_some_progress = 0;
3673 
3674         /*
3675          * Acquire the oom lock.  If that fails, somebody else is
3676          * making progress for us.
3677          */
3678         if (!mutex_trylock(&oom_lock)) {
3679                 *did_some_progress = 1;
3680                 schedule_timeout_uninterruptible(1);
3681                 return NULL;
3682         }
3683 
3684         /*
3685          * Go through the zonelist yet one more time, keep very high watermark
3686          * here, this is only to catch a parallel oom killing, we must fail if
3687          * we're still under heavy pressure. But make sure that this reclaim
3688          * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY
3689          * allocation which will never fail due to oom_lock already held.
3690          */
3691         page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) &
3692                                       ~__GFP_DIRECT_RECLAIM, order,
3693                                       ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
3694         if (page)
3695                 goto out;
3696 
3697         /* Coredumps can quickly deplete all memory reserves */
3698         if (current->flags & PF_DUMPCORE)
3699                 goto out;
3700         /* The OOM killer will not help higher order allocs */
3701         if (order > PAGE_ALLOC_COSTLY_ORDER)
3702                 goto out;
3703         /*
3704          * We have already exhausted all our reclaim opportunities without any
3705          * success so it is time to admit defeat. We will skip the OOM killer
3706          * because it is very likely that the caller has a more reasonable
3707          * fallback than shooting a random task.
3708          */
3709         if (gfp_mask & __GFP_RETRY_MAYFAIL)
3710                 goto out;
3711         /* The OOM killer does not needlessly kill tasks for lowmem */
3712         if (ac->high_zoneidx < ZONE_NORMAL)
3713                 goto out;
3714         if (pm_suspended_storage())
3715                 goto out;
3716         /*
3717          * XXX: GFP_NOFS allocations should rather fail than rely on
3718          * other request to make a forward progress.
3719          * We are in an unfortunate situation where out_of_memory cannot
3720          * do much for this context but let's try it to at least get
3721          * access to memory reserved if the current task is killed (see
3722          * out_of_memory). Once filesystems are ready to handle allocation
3723          * failures more gracefully we should just bail out here.
3724          */
3725 
3726         /* The OOM killer may not free memory on a specific node */
3727         if (gfp_mask & __GFP_THISNODE)
3728                 goto out;
3729 
3730         /* Exhausted what can be done so it's blame time */
3731         if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
3732                 *did_some_progress = 1;
3733 
3734                 /*
3735                  * Help non-failing allocations by giving them access to memory
3736                  * reserves
3737                  */
3738                 if (gfp_mask & __GFP_NOFAIL)
3739                         page = __alloc_pages_cpuset_fallback(gfp_mask, order,
3740                                         ALLOC_NO_WATERMARKS, ac);
3741         }
3742 out:
3743         mutex_unlock(&oom_lock);
3744         return page;
3745 }
3746 
3747 /*
3748  * Maximum number of compaction retries wit a progress before OOM
3749  * killer is consider as the only way to move forward.
3750  */
3751 #define MAX_COMPACT_RETRIES 16
3752 
3753 #ifdef CONFIG_COMPACTION
3754 /* Try memory compaction for high-order allocations before reclaim */
3755 static struct page *
3756 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
3757                 unsigned int alloc_flags, const struct alloc_context *ac,
3758                 enum compact_priority prio, enum compact_result *compact_result)
3759 {
3760         struct page *page = NULL;
3761         unsigned long pflags;
3762         unsigned int noreclaim_flag;
3763 
3764         if (!order)
3765                 return NULL;
3766 
3767         psi_memstall_enter(&pflags);
3768         noreclaim_flag = memalloc_noreclaim_save();
3769 
3770         *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
3771                                                                 prio, &page);
3772 
3773         memalloc_noreclaim_restore(noreclaim_flag);
3774         psi_memstall_leave(&pflags);
3775 
3776         if (*compact_result <= COMPACT_INACTIVE) {
3777                 WARN_ON_ONCE(page);
3778                 return NULL;
3779         }
3780 
3781         /*
3782          * At least in one zone compaction wasn't deferred or skipped, so let's
3783          * count a compaction stall
3784          */
3785         count_vm_event(COMPACTSTALL);
3786 
3787         /* Prep a captured page if available */
3788         if (page)
3789                 prep_new_page(page, order, gfp_mask, alloc_flags);
3790 
3791         /* Try get a page from the freelist if available */
3792         if (!page)
3793                 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
3794 
3795         if (page) {
3796                 struct zone *zone = page_zone(page);
3797 
3798                 zone->compact_blockskip_flush = false;
3799                 compaction_defer_reset(zone, order, true);
3800                 count_vm_event(COMPACTSUCCESS);
3801                 return page;
3802         }
3803 
3804         /*
3805          * It's bad if compaction run occurs and fails. The most likely reason
3806          * is that pages exist, but not enough to satisfy watermarks.
3807          */
3808         count_vm_event(COMPACTFAIL);
3809 
3810         cond_resched();
3811 
3812         return NULL;
3813 }
3814 
3815 static inline bool
3816 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags,
3817                      enum compact_result compact_result,
3818                      enum compact_priority *compact_priority,
3819                      int *compaction_retries)
3820 {
3821         int max_retries = MAX_COMPACT_RETRIES;
3822         int min_priority;
3823         bool ret = false;
3824         int retries = *compaction_retries;
3825         enum compact_priority priority = *compact_priority;
3826 
3827         if (!order)
3828                 return false;
3829 
3830         if (compaction_made_progress(compact_result))
3831                 (*compaction_retries)++;
3832 
3833         /*
3834          * compaction considers all the zone as desperately out of memory
3835          * so it doesn't really make much sense to retry except when the
3836          * failure could be caused by insufficient priority
3837          */
3838         if (compaction_failed(compact_result))
3839                 goto check_priority;
3840 
3841         /*
3842          * make sure the compaction wasn't deferred or didn't bail out early
3843          * due to locks contention before we declare that we should give up.
3844          * But do not retry if the given zonelist is not suitable for
3845          * compaction.
3846          */
3847         if (compaction_withdrawn(compact_result)) {
3848                 ret = compaction_zonelist_suitable(ac, order, alloc_flags);
3849                 goto out;
3850         }
3851 
3852         /*
3853          * !costly requests are much more important than __GFP_RETRY_MAYFAIL
3854          * costly ones because they are de facto nofail and invoke OOM
3855          * killer to move on while costly can fail and users are ready
3856          * to cope with that. 1/4 retries is rather arbitrary but we
3857          * would need much more detailed feedback from compaction to
3858          * make a better decision.
3859          */
3860         if (order > PAGE_ALLOC_COSTLY_ORDER)
3861                 max_retries /= 4;
3862         if (*compaction_retries <= max_retries) {
3863                 ret = true;
3864                 goto out;
3865         }
3866 
3867         /*
3868          * Make sure there are attempts at the highest priority if we exhausted
3869          * all retries or failed at the lower priorities.
3870          */
3871 check_priority:
3872         min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ?
3873                         MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY;
3874 
3875         if (*compact_priority > min_priority) {
3876                 (*compact_priority)--;
3877                 *compaction_retries = 0;
3878                 ret = true;
3879         }
3880 out:
3881         trace_compact_retry(order, priority, compact_result, retries, max_retries, ret);
3882         return ret;
3883 }
3884 #else
3885 static inline struct page *
3886 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
3887                 unsigned int alloc_flags, const struct alloc_context *ac,
3888                 enum compact_priority prio, enum compact_result *compact_result)
3889 {
3890         *compact_result = COMPACT_SKIPPED;
3891         return NULL;
3892 }
3893 
3894 static inline bool
3895 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags,
3896                      enum compact_result compact_result,
3897                      enum compact_priority *compact_priority,
3898                      int *compaction_retries)
3899 {
3900         struct zone *zone;
3901         struct zoneref *z;
3902 
3903         if (!order || order > PAGE_ALLOC_COSTLY_ORDER)
3904                 return false;
3905 
3906         /*
3907          * There are setups with compaction disabled which would prefer to loop
3908          * inside the allocator rather than hit the oom killer prematurely.
3909          * Let's give them a good hope and keep retrying while the order-0
3910          * watermarks are OK.
3911          */
3912         for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
3913                                         ac->nodemask) {
3914                 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone),
3915                                         ac_classzone_idx(ac), alloc_flags))
3916                         return true;
3917         }
3918         return false;
3919 }
3920 #endif /* CONFIG_COMPACTION */
3921 
3922 #ifdef CONFIG_LOCKDEP
3923 static struct lockdep_map __fs_reclaim_map =
3924         STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map);
3925 
3926 static bool __need_fs_reclaim(gfp_t gfp_mask)
3927 {
3928         gfp_mask = current_gfp_context(gfp_mask);
3929 
3930         /* no reclaim without waiting on it */
3931         if (!(gfp_mask & __GFP_DIRECT_RECLAIM))
3932                 return false;
3933 
3934         /* this guy won't enter reclaim */
3935         if (current->flags & PF_MEMALLOC)
3936                 return false;
3937 
3938         /* We're only interested __GFP_FS allocations for now */
3939         if (!(gfp_mask & __GFP_FS))
3940                 return false;
3941 
3942         if (gfp_mask & __GFP_NOLOCKDEP)
3943                 return false;
3944 
3945         return true;
3946 }
3947 
3948 void __fs_reclaim_acquire(void)
3949 {
3950         lock_map_acquire(&__fs_reclaim_map);
3951 }
3952 
3953 void __fs_reclaim_release(void)
3954 {
3955         lock_map_release(&__fs_reclaim_map);
3956 }
3957 
3958 void fs_reclaim_acquire(gfp_t gfp_mask)
3959 {
3960         if (__need_fs_reclaim(gfp_mask))
3961                 __fs_reclaim_acquire();
3962 }
3963 EXPORT_SYMBOL_GPL(fs_reclaim_acquire);
3964 
3965 void fs_reclaim_release(gfp_t gfp_mask)
3966 {
3967         if (__need_fs_reclaim(gfp_mask))
3968                 __fs_reclaim_release();
3969 }
3970 EXPORT_SYMBOL_GPL(fs_reclaim_release);
3971 #endif
3972 
3973 /* Perform direct synchronous page reclaim */
3974 static int
3975 __perform_reclaim(gfp_t gfp_mask, unsigned int order,
3976                                         const struct alloc_context *ac)
3977 {
3978         struct reclaim_state reclaim_state;
3979         int progress;
3980         unsigned int noreclaim_flag;
3981         unsigned long pflags;
3982 
3983         cond_resched();
3984 
3985         /* We now go into synchronous reclaim */
3986         cpuset_memory_pressure_bump();
3987         psi_memstall_enter(&pflags);
3988         fs_reclaim_acquire(gfp_mask);
3989         noreclaim_flag = memalloc_noreclaim_save();
3990         reclaim_state.reclaimed_slab = 0;
3991         current->reclaim_state = &reclaim_state;
3992 
3993         progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
3994                                                                 ac->nodemask);
3995 
3996         current->reclaim_state = NULL;
3997         memalloc_noreclaim_restore(noreclaim_flag);
3998         fs_reclaim_release(gfp_mask);
3999         psi_memstall_leave(&pflags);
4000 
4001         cond_resched();
4002 
4003         return progress;
4004 }
4005 
4006 /* The really slow allocator path where we enter direct reclaim */
4007 static inline struct page *
4008 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
4009                 unsigned int alloc_flags, const struct alloc_context *ac,
4010                 unsigned long *did_some_progress)
4011 {
4012         struct page *page = NULL;
4013         bool drained = false;
4014 
4015         *did_some_progress = __perform_reclaim(gfp_mask, order, ac);
4016         if (unlikely(!(*did_some_progress)))
4017                 return NULL;
4018 
4019 retry:
4020         page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4021 
4022         /*
4023          * If an allocation failed after direct reclaim, it could be because
4024          * pages are pinned on the per-cpu lists or in high alloc reserves.
4025          * Shrink them them and try again
4026          */
4027         if (!page && !drained) {
4028                 unreserve_highatomic_pageblock(ac, false);
4029                 drain_all_pages(NULL);
4030                 drained = true;
4031                 goto retry;
4032         }
4033 
4034         return page;
4035 }
4036 
4037 static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,
4038                              const struct alloc_context *ac)
4039 {
4040         struct zoneref *z;
4041         struct zone *zone;
4042         pg_data_t *last_pgdat = NULL;
4043         enum zone_type high_zoneidx = ac->high_zoneidx;
4044 
4045         for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, high_zoneidx,
4046                                         ac->nodemask) {
4047                 if (last_pgdat != zone->zone_pgdat)
4048                         wakeup_kswapd(zone, gfp_mask, order, high_zoneidx);
4049                 last_pgdat = zone->zone_pgdat;
4050         }
4051 }
4052 
4053 static inline unsigned int
4054 gfp_to_alloc_flags(gfp_t gfp_mask)
4055 {
4056         unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
4057 
4058         /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
4059         BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
4060 
4061         /*
4062          * The caller may dip into page reserves a bit more if the caller
4063          * cannot run direct reclaim, or if the caller has realtime scheduling
4064          * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will
4065          * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH).
4066          */
4067         alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH);
4068 
4069         if (gfp_mask & __GFP_ATOMIC) {
4070                 /*
4071                  * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
4072                  * if it can't schedule.
4073                  */
4074                 if (!(gfp_mask & __GFP_NOMEMALLOC))
4075                         alloc_flags |= ALLOC_HARDER;
4076                 /*
4077                  * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
4078                  * comment for __cpuset_node_allowed().
4079                  */
4080                 alloc_flags &= ~ALLOC_CPUSET;
4081         } else if (unlikely(rt_task(current)) && !in_interrupt())
4082                 alloc_flags |= ALLOC_HARDER;
4083 
4084         if (gfp_mask & __GFP_KSWAPD_RECLAIM)
4085                 alloc_flags |= ALLOC_KSWAPD;
4086 
4087 #ifdef CONFIG_CMA
4088         if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
4089                 alloc_flags |= ALLOC_CMA;
4090 #endif
4091         return alloc_flags;
4092 }
4093 
4094 static bool oom_reserves_allowed(struct task_struct *tsk)
4095 {
4096         if (!tsk_is_oom_victim(tsk))
4097                 return false;
4098 
4099         /*
4100          * !MMU doesn't have oom reaper so give access to memory reserves
4101          * only to the thread with TIF_MEMDIE set
4102          */
4103         if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE))
4104                 return false;
4105 
4106         return true;
4107 }
4108 
4109 /*
4110  * Distinguish requests which really need access to full memory
4111  * reserves from oom victims which can live with a portion of it
4112  */
4113 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask)
4114 {
4115         if (unlikely(gfp_mask & __GFP_NOMEMALLOC))
4116                 return 0;
4117         if (gfp_mask & __GFP_MEMALLOC)
4118                 return ALLOC_NO_WATERMARKS;
4119         if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
4120                 return ALLOC_NO_WATERMARKS;
4121         if (!in_interrupt()) {
4122                 if (current->flags & PF_MEMALLOC)
4123                         return ALLOC_NO_WATERMARKS;
4124                 else if (oom_reserves_allowed(current))
4125                         return ALLOC_OOM;
4126         }
4127 
4128         return 0;
4129 }
4130 
4131 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
4132 {
4133         return !!__gfp_pfmemalloc_flags(gfp_mask);
4134 }
4135 
4136 /*
4137  * Checks whether it makes sense to retry the reclaim to make a forward progress
4138  * for the given allocation request.
4139  *
4140  * We give up when we either have tried MAX_RECLAIM_RETRIES in a row
4141  * without success, or when we couldn't even meet the watermark if we
4142  * reclaimed all remaining pages on the LRU lists.
4143  *
4144  * Returns true if a retry is viable or false to enter the oom path.
4145  */
4146 static inline bool
4147 should_reclaim_retry(gfp_t gfp_mask, unsigned order,
4148                      struct alloc_context *ac, int alloc_flags,
4149                      bool did_some_progress, int *no_progress_loops)
4150 {
4151         struct zone *zone;
4152         struct zoneref *z;
4153         bool ret = false;
4154 
4155         /*
4156          * Costly allocations might have made a progress but this doesn't mean
4157          * their order will become available due to high fragmentation so
4158          * always increment the no progress counter for them
4159          */
4160         if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
4161                 *no_progress_loops = 0;
4162         else
4163                 (*no_progress_loops)++;
4164 
4165         /*
4166          * Make sure we converge to OOM if we cannot make any progress
4167          * several times in the row.
4168          */
4169         if (*no_progress_loops > MAX_RECLAIM_RETRIES) {
4170                 /* Before OOM, exhaust highatomic_reserve */
4171                 return unreserve_highatomic_pageblock(ac, true);
4172         }
4173 
4174         /*
4175          * Keep reclaiming pages while there is a chance this will lead
4176          * somewhere.  If none of the target zones can satisfy our allocation
4177          * request even if all reclaimable pages are considered then we are
4178          * screwed and have to go OOM.
4179          */
4180         for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
4181                                         ac->nodemask) {
4182                 unsigned long available;
4183                 unsigned long reclaimable;
4184                 unsigned long min_wmark = min_wmark_pages(zone);
4185                 bool wmark;
4186 
4187                 available = reclaimable = zone_reclaimable_pages(zone);
4188                 available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
4189 
4190                 /*
4191                  * Would the allocation succeed if we reclaimed all
4192                  * reclaimable pages?
4193                  */
4194                 wmark = __zone_watermark_ok(zone, order, min_wmark,
4195                                 ac_classzone_idx(ac), alloc_flags, available);
4196                 trace_reclaim_retry_zone(z, order, reclaimable,
4197                                 available, min_wmark, *no_progress_loops, wmark);
4198                 if (wmark) {
4199                         /*
4200                          * If we didn't make any progress and have a lot of
4201                          * dirty + writeback pages then we should wait for
4202                          * an IO to complete to slow down the reclaim and
4203                          * prevent from pre mature OOM
4204                          */
4205                         if (!did_some_progress) {
4206                                 unsigned long write_pending;
4207 
4208                                 write_pending = zone_page_state_snapshot(zone,
4209                                                         NR_ZONE_WRITE_PENDING);
4210 
4211                                 if (2 * write_pending > reclaimable) {
4212                                         congestion_wait(BLK_RW_ASYNC, HZ/10);
4213                                         return true;
4214                                 }
4215                         }
4216 
4217                         ret = true;
4218                         goto out;
4219                 }
4220         }
4221 
4222 out:
4223         /*
4224          * Memory allocation/reclaim might be called from a WQ context and the
4225          * current implementation of the WQ concurrency control doesn't
4226          * recognize that a particular WQ is congested if the worker thread is
4227          * looping without ever sleeping. Therefore we have to do a short sleep
4228          * here rather than calling cond_resched().
4229          */
4230         if (current->flags & PF_WQ_WORKER)
4231                 schedule_timeout_uninterruptible(1);
4232         else
4233                 cond_resched();
4234         return ret;
4235 }
4236 
4237 static inline bool
4238 check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac)
4239 {
4240         /*
4241          * It's possible that cpuset's mems_allowed and the nodemask from
4242          * mempolicy don't intersect. This should be normally dealt with by
4243          * policy_nodemask(), but it's possible to race with cpuset update in
4244          * such a way the check therein was true, and then it became false
4245          * before we got our cpuset_mems_cookie here.
4246          * This assumes that for all allocations, ac->nodemask can come only
4247          * from MPOL_BIND mempolicy (whose documented semantics is to be ignored
4248          * when it does not intersect with the cpuset restrictions) or the
4249          * caller can deal with a violated nodemask.
4250          */
4251         if (cpusets_enabled() && ac->nodemask &&
4252                         !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) {
4253                 ac->nodemask = NULL;
4254                 return true;
4255         }
4256 
4257         /*
4258          * When updating a task's mems_allowed or mempolicy nodemask, it is
4259          * possible to race with parallel threads in such a way that our
4260          * allocation can fail while the mask is being updated. If we are about
4261          * to fail, check if the cpuset changed during allocation and if so,
4262          * retry.
4263          */
4264         if (read_mems_allowed_retry(cpuset_mems_cookie))
4265                 return true;
4266 
4267         return false;
4268 }
4269 
4270 static inline struct page *
4271 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
4272                                                 struct alloc_context *ac)
4273 {
4274         bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
4275         const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER;
4276         struct page *page = NULL;
4277         unsigned int alloc_flags;
4278         unsigned long did_some_progress;
4279         enum compact_priority compact_priority;
4280         enum compact_result compact_result;
4281         int compaction_retries;
4282         int no_progress_loops;
4283         unsigned int cpuset_mems_cookie;
4284         int reserve_flags;
4285 
4286         /*
4287          * We also sanity check to catch abuse of atomic reserves being used by
4288          * callers that are not in atomic context.
4289          */
4290         if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) ==
4291                                 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
4292                 gfp_mask &= ~__GFP_ATOMIC;
4293 
4294 retry_cpuset:
4295         compaction_retries = 0;
4296         no_progress_loops = 0;
4297         compact_priority = DEF_COMPACT_PRIORITY;
4298         cpuset_mems_cookie = read_mems_allowed_begin();
4299 
4300         /*
4301          * The fast path uses conservative alloc_flags to succeed only until
4302          * kswapd needs to be woken up, and to avoid the cost of setting up
4303          * alloc_flags precisely. So we do that now.
4304          */
4305         alloc_flags = gfp_to_alloc_flags(gfp_mask);
4306 
4307         /*
4308          * We need to recalculate the starting point for the zonelist iterator
4309          * because we might have used different nodemask in the fast path, or
4310          * there was a cpuset modification and we are retrying - otherwise we
4311          * could end up iterating over non-eligible zones endlessly.
4312          */
4313         ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
4314                                         ac->high_zoneidx, ac->nodemask);
4315         if (!ac->preferred_zoneref->zone)
4316                 goto nopage;
4317 
4318         if (alloc_flags & ALLOC_KSWAPD)
4319                 wake_all_kswapds(order, gfp_mask, ac);
4320 
4321         /*
4322          * The adjusted alloc_flags might result in immediate success, so try
4323          * that first
4324          */
4325         page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4326         if (page)
4327                 goto got_pg;
4328 
4329         /*
4330          * For costly allocations, try direct compaction first, as it's likely
4331          * that we have enough base pages and don't need to reclaim. For non-
4332          * movable high-order allocations, do that as well, as compaction will
4333          * try prevent permanent fragmentation by migrating from blocks of the
4334          * same migratetype.
4335          * Don't try this for allocations that are allowed to ignore
4336          * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen.
4337          */
4338         if (can_direct_reclaim &&
4339                         (costly_order ||
4340                            (order > 0 && ac->migratetype != MIGRATE_MOVABLE))
4341                         && !gfp_pfmemalloc_allowed(gfp_mask)) {
4342                 page = __alloc_pages_direct_compact(gfp_mask, order,
4343                                                 alloc_flags, ac,
4344                                                 INIT_COMPACT_PRIORITY,
4345                                                 &compact_result);
4346                 if (page)
4347                         goto got_pg;
4348 
4349                 /*
4350                  * Checks for costly allocations with __GFP_NORETRY, which
4351                  * includes THP page fault allocations
4352                  */
4353                 if (costly_order && (gfp_mask & __GFP_NORETRY)) {
4354                         /*
4355                          * If compaction is deferred for high-order allocations,
4356                          * it is because sync compaction recently failed. If
4357                          * this is the case and the caller requested a THP
4358                          * allocation, we do not want to heavily disrupt the
4359                          * system, so we fail the allocation instead of entering
4360                          * direct reclaim.
4361                          */
4362                         if (compact_result == COMPACT_DEFERRED)
4363                                 goto nopage;
4364 
4365                         /*
4366                          * Looks like reclaim/compaction is worth trying, but
4367                          * sync compaction could be very expensive, so keep
4368                          * using async compaction.
4369                          */
4370                         compact_priority = INIT_COMPACT_PRIORITY;
4371                 }
4372         }
4373 
4374 retry:
4375         /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */
4376         if (alloc_flags & ALLOC_KSWAPD)
4377                 wake_all_kswapds(order, gfp_mask, ac);
4378 
4379         reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);
4380         if (reserve_flags)
4381                 alloc_flags = reserve_flags;
4382 
4383         /*
4384          * Reset the nodemask and zonelist iterators if memory policies can be
4385          * ignored. These allocations are high priority and system rather than
4386          * user oriented.
4387          */
4388         if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) {
4389                 ac->nodemask = NULL;
4390                 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
4391                                         ac->high_zoneidx, ac->nodemask);
4392         }
4393 
4394         /* Attempt with potentially adjusted zonelist and alloc_flags */
4395         page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4396         if (page)
4397                 goto got_pg;
4398 
4399         /* Caller is not willing to reclaim, we can't balance anything */
4400         if (!can_direct_reclaim)
4401                 goto nopage;
4402 
4403         /* Avoid recursion of direct reclaim */
4404         if (current->flags & PF_MEMALLOC)
4405                 goto nopage;
4406 
4407         /* Try direct reclaim and then allocating */
4408         page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
4409                                                         &did_some_progress);
4410         if (page)
4411                 goto got_pg;
4412 
4413         /* Try direct compaction and then allocating */
4414         page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
4415                                         compact_priority, &compact_result);
4416         if (page)
4417                 goto got_pg;
4418 
4419         /* Do not loop if specifically requested */
4420         if (gfp_mask & __GFP_NORETRY)
4421                 goto nopage;
4422 
4423         /*
4424          * Do not retry costly high order allocations unless they are
4425          * __GFP_RETRY_MAYFAIL
4426          */
4427         if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL))
4428                 goto nopage;
4429 
4430         if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
4431                                  did_some_progress > 0, &no_progress_loops))
4432                 goto retry;
4433 
4434         /*
4435          * It doesn't make any sense to retry for the compaction if the order-0
4436          * reclaim is not able to make any progress because the current
4437          * implementation of the compaction depends on the sufficient amount
4438          * of free memory (see __compaction_suitable)
4439          */
4440         if (did_some_progress > 0 &&
4441                         should_compact_retry(ac, order, alloc_flags,
4442                                 compact_result, &compact_priority,
4443                                 &compaction_retries))
4444                 goto retry;
4445 
4446 
4447         /* Deal with possible cpuset update races before we start OOM killing */
4448         if (check_retry_cpuset(cpuset_mems_cookie, ac))
4449                 goto retry_cpuset;
4450 
4451         /* Reclaim has failed us, start killing things */
4452         page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
4453         if (page)
4454                 goto got_pg;
4455 
4456         /* Avoid allocations with no watermarks from looping endlessly */
4457         if (tsk_is_oom_victim(current) &&
4458             (alloc_flags == ALLOC_OOM ||
4459              (gfp_mask & __GFP_NOMEMALLOC)))
4460                 goto nopage;
4461 
4462         /* Retry as long as the OOM killer is making progress */
4463         if (did_some_progress) {
4464                 no_progress_loops = 0;
4465                 goto retry;
4466         }
4467 
4468 nopage:
4469         /* Deal with possible cpuset update races before we fail */
4470         if (check_retry_cpuset(cpuset_mems_cookie, ac))
4471                 goto retry_cpuset;
4472 
4473         /*
4474          * Make sure that __GFP_NOFAIL request doesn't leak out and make sure
4475          * we always retry
4476          */
4477         if (gfp_mask & __GFP_NOFAIL) {
4478                 /*
4479                  * All existing users of the __GFP_NOFAIL are blockable, so warn
4480                  * of any new users that actually require GFP_NOWAIT
4481                  */
4482                 if (WARN_ON_ONCE(!can_direct_reclaim))
4483                         goto fail;
4484 
4485                 /*
4486                  * PF_MEMALLOC request from this context is rather bizarre
4487                  * because we cannot reclaim anything and only can loop waiting
4488                  * for somebody to do a work for us
4489                  */
4490                 WARN_ON_ONCE(current->flags & PF_MEMALLOC);
4491 
4492                 /*
4493                  * non failing costly orders are a hard requirement which we
4494                  * are not prepared for much so let's warn about these users
4495                  * so that we can identify them and convert them to something
4496                  * else.
4497                  */
4498                 WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER);
4499 
4500                 /*
4501                  * Help non-failing allocations by giving them access to memory
4502                  * reserves but do not use ALLOC_NO_WATERMARKS because this
4503                  * could deplete whole memory reserves which would just make
4504                  * the situation worse
4505                  */
4506                 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac);
4507                 if (page)
4508                         goto got_pg;
4509 
4510                 cond_resched();
4511                 goto retry;
4512         }
4513 fail:
4514         warn_alloc(gfp_mask, ac->nodemask,
4515                         "page allocation failure: order:%u", order);
4516 got_pg:
4517         return page;
4518 }
4519 
4520 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
4521                 int preferred_nid, nodemask_t *nodemask,
4522                 struct alloc_context *ac, gfp_t *alloc_mask,
4523                 unsigned int *alloc_flags)
4524 {
4525         ac->high_zoneidx = gfp_zone(gfp_mask);
4526         ac->zonelist = node_zonelist(preferred_nid, gfp_mask);
4527         ac->nodemask = nodemask;
4528         ac->migratetype = gfpflags_to_migratetype(gfp_mask);
4529 
4530         if (cpusets_enabled()) {
4531                 *alloc_mask |= __GFP_HARDWALL;
4532                 if (!ac->nodemask)
4533                         ac->nodemask = &cpuset_current_mems_allowed;
4534                 else
4535                         *alloc_flags |= ALLOC_CPUSET;
4536         }
4537 
4538         fs_reclaim_acquire(gfp_mask);
4539         fs_reclaim_release(gfp_mask);
4540 
4541         might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);
4542 
4543         if (should_fail_alloc_page(gfp_mask, order))
4544                 return false;
4545 
4546         if (IS_ENABLED(CONFIG_CMA) && ac->migratetype == MIGRATE_MOVABLE)
4547                 *alloc_flags |= ALLOC_CMA;
4548 
4549         return true;
4550 }
4551 
4552 /* Determine whether to spread dirty pages and what the first usable zone */
4553 static inline void finalise_ac(gfp_t gfp_mask, struct alloc_context *ac)
4554 {
4555         /* Dirty zone balancing only done in the fast path */
4556         ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE);
4557 
4558         /*
4559          * The preferred zone is used for statistics but crucially it is
4560          * also used as the starting point for the zonelist iterator. It
4561          * may get reset for allocations that ignore memory policies.
4562          */
4563         ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
4564                                         ac->high_zoneidx, ac->nodemask);
4565 }
4566 
4567 /*
4568  * This is the 'heart' of the zoned buddy allocator.
4569  */
4570 struct page *
4571 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid,
4572                                                         nodemask_t *nodemask)
4573 {
4574         struct page *page;
4575         unsigned int alloc_flags = ALLOC_WMARK_LOW;
4576         gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */
4577         struct alloc_context ac = { };
4578 
4579         /*
4580          * There are several places where we assume that the order value is sane
4581          * so bail out early if the request is out of bound.
4582          */
4583         if (unlikely(order >= MAX_ORDER)) {
4584                 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
4585                 return NULL;
4586         }
4587 
4588         gfp_mask &= gfp_allowed_mask;
4589         alloc_mask = gfp_mask;
4590         if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags))
4591                 return NULL;
4592 
4593         finalise_ac(gfp_mask, &ac);
4594 
4595         /*
4596          * Forbid the first pass from falling back to types that fragment
4597          * memory until all local zones are considered.
4598          */
4599         alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp_mask);
4600 
4601         /* First allocation attempt */
4602         page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac);
4603         if (likely(page))
4604                 goto out;
4605 
4606         /*
4607          * Apply scoped allocation constraints. This is mainly about GFP_NOFS
4608          * resp. GFP_NOIO which has to be inherited for all allocation requests
4609          * from a particular context which has been marked by
4610          * memalloc_no{fs,io}_{save,restore}.
4611          */
4612         alloc_mask = current_gfp_context(gfp_mask);
4613         ac.spread_dirty_pages = false;
4614 
4615         /*
4616          * Restore the original nodemask if it was potentially replaced with
4617          * &cpuset_current_mems_allowed to optimize the fast-path attempt.
4618          */
4619         if (unlikely(ac.nodemask != nodemask))
4620                 ac.nodemask = nodemask;
4621 
4622         page = __alloc_pages_slowpath(alloc_mask, order, &ac);
4623 
4624 out:
4625         if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page &&
4626             unlikely(__memcg_kmem_charge(page, gfp_mask, order) != 0)) {
4627                 __free_pages(page, order);
4628                 page = NULL;
4629         }
4630 
4631         trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);
4632 
4633         return page;
4634 }
4635 EXPORT_SYMBOL(__alloc_pages_nodemask);
4636 
4637 /*
4638  * Common helper functions. Never use with __GFP_HIGHMEM because the returned
4639  * address cannot represent highmem pages. Use alloc_pages and then kmap if
4640  * you need to access high mem.
4641  */
4642 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
4643 {
4644         struct page *page;
4645 
4646         page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order);
4647         if (!page)
4648                 return 0;
4649         return (unsigned long) page_address(page);
4650 }
4651 EXPORT_SYMBOL(__get_free_pages);
4652 
4653 unsigned long get_zeroed_page(gfp_t gfp_mask)
4654 {
4655         return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
4656 }
4657 EXPORT_SYMBOL(get_zeroed_page);
4658 
4659 static inline void free_the_page(struct page *page, unsigned int order)
4660 {
4661         if (order == 0)         /* Via pcp? */
4662                 free_unref_page(page);
4663         else
4664                 __free_pages_ok(page, order);
4665 }
4666 
4667 void __free_pages(struct page *page, unsigned int order)
4668 {
4669         if (put_page_testzero(page))
4670                 free_the_page(page, order);
4671 }
4672 EXPORT_SYMBOL(__free_pages);
4673 
4674 void free_pages(unsigned long addr, unsigned int order)
4675 {
4676         if (addr != 0) {
4677                 VM_BUG_ON(!virt_addr_valid((void *)addr));
4678                 __free_pages(virt_to_page((void *)addr), order);
4679         }
4680 }
4681 
4682 EXPORT_SYMBOL(free_pages);
4683 
4684 /*
4685  * Page Fragment:
4686  *  An arbitrary-length arbitrary-offset area of memory which resides
4687  *  within a 0 or higher order page.  Multiple fragments within that page
4688  *  are individually refcounted, in the page's reference counter.
4689  *
4690  * The page_frag functions below provide a simple allocation framework for
4691  * page fragments.  This is used by the network stack and network device
4692  * drivers to provide a backing region of memory for use as either an
4693  * sk_buff->head, or to be used in the "frags" portion of skb_shared_info.
4694  */
4695 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
4696                                              gfp_t gfp_mask)
4697 {
4698         struct page *page = NULL;
4699         gfp_t gfp = gfp_mask;
4700 
4701 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
4702         gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
4703                     __GFP_NOMEMALLOC;
4704         page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
4705                                 PAGE_FRAG_CACHE_MAX_ORDER);
4706         nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE;
4707 #endif
4708         if (unlikely(!page))
4709                 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
4710 
4711         nc->va = page ? page_address(page) : NULL;
4712 
4713         return page;
4714 }
4715 
4716 void __page_frag_cache_drain(struct page *page, unsigned int count)
4717 {
4718         VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
4719 
4720         if (page_ref_sub_and_test(page, count))
4721                 free_the_page(page, compound_order(page));
4722 }
4723 EXPORT_SYMBOL(__page_frag_cache_drain);
4724 
4725 void *page_frag_alloc(struct page_frag_cache *nc,
4726                       unsigned int fragsz, gfp_t gfp_mask)
4727 {
4728         unsigned int size = PAGE_SIZE;
4729         struct page *page;
4730         int offset;
4731 
4732         if (unlikely(!nc->va)) {
4733 refill:
4734                 page = __page_frag_cache_refill(nc, gfp_mask);
4735                 if (!page)
4736                         return NULL;
4737 
4738 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
4739                 /* if size can vary use size else just use PAGE_SIZE */
4740                 size = nc->size;
4741 #endif
4742                 /* Even if we own the page, we do not use atomic_set().
4743                  * This would break get_page_unless_zero() users.
4744                  */
4745                 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE);
4746 
4747                 /* reset page count bias and offset to start of new frag */
4748                 nc->pfmemalloc = page_is_pfmemalloc(page);
4749                 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
4750                 nc->offset = size;
4751         }
4752 
4753         offset = nc->offset - fragsz;
4754         if (unlikely(offset < 0)) {
4755                 page = virt_to_page(nc->va);
4756 
4757                 if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
4758                         goto refill;
4759 
4760 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
4761                 /* if size can vary use size else just use PAGE_SIZE */
4762                 size = nc->size;
4763 #endif
4764                 /* OK, page count is 0, we can safely set it */
4765                 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);
4766 
4767                 /* reset page count bias and offset to start of new frag */
4768                 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
4769                 offset = size - fragsz;
4770         }
4771 
4772         nc->pagecnt_bias--;
4773         nc->offset = offset;
4774 
4775         return nc->va + offset;
4776 }
4777 EXPORT_SYMBOL(page_frag_alloc);
4778 
4779 /*
4780  * Frees a page fragment allocated out of either a compound or order 0 page.
4781  */
4782 void page_frag_free(void *addr)
4783 {
4784         struct page *page = virt_to_head_page(addr);
4785 
4786         if (unlikely(put_page_testzero(page)))
4787                 free_the_page(page, compound_order(page));
4788 }
4789 EXPORT_SYMBOL(page_frag_free);
4790 
4791 static void *make_alloc_exact(unsigned long addr, unsigned int order,
4792                 size_t size)
4793 {
4794         if (addr) {
4795                 unsigned long alloc_end = addr + (PAGE_SIZE << order);
4796                 unsigned long used = addr + PAGE_ALIGN(size);
4797 
4798                 split_page(virt_to_page((void *)addr), order);
4799                 while (used < alloc_end) {
4800                         free_page(used);
4801                         used += PAGE_SIZE;
4802                 }
4803         }
4804         return (void *)addr;
4805 }
4806 
4807 /**
4808  * alloc_pages_exact - allocate an exact number physically-contiguous pages.
4809  * @size: the number of bytes to allocate
4810  * @gfp_mask: GFP flags for the allocation
4811  *
4812  * This function is similar to alloc_pages(), except that it allocates the
4813  * minimum number of pages to satisfy the request.  alloc_pages() can only
4814  * allocate memory in power-of-two pages.
4815  *
4816  * This function is also limited by MAX_ORDER.
4817  *
4818  * Memory allocated by this function must be released by free_pages_exact().
4819  *
4820  * Return: pointer to the allocated area or %NULL in case of error.
4821  */
4822 void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
4823 {
4824         unsigned int order = get_order(size);
4825         unsigned long addr;
4826 
4827         addr = __get_free_pages(gfp_mask, order);
4828         return make_alloc_exact(addr, order, size);
4829 }
4830 EXPORT_SYMBOL(alloc_pages_exact);
4831 
4832 /**
4833  * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
4834  *                         pages on a node.
4835  * @nid: the preferred node ID where memory should be allocated
4836  * @size: the number of bytes to allocate
4837  * @gfp_mask: GFP flags for the allocation
4838  *
4839  * Like alloc_pages_exact(), but try to allocate on node nid first before falling
4840  * back.
4841  *
4842  * Return: pointer to the allocated area or %NULL in case of error.
4843  */
4844 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
4845 {
4846         unsigned int order = get_order(size);
4847         struct page *p = alloc_pages_node(nid, gfp_mask, order);
4848         if (!p)
4849                 return NULL;
4850         return make_alloc_exact((unsigned long)page_address(p), order, size);
4851 }
4852 
4853 /**
4854  * free_pages_exact - release memory allocated via alloc_pages_exact()
4855  * @virt: the value returned by alloc_pages_exact.
4856  * @size: size of allocation, same value as passed to alloc_pages_exact().
4857  *
4858  * Release the memory allocated by a previous call to alloc_pages_exact.
4859  */
4860 void free_pages_exact(void *virt, size_t size)
4861 {
4862         unsigned long addr = (unsigned long)virt;
4863         unsigned long end = addr + PAGE_ALIGN(size);
4864 
4865         while (addr < end) {
4866                 free_page(addr);
4867                 addr += PAGE_SIZE;
4868         }
4869 }
4870 EXPORT_SYMBOL(free_pages_exact);
4871 
4872 /**
4873  * nr_free_zone_pages - count number of pages beyond high watermark
4874  * @offset: The zone index of the highest zone
4875  *
4876  * nr_free_zone_pages() counts the number of pages which are beyond the
4877  * high watermark within all zones at or below a given zone index.  For each
4878  * zone, the number of pages is calculated as:
4879  *
4880  *     nr_free_zone_pages = managed_pages - high_pages
4881  *
4882  * Return: number of pages beyond high watermark.
4883  */
4884 static unsigned long nr_free_zone_pages(int offset)
4885 {
4886         struct zoneref *z;
4887         struct zone *zone;
4888 
4889         /* Just pick one node, since fallback list is circular */
4890         unsigned long sum = 0;
4891 
4892         struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
4893 
4894         for_each_zone_zonelist(zone, z, zonelist, offset) {
4895                 unsigned long size = zone_managed_pages(zone);
4896                 unsigned long high = high_wmark_pages(zone);
4897                 if (size > high)
4898                         sum += size - high;
4899         }
4900 
4901         return sum;
4902 }
4903 
4904 /**
4905  * nr_free_buffer_pages - count number of pages beyond high watermark
4906  *
4907  * nr_free_buffer_pages() counts the number of pages which are beyond the high
4908  * watermark within ZONE_DMA and ZONE_NORMAL.
4909  *
4910  * Return: number of pages beyond high watermark within ZONE_DMA and
4911  * ZONE_NORMAL.
4912  */
4913 unsigned long nr_free_buffer_pages(void)
4914 {
4915         return nr_free_zone_pages(gfp_zone(GFP_USER));
4916 }
4917 EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
4918 
4919 /**
4920  * nr_free_pagecache_pages - count number of pages beyond high watermark
4921  *
4922  * nr_free_pagecache_pages() counts the number of pages which are beyond the
4923  * high watermark within all zones.
4924  *
4925  * Return: number of pages beyond high watermark within all zones.
4926  */
4927 unsigned long nr_free_pagecache_pages(void)
4928 {
4929         return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
4930 }
4931 
4932 static inline void show_node(struct zone *zone)
4933 {
4934         if (IS_ENABLED(CONFIG_NUMA))
4935                 printk("Node %d ", zone_to_nid(zone));
4936 }
4937 
4938 long si_mem_available(void)
4939 {
4940         long available;
4941         unsigned long pagecache;
4942         unsigned long wmark_low = 0;
4943         unsigned long pages[NR_LRU_LISTS];
4944         unsigned long reclaimable;
4945         struct zone *zone;
4946         int lru;
4947 
4948         for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
4949                 pages[lru] = global_node_page_state(NR_LRU_BASE + lru);
4950 
4951         for_each_zone(zone)
4952                 wmark_low += low_wmark_pages(zone);
4953 
4954         /*
4955          * Estimate the amount of memory available for userspace allocations,
4956          * without causing swapping.
4957          */
4958         available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages;
4959 
4960         /*
4961          * Not all the page cache can be freed, otherwise the system will
4962          * start swapping. Assume at least half of the page cache, or the
4963          * low watermark worth of cache, needs to stay.
4964          */
4965         pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE];
4966         pagecache -= min(pagecache / 2, wmark_low);
4967         available += pagecache;
4968 
4969         /*
4970          * Part of the reclaimable slab and other kernel memory consists of
4971          * items that are in use, and cannot be freed. Cap this estimate at the
4972          * low watermark.
4973          */
4974         reclaimable = global_node_page_state(NR_SLAB_RECLAIMABLE) +
4975                         global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE);
4976         available += reclaimable - min(reclaimable / 2, wmark_low);
4977 
4978         if (available < 0)
4979                 available = 0;
4980         return available;
4981 }
4982 EXPORT_SYMBOL_GPL(si_mem_available);
4983 
4984 void si_meminfo(struct sysinfo *val)
4985 {
4986         val->totalram = totalram_pages();
4987         val->sharedram = global_node_page_state(NR_SHMEM);
4988         val->freeram = global_zone_page_state(NR_FREE_PAGES);
4989         val->bufferram = nr_blockdev_pages();
4990         val->totalhigh = totalhigh_pages();
4991         val->freehigh = nr_free_highpages();
4992         val->mem_unit = PAGE_SIZE;
4993 }
4994 
4995 EXPORT_SYMBOL(si_meminfo);
4996 
4997 #ifdef CONFIG_NUMA
4998 void si_meminfo_node(struct sysinfo *val, int nid)
4999 {
5000         int zone_type;          /* needs to be signed */
5001         unsigned long managed_pages = 0;
5002         unsigned long managed_highpages = 0;
5003         unsigned long free_highpages = 0;
5004         pg_data_t *pgdat = NODE_DATA(nid);
5005 
5006         for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
5007                 managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]);
5008         val->totalram = managed_pages;
5009         val->sharedram = node_page_state(pgdat, NR_SHMEM);
5010         val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES);
5011 #ifdef CONFIG_HIGHMEM
5012         for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
5013                 struct zone *zone = &pgdat->node_zones[zone_type];
5014 
5015                 if (is_highmem(zone)) {
5016                         managed_highpages += zone_managed_pages(zone);
5017                         free_highpages += zone_page_state(zone, NR_FREE_PAGES);
5018                 }
5019         }
5020         val->totalhigh = managed_highpages;
5021         val->freehigh = free_highpages;
5022 #else
5023         val->totalhigh = managed_highpages;
5024         val->freehigh = free_highpages;
5025 #endif
5026         val->mem_unit = PAGE_SIZE;
5027 }
5028 #endif
5029 
5030 /*
5031  * Determine whether the node should be displayed or not, depending on whether
5032  * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
5033  */
5034 static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask)
5035 {
5036         if (!(flags & SHOW_MEM_FILTER_NODES))
5037                 return false;
5038 
5039         /*
5040          * no node mask - aka implicit memory numa policy. Do not bother with
5041          * the synchronization - read_mems_allowed_begin - because we do not
5042          * have to be precise here.
5043          */
5044         if (!nodemask)
5045                 nodemask = &cpuset_current_mems_allowed;
5046 
5047         return !node_isset(nid, *nodemask);
5048 }
5049 
5050 #define K(x) ((x) << (PAGE_SHIFT-10))
5051 
5052 static void show_migration_types(unsigned char type)
5053 {
5054         static const char types[MIGRATE_TYPES] = {
5055                 [MIGRATE_UNMOVABLE]     = 'U',
5056                 [MIGRATE_MOVABLE]       = 'M',
5057                 [MIGRATE_RECLAIMABLE]   = 'E',
5058                 [MIGRATE_HIGHATOMIC]    = 'H',
5059 #ifdef CONFIG_CMA
5060                 [MIGRATE_CMA]           = 'C',
5061 #endif
5062 #ifdef CONFIG_MEMORY_ISOLATION
5063                 [MIGRATE_ISOLATE]       = 'I',
5064 #endif
5065         };
5066         char tmp[MIGRATE_TYPES + 1];
5067         char *p = tmp;
5068         int i;
5069 
5070         for (i = 0; i < MIGRATE_TYPES; i++) {
5071                 if (type & (1 << i))
5072                         *p++ = types[i];
5073         }
5074 
5075         *p = '\0';
5076         printk(KERN_CONT "(%s) ", tmp);
5077 }
5078 
5079 /*
5080  * Show free area list (used inside shift_scroll-lock stuff)
5081  * We also calculate the percentage fragmentation. We do this by counting the
5082  * memory on each free list with the exception of the first item on the list.
5083  *
5084  * Bits in @filter:
5085  * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's
5086  *   cpuset.
5087  */
5088 void show_free_areas(unsigned int filter, nodemask_t *nodemask)
5089 {
5090         unsigned long free_pcp = 0;
5091         int cpu;
5092         struct zone *zone;
5093         pg_data_t *pgdat;
5094 
5095         for_each_populated_zone(zone) {
5096                 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
5097                         continue;
5098 
5099                 for_each_online_cpu(cpu)
5100                         free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
5101         }
5102 
5103         printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
5104                 " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
5105                 " unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n"
5106                 " slab_reclaimable:%lu slab_unreclaimable:%lu\n"
5107                 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
5108                 " free:%lu free_pcp:%lu free_cma:%lu\n",
5109                 global_node_page_state(NR_ACTIVE_ANON),
5110                 global_node_page_state(NR_INACTIVE_ANON),
5111                 global_node_page_state(NR_ISOLATED_ANON),
5112                 global_node_page_state(NR_ACTIVE_FILE),
5113                 global_node_page_state(NR_INACTIVE_FILE),
5114                 global_node_page_state(NR_ISOLATED_FILE),
5115                 global_node_page_state(NR_UNEVICTABLE),
5116                 global_node_page_state(NR_FILE_DIRTY),
5117                 global_node_page_state(NR_WRITEBACK),
5118                 global_node_page_state(NR_UNSTABLE_NFS),
5119                 global_node_page_state(NR_SLAB_RECLAIMABLE),
5120                 global_node_page_state(NR_SLAB_UNRECLAIMABLE),
5121                 global_node_page_state(NR_FILE_MAPPED),
5122                 global_node_page_state(NR_SHMEM),
5123                 global_zone_page_state(NR_PAGETABLE),
5124                 global_zone_page_state(NR_BOUNCE),
5125                 global_zone_page_state(NR_FREE_PAGES),
5126                 free_pcp,
5127                 global_zone_page_state(NR_FREE_CMA_PAGES));
5128 
5129         for_each_online_pgdat(pgdat) {
5130                 if (show_mem_node_skip(filter, pgdat->node_id, nodemask))
5131                         continue;
5132 
5133                 printk("Node %d"
5134                         " active_anon:%lukB"
5135                         " inactive_anon:%lukB"
5136                         " active_file:%lukB"
5137                         " inactive_file:%lukB"
5138                         " unevictable:%lukB"
5139                         " isolated(anon):%lukB"
5140                         " isolated(file):%lukB"
5141                         " mapped:%lukB"
5142                         " dirty:%lukB"
5143                         " writeback:%lukB"
5144                         " shmem:%lukB"
5145 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
5146                         " shmem_thp: %lukB"
5147                         " shmem_pmdmapped: %lukB"
5148                         " anon_thp: %lukB"
5149 #endif
5150                         " writeback_tmp:%lukB"
5151                         " unstable:%lukB"
5152                         " all_unreclaimable? %s"
5153                         "\n",
5154                         pgdat->node_id,
5155                         K(node_page_state(pgdat, NR_ACTIVE_ANON)),
5156                         K(node_page_state(pgdat, NR_INACTIVE_ANON)),
5157                         K(node_page_state(pgdat, NR_ACTIVE_FILE)),
5158                         K(node_page_state(pgdat, NR_INACTIVE_FILE)),
5159                         K(node_page_state(pgdat, NR_UNEVICTABLE)),
5160                         K(node_page_state(pgdat, NR_ISOLATED_ANON)),
5161                         K(node_page_state(pgdat, NR_ISOLATED_FILE)),
5162                         K(node_page_state(pgdat, NR_FILE_MAPPED)),
5163                         K(node_page_state(pgdat, NR_FILE_DIRTY)),
5164                         K(node_page_state(pgdat, NR_WRITEBACK)),
5165                         K(node_page_state(pgdat, NR_SHMEM)),
5166 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
5167                         K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR),
5168                         K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)
5169                                         * HPAGE_PMD_NR),
5170                         K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR),
5171 #endif
5172                         K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
5173                         K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
5174                         pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ?
5175                                 "yes" : "no");
5176         }
5177 
5178         for_each_populated_zone(zone) {
5179                 int i;
5180 
5181                 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
5182                         continue;
5183 
5184                 free_pcp = 0;
5185                 for_each_online_cpu(cpu)
5186                         free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
5187 
5188                 show_node(zone);
5189                 printk(KERN_CONT
5190                         "%s"
5191                         " free:%lukB"
5192                         " min:%lukB"
5193                         " low:%lukB"
5194                         " high:%lukB"
5195                         " active_anon:%lukB"
5196                         " inactive_anon:%lukB"
5197                         " active_file:%lukB"
5198                         " inactive_file:%lukB"
5199                         " unevictable:%lukB"
5200                         " writepending:%lukB"
5201                         " present:%lukB"
5202                         " managed:%lukB"
5203                         " mlocked:%lukB"
5204                         " kernel_stack:%lukB"
5205                         " pagetables:%lukB"
5206                         " bounce:%lukB"
5207                         " free_pcp:%lukB"
5208                         " local_pcp:%ukB"
5209                         " free_cma:%lukB"
5210                         "\n",
5211                         zone->name,
5212                         K(zone_page_state(zone, NR_FREE_PAGES)),
5213                         K(min_wmark_pages(zone)),
5214                         K(low_wmark_pages(zone)),
5215                         K(high_wmark_pages(zone)),
5216                         K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)),
5217                         K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)),
5218                         K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)),
5219                         K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)),
5220                         K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)),
5221                         K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)),
5222                         K(zone->present_pages),
5223                         K(zone_managed_pages(zone)),
5224                         K(zone_page_state(zone, NR_MLOCK)),
5225                         zone_page_state(zone, NR_KERNEL_STACK_KB),
5226                         K(zone_page_state(zone, NR_PAGETABLE)),
5227                         K(zone_page_state(zone, NR_BOUNCE)),
5228                         K(free_pcp),
5229                         K(this_cpu_read(zone->pageset->pcp.count)),
5230                         K(zone_page_state(zone, NR_FREE_CMA_PAGES)));
5231                 printk("lowmem_reserve[]:");
5232                 for (i = 0; i < MAX_NR_ZONES; i++)
5233                         printk(KERN_CONT " %ld", zone->lowmem_reserve[i]);
5234                 printk(KERN_CONT "\n");
5235         }
5236 
5237         for_each_populated_zone(zone) {
5238                 unsigned int order;
5239                 unsigned long nr[MAX_ORDER], flags, total = 0;
5240                 unsigned char types[MAX_ORDER];
5241 
5242                 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
5243                         continue;
5244                 show_node(zone);
5245                 printk(KERN_CONT "%s: ", zone->name);
5246 
5247                 spin_lock_irqsave(&zone->lock, flags);
5248                 for (order = 0; order < MAX_ORDER; order++) {
5249                         struct free_area *area = &zone->free_area[order];
5250                         int type;
5251 
5252                         nr[order] = area->nr_free;
5253                         total += nr[order] << order;
5254 
5255                         types[order] = 0;
5256                         for (type = 0; type < MIGRATE_TYPES; type++) {
5257                                 if (!list_empty(&area->free_list[type]))
5258                                         types[order] |= 1 << type;
5259                         }
5260                 }
5261                 spin_unlock_irqrestore(&zone->lock, flags);
5262                 for (order = 0; order < MAX_ORDER; order++) {
5263                         printk(KERN_CONT "%lu*%lukB ",
5264                                nr[order], K(1UL) << order);
5265                         if (nr[order])
5266                                 show_migration_types(types[order]);
5267                 }
5268                 printk(KERN_CONT "= %lukB\n", K(total));
5269         }
5270 
5271         hugetlb_show_meminfo();
5272 
5273         printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES));
5274 
5275         show_swap_cache_info();
5276 }
5277 
5278 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
5279 {
5280         zoneref->zone = zone;
5281         zoneref->zone_idx = zone_idx(zone);
5282 }
5283 
5284 /*
5285  * Builds allocation fallback zone lists.
5286  *
5287  * Add all populated zones of a node to the zonelist.
5288  */
5289 static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs)
5290 {
5291         struct zone *zone;
5292         enum zone_type zone_type = MAX_NR_ZONES;
5293         int nr_zones = 0;
5294 
5295         do {
5296                 zone_type--;
5297                 zone = pgdat->node_zones + zone_type;
5298                 if (managed_zone(zone)) {
5299                         zoneref_set_zone(zone, &zonerefs[nr_zones++]);
5300                         check_highest_zone(zone_type);
5301                 }
5302         } while (zone_type);
5303 
5304         return nr_zones;
5305 }
5306 
5307 #ifdef CONFIG_NUMA
5308 
5309 static int __parse_numa_zonelist_order(char *s)
5310 {
5311         /*
5312          * We used to support different zonlists modes but they turned
5313          * out to be just not useful. Let's keep the warning in place
5314          * if somebody still use the cmd line parameter so that we do
5315          * not fail it silently
5316          */
5317         if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) {
5318                 pr_warn("Ignoring unsupported numa_zonelist_order value:  %s\n", s);
5319                 return -EINVAL;
5320         }
5321         return 0;
5322 }
5323 
5324 static __init int setup_numa_zonelist_order(char *s)
5325 {
5326         if (!s)
5327                 return 0;
5328 
5329         return __parse_numa_zonelist_order(s);
5330 }
5331 early_param("numa_zonelist_order", setup_numa_zonelist_order);
5332 
5333 char numa_zonelist_order[] = "Node";
5334 
5335 /*
5336  * sysctl handler for numa_zonelist_order
5337  */
5338 int numa_zonelist_order_handler(struct ctl_table *table, int write,
5339                 void __user *buffer, size_t *length,
5340                 loff_t *ppos)
5341 {
5342         char *str;
5343         int ret;
5344 
5345         if (!write)
5346                 return proc_dostring(table, write, buffer, length, ppos);
5347         str = memdup_user_nul(buffer, 16);
5348         if (IS_ERR(str))
5349                 return PTR_ERR(str);
5350 
5351         ret = __parse_numa_zonelist_order(str);
5352         kfree(str);
5353         return ret;
5354 }
5355 
5356 
5357 #define MAX_NODE_LOAD (nr_online_nodes)
5358 static int node_load[MAX_NUMNODES];
5359 
5360 /**
5361  * find_next_best_node - find the next node that should appear in a given node's fallback list
5362  * @node: node whose fallback list we're appending
5363  * @used_node_mask: nodemask_t of already used nodes
5364  *
5365  * We use a number of factors to determine which is the next node that should
5366  * appear on a given node's fallback list.  The node should not have appeared
5367  * already in @node's fallback list, and it should be the next closest node
5368  * according to the distance array (which contains arbitrary distance values
5369  * from each node to each node in the system), and should also prefer nodes
5370  * with no CPUs, since presumably they'll have very little allocation pressure
5371  * on them otherwise.
5372  *
5373  * Return: node id of the found node or %NUMA_NO_NODE if no node is found.
5374  */
5375 static int find_next_best_node(int node, nodemask_t *used_node_mask)
5376 {
5377         int n, val;
5378         int min_val = INT_MAX;
5379         int best_node = NUMA_NO_NODE;
5380         const struct cpumask *tmp = cpumask_of_node(0);
5381 
5382         /* Use the local node if we haven't already */
5383         if (!node_isset(node, *used_node_mask)) {
5384                 node_set(node, *used_node_mask);
5385                 return node;
5386         }
5387 
5388         for_each_node_state(n, N_MEMORY) {
5389 
5390                 /* Don't want a node to appear more than once */
5391                 if (node_isset(n, *used_node_mask))
5392                         continue;
5393 
5394                 /* Use the distance array to find the distance */
5395                 val = node_distance(node, n);
5396 
5397                 /* Penalize nodes under us ("prefer the next node") */
5398                 val += (n < node);
5399 
5400                 /* Give preference to headless and unused nodes */
5401                 tmp = cpumask_of_node(n);
5402                 if (!cpumask_empty(tmp))
5403                         val += PENALTY_FOR_NODE_WITH_CPUS;
5404 
5405                 /* Slight preference for less loaded node */
5406                 val *= (MAX_NODE_LOAD*MAX_NUMNODES);
5407                 val += node_load[n];
5408 
5409                 if (val < min_val) {
5410                         min_val = val;
5411                         best_node = n;
5412                 }
5413         }
5414 
5415         if (best_node >= 0)
5416                 node_set(best_node, *used_node_mask);
5417 
5418         return best_node;
5419 }
5420 
5421 
5422 /*
5423  * Build zonelists ordered by node and zones within node.
5424  * This results in maximum locality--normal zone overflows into local
5425  * DMA zone, if any--but risks exhausting DMA zone.
5426  */
5427 static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order,
5428                 unsigned nr_nodes)
5429 {
5430         struct zoneref *zonerefs;
5431         int i;
5432 
5433         zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
5434 
5435         for (i = 0; i < nr_nodes; i++) {
5436                 int nr_zones;
5437 
5438                 pg_data_t *node = NODE_DATA(node_order[i]);
5439 
5440                 nr_zones = build_zonerefs_node(node, zonerefs);
5441                 zonerefs += nr_zones;
5442         }
5443         zonerefs->zone = NULL;
5444         zonerefs->zone_idx = 0;
5445 }
5446 
5447 /*
5448  * Build gfp_thisnode zonelists
5449  */
5450 static void build_thisnode_zonelists(pg_data_t *pgdat)
5451 {
5452         struct zoneref *zonerefs;
5453         int nr_zones;
5454 
5455         zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs;
5456         nr_zones = build_zonerefs_node(pgdat, zonerefs);
5457         zonerefs += nr_zones;
5458         zonerefs->zone = NULL;
5459         zonerefs->zone_idx = 0;
5460 }
5461 
5462 /*
5463  * Build zonelists ordered by zone and nodes within zones.
5464  * This results in conserving DMA zone[s] until all Normal memory is
5465  * exhausted, but results in overflowing to remote node while memory
5466  * may still exist in local DMA zone.
5467  */
5468 
5469 static void build_zonelists(pg_data_t *pgdat)
5470 {
5471         static int node_order[MAX_NUMNODES];
5472         int node, load, nr_nodes = 0;
5473         nodemask_t used_mask;
5474         int local_node, prev_node;
5475 
5476         /* NUMA-aware ordering of nodes */
5477         local_node = pgdat->node_id;
5478         load = nr_online_nodes;
5479         prev_node = local_node;
5480         nodes_clear(used_mask);
5481 
5482         memset(node_order, 0, sizeof(node_order));
5483         while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
5484                 /*
5485                  * We don't want to pressure a particular node.
5486                  * So adding penalty to the first node in same
5487                  * distance group to make it round-robin.
5488                  */
5489                 if (node_distance(local_node, node) !=
5490                     node_distance(local_node, prev_node))
5491                         node_load[node] = load;
5492 
5493                 node_order[nr_nodes++] = node;
5494                 prev_node = node;
5495                 load--;
5496         }
5497 
5498         build_zonelists_in_node_order(pgdat, node_order, nr_nodes);
5499         build_thisnode_zonelists(pgdat);
5500 }
5501 
5502 #ifdef CONFIG_HAVE_MEMORYLESS_NODES
5503 /*
5504  * Return node id of node used for "local" allocations.
5505  * I.e., first node id of first zone in arg node's generic zonelist.
5506  * Used for initializing percpu 'numa_mem', which is used primarily
5507  * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
5508  */
5509 int local_memory_node(int node)
5510 {
5511         struct zoneref *z;
5512 
5513         z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
5514                                    gfp_zone(GFP_KERNEL),
5515                                    NULL);
5516         return zone_to_nid(z->zone);
5517 }
5518 #endif
5519 
5520 static void setup_min_unmapped_ratio(void);
5521 static void setup_min_slab_ratio(void);
5522 #else   /* CONFIG_NUMA */
5523 
5524 static void build_zonelists(pg_data_t *pgdat)
5525 {
5526         int node, local_node;
5527         struct zoneref *zonerefs;
5528         int nr_zones;
5529 
5530         local_node = pgdat->node_id;
5531 
5532         zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
5533         nr_zones = build_zonerefs_node(pgdat, zonerefs);
5534         zonerefs += nr_zones;
5535 
5536         /*
5537          * Now we build the zonelist so that it contains the zones
5538          * of all the other nodes.
5539          * We don't want to pressure a particular node, so when
5540          * building the zones for node N, we make sure that the
5541          * zones coming right after the local ones are those from
5542          * node N+1 (modulo N)
5543          */
5544         for (node = local_node + 1; node < MAX_NUMNODES; node++) {
5545                 if (!node_online(node))
5546                         continue;
5547                 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
5548                 zonerefs += nr_zones;
5549         }
5550         for (node = 0; node < local_node; node++) {
5551                 if (!node_online(node))
5552                         continue;
5553                 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
5554                 zonerefs += nr_zones;
5555         }
5556 
5557         zonerefs->zone = NULL;
5558         zonerefs->zone_idx = 0;
5559 }
5560 
5561 #endif  /* CONFIG_NUMA */
5562 
5563 /*
5564  * Boot pageset table. One per cpu which is going to be used for all
5565  * zones and all nodes. The parameters will be set in such a way
5566  * that an item put on a list will immediately be handed over to
5567  * the buddy list. This is safe since pageset manipulation is done
5568  * with interrupts disabled.
5569  *
5570  * The boot_pagesets must be kept even after bootup is complete for
5571  * unused processors and/or zones. They do play a role for bootstrapping
5572  * hotplugged processors.
5573  *
5574  * zoneinfo_show() and maybe other functions do
5575  * not check if the processor is online before following the pageset pointer.
5576  * Other parts of the kernel may not check if the zone is available.
5577  */
5578 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
5579 static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
5580 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
5581 
5582 static void __build_all_zonelists(void *data)
5583 {
5584         int nid;
5585         int __maybe_unused cpu;
5586         pg_data_t *self = data;
5587         static DEFINE_SPINLOCK(lock);
5588 
5589         spin_lock(&lock);
5590 
5591 #ifdef CONFIG_NUMA
5592         memset(node_load, 0, sizeof(node_load));
5593 #endif
5594 
5595         /*
5596          * This node is hotadded and no memory is yet present.   So just
5597          * building zonelists is fine - no need to touch other nodes.
5598          */
5599         if (self && !node_online(self->node_id)) {
5600                 build_zonelists(self);
5601         } else {
5602                 for_each_online_node(nid) {
5603                         pg_data_t *pgdat = NODE_DATA(nid);
5604 
5605                         build_zonelists(pgdat);
5606                 }
5607 
5608 #ifdef CONFIG_HAVE_MEMORYLESS_NODES
5609                 /*
5610                  * We now know the "local memory node" for each node--
5611                  * i.e., the node of the first zone in the generic zonelist.
5612                  * Set up numa_mem percpu variable for on-line cpus.  During
5613                  * boot, only the boot cpu should be on-line;  we'll init the
5614                  * secondary cpus' numa_mem as they come on-line.  During
5615                  * node/memory hotplug, we'll fixup all on-line cpus.
5616                  */
5617                 for_each_online_cpu(cpu)
5618                         set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
5619 #endif
5620         }
5621 
5622         spin_unlock(&lock);
5623 }
5624 
5625 static noinline void __init
5626 build_all_zonelists_init(void)
5627 {
5628         int cpu;
5629 
5630         __build_all_zonelists(NULL);
5631 
5632         /*
5633          * Initialize the boot_pagesets that are going to be used
5634          * for bootstrapping processors. The real pagesets for
5635          * each zone will be allocated later when the per cpu
5636          * allocator is available.
5637          *
5638          * boot_pagesets are used also for bootstrapping offline
5639          * cpus if the system is already booted because the pagesets
5640          * are needed to initialize allocators on a specific cpu too.
5641          * F.e. the percpu allocator needs the page allocator which
5642          * needs the percpu allocator in order to allocate its pagesets
5643          * (a chicken-egg dilemma).
5644          */
5645         for_each_possible_cpu(cpu)
5646                 setup_pageset(&per_cpu(boot_pageset, cpu), 0);
5647 
5648         mminit_verify_zonelist();
5649         cpuset_init_current_mems_allowed();
5650 }
5651 
5652 /*
5653  * unless system_state == SYSTEM_BOOTING.
5654  *
5655  * __ref due to call of __init annotated helper build_all_zonelists_init
5656  * [protected by SYSTEM_BOOTING].
5657  */
5658 void __ref build_all_zonelists(pg_data_t *pgdat)
5659 {
5660         if (system_state == SYSTEM_BOOTING) {
5661                 build_all_zonelists_init();
5662         } else {
5663                 __build_all_zonelists(pgdat);
5664                 /* cpuset refresh routine should be here */
5665         }
5666         vm_total_pages = nr_free_pagecache_pages();
5667         /*
5668          * Disable grouping by mobility if the number of pages in the
5669          * system is too low to allow the mechanism to work. It would be
5670          * more accurate, but expensive to check per-zone. This check is
5671          * made on memory-hotadd so a system can start with mobility
5672          * disabled and enable it later
5673          */
5674         if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
5675                 page_group_by_mobility_disabled = 1;
5676         else
5677                 page_group_by_mobility_disabled = 0;
5678 
5679         pr_info("Built %u zonelists, mobility grouping %s.  Total pages: %ld\n",
5680                 nr_online_nodes,
5681                 page_group_by_mobility_disabled ? "off" : "on",
5682                 vm_total_pages);
5683 #ifdef CONFIG_NUMA
5684         pr_info("Policy zone: %s\n", zone_names[policy_zone]);
5685 #endif
5686 }
5687 
5688 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */
5689 static bool __meminit
5690 overlap_memmap_init(unsigned long zone, unsigned long *pfn)
5691 {
5692 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
5693         static struct memblock_region *r;
5694 
5695         if (mirrored_kernelcore && zone == ZONE_MOVABLE) {
5696                 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) {
5697                         for_each_memblock(memory, r) {
5698                                 if (*pfn < memblock_region_memory_end_pfn(r))
5699                                         break;
5700                         }
5701                 }
5702                 if (*pfn >= memblock_region_memory_base_pfn(r) &&
5703                     memblock_is_mirror(r)) {
5704                         *pfn = memblock_region_memory_end_pfn(r);
5705                         return true;
5706                 }
5707         }
5708 #endif
5709         return false;
5710 }
5711 
5712 /*
5713  * Initially all pages are reserved - free ones are freed
5714  * up by memblock_free_all() once the early boot process is
5715  * done. Non-atomic initialization, single-pass.
5716  */
5717 void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
5718                 unsigned long start_pfn, enum memmap_context context,
5719                 struct vmem_altmap *altmap)
5720 {
5721         unsigned long pfn, end_pfn = start_pfn + size;
5722         struct page *page;
5723 
5724         if (highest_memmap_pfn < end_pfn - 1)
5725                 highest_memmap_pfn = end_pfn - 1;
5726 
5727 #ifdef CONFIG_ZONE_DEVICE
5728         /*
5729          * Honor reservation requested by the driver for this ZONE_DEVICE
5730          * memory. We limit the total number of pages to initialize to just
5731          * those that might contain the memory mapping. We will defer the
5732          * ZONE_DEVICE page initialization until after we have released
5733          * the hotplug lock.
5734          */
5735         if (zone == ZONE_DEVICE) {
5736                 if (!altmap)
5737                         return;
5738 
5739                 if (start_pfn == altmap->base_pfn)
5740                         start_pfn += altmap->reserve;
5741                 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
5742         }
5743 #endif
5744 
5745         for (pfn = start_pfn; pfn < end_pfn; pfn++) {
5746                 /*
5747                  * There can be holes in boot-time mem_map[]s handed to this
5748                  * function.  They do not exist on hotplugged memory.
5749                  */
5750                 if (context == MEMMAP_EARLY) {
5751                         if (!early_pfn_valid(pfn))
5752                                 continue;
5753                         if (!early_pfn_in_nid(pfn, nid))
5754                                 continue;
5755                         if (overlap_memmap_init(zone, &pfn))
5756                                 continue;
5757                         if (defer_init(nid, pfn, end_pfn))
5758                                 break;
5759                 }
5760 
5761                 page = pfn_to_page(pfn);
5762                 __init_single_page(page, pfn, zone, nid);
5763                 if (context == MEMMAP_HOTPLUG)
5764                         __SetPageReserved(page);
5765 
5766                 /*
5767                  * Mark the block movable so that blocks are reserved for
5768                  * movable at startup. This will force kernel allocations
5769                  * to reserve their blocks rather than leaking throughout
5770                  * the address space during boot when many long-lived
5771                  * kernel allocations are made.
5772                  *
5773                  * bitmap is created for zone's valid pfn range. but memmap
5774                  * can be created for invalid pages (for alignment)
5775                  * check here not to call set_pageblock_migratetype() against
5776                  * pfn out of zone.
5777                  */
5778                 if (!(pfn & (pageblock_nr_pages - 1))) {
5779                         set_pageblock_migratetype(page, MIGRATE_MOVABLE);
5780                         cond_resched();
5781                 }
5782         }
5783 }
5784 
5785 #ifdef CONFIG_ZONE_DEVICE
5786 void __ref memmap_init_zone_device(struct zone *zone,
5787                                    unsigned long start_pfn,
5788                                    unsigned long size,
5789                                    struct dev_pagemap *pgmap)
5790 {
5791         unsigned long pfn, end_pfn = start_pfn + size;
5792         struct pglist_data *pgdat = zone->zone_pgdat;
5793         unsigned long zone_idx = zone_idx(zone);
5794         unsigned long start = jiffies;
5795         int nid = pgdat->node_id;
5796 
5797         if (WARN_ON_ONCE(!pgmap || !is_dev_zone(zone)))
5798                 return;
5799 
5800         /*
5801          * The call to memmap_init_zone should have already taken care
5802          * of the pages reserved for the memmap, so we can just jump to
5803          * the end of that region and start processing the device pages.
5804          */
5805         if (pgmap->altmap_valid) {
5806                 struct vmem_altmap *altmap = &pgmap->altmap;
5807 
5808                 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
5809                 size = end_pfn - start_pfn;
5810         }
5811 
5812         for (pfn = start_pfn; pfn < end_pfn; pfn++) {
5813                 struct page *page = pfn_to_page(pfn);
5814 
5815                 __init_single_page(page, pfn, zone_idx, nid);
5816 
5817                 /*
5818                  * Mark page reserved as it will need to wait for onlining
5819                  * phase for it to be fully associated with a zone.
5820                  *
5821                  * We can use the non-atomic __set_bit operation for setting
5822                  * the flag as we are still initializing the pages.
5823                  */
5824                 __SetPageReserved(page);
5825 
5826                 /*
5827                  * ZONE_DEVICE pages union ->lru with a ->pgmap back
5828                  * pointer and hmm_data.  It is a bug if a ZONE_DEVICE
5829                  * page is ever freed or placed on a driver-private list.
5830                  */
5831                 page->pgmap = pgmap;
5832                 page->hmm_data = 0;
5833 
5834                 /*
5835                  * Mark the block movable so that blocks are reserved for
5836                  * movable at startup. This will force kernel allocations
5837                  * to reserve their blocks rather than leaking throughout
5838                  * the address space during boot when many long-lived
5839                  * kernel allocations are made.
5840                  *
5841                  * bitmap is created for zone's valid pfn range. but memmap
5842                  * can be created for invalid pages (for alignment)
5843                  * check here not to call set_pageblock_migratetype() against
5844                  * pfn out of zone.
5845                  *
5846                  * Please note that MEMMAP_HOTPLUG path doesn't clear memmap
5847                  * because this is done early in sparse_add_one_section
5848                  */
5849                 if (!(pfn & (pageblock_nr_pages - 1))) {
5850                         set_pageblock_migratetype(page, MIGRATE_MOVABLE);
5851                         cond_resched();
5852                 }
5853         }
5854 
5855         pr_info("%s initialised, %lu pages in %ums\n", dev_name(pgmap->dev),
5856                 size, jiffies_to_msecs(jiffies - start));
5857 }
5858 
5859 #endif
5860 static void __meminit zone_init_free_lists(struct zone *zone)
5861 {
5862         unsigned int order, t;
5863         for_each_migratetype_order(order, t) {
5864                 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
5865                 zone->free_area[order].nr_free = 0;
5866         }
5867 }
5868 
5869 void __meminit __weak memmap_init(unsigned long size, int nid,
5870                                   unsigned long zone, unsigned long start_pfn)
5871 {
5872         memmap_init_zone(size, nid, zone, start_pfn, MEMMAP_EARLY, NULL);
5873 }
5874 
5875 static int zone_batchsize(struct zone *zone)
5876 {
5877 #ifdef CONFIG_MMU
5878         int batch;
5879 
5880         /*
5881          * The per-cpu-pages pools are set to around 1000th of the
5882          * size of the zone.
5883          */
5884         batch = zone_managed_pages(zone) / 1024;
5885         /* But no more than a meg. */
5886         if (batch * PAGE_SIZE > 1024 * 1024)
5887                 batch = (1024 * 1024) / PAGE_SIZE;
5888         batch /= 4;             /* We effectively *= 4 below */
5889         if (batch < 1)
5890                 batch = 1;
5891 
5892         /*
5893          * Clamp the batch to a 2^n - 1 value. Having a power
5894          * of 2 value was found to be more likely to have
5895          * suboptimal cache aliasing properties in some cases.
5896          *
5897          * For example if 2 tasks are alternately allocating
5898          * batches of pages, one task can end up with a lot
5899          * of pages of one half of the possible page colors
5900          * and the other with pages of the other colors.
5901          */
5902         batch = rounddown_pow_of_two(batch + batch/2) - 1;
5903 
5904         return batch;
5905 
5906 #else
5907         /* The deferral and batching of frees should be suppressed under NOMMU
5908          * conditions.
5909          *
5910          * The problem is that NOMMU needs to be able to allocate large chunks
5911          * of contiguous memory as there's no hardware page translation to
5912          * assemble apparent contiguous memory from discontiguous pages.
5913          *
5914          * Queueing large contiguous runs of pages for batching, however,
5915          * causes the pages to actually be freed in smaller chunks.  As there
5916          * can be a significant delay between the individual batches being
5917          * recycled, this leads to the once large chunks of space being
5918          * fragmented and becoming unavailable for high-order allocations.
5919          */
5920         return 0;
5921 #endif
5922 }
5923 
5924 /*
5925  * pcp->high and pcp->batch values are related and dependent on one another:
5926  * ->batch must never be higher then ->high.
5927  * The following function updates them in a safe manner without read side
5928  * locking.
5929  *
5930  * Any new users of pcp->batch and pcp->high should ensure they can cope with
5931  * those fields changing asynchronously (acording the the above rule).
5932  *
5933  * mutex_is_locked(&pcp_batch_high_lock) required when calling this function
5934  * outside of boot time (or some other assurance that no concurrent updaters
5935  * exist).
5936  */
5937 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high,
5938                 unsigned long batch)
5939 {
5940        /* start with a fail safe value for batch */
5941         pcp->batch = 1;
5942         smp_wmb();
5943 
5944        /* Update high, then batch, in order */
5945         pcp->high = high;
5946         smp_wmb();
5947 
5948         pcp->batch = batch;
5949 }
5950 
5951 /* a companion to pageset_set_high() */
5952 static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch)
5953 {
5954         pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch));
5955 }
5956 
5957 static void pageset_init(struct per_cpu_pageset *p)
5958 {
5959         struct per_cpu_pages *pcp;
5960         int migratetype;
5961 
5962         memset(p, 0, sizeof(*p));
5963 
5964         pcp = &p->pcp;
5965         for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
5966                 INIT_LIST_HEAD(&pcp->lists[migratetype]);
5967 }
5968 
5969 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
5970 {
5971         pageset_init(p);
5972         pageset_set_batch(p, batch);
5973 }
5974 
5975 /*
5976  * pageset_set_high() sets the high water mark for hot per_cpu_pagelist
5977  * to the value high for the pageset p.
5978  */
5979 static void pageset_set_high(struct per_cpu_pageset *p,
5980                                 unsigned long high)
5981 {
5982         unsigned long batch = max(1UL, high / 4);
5983         if ((high / 4) > (PAGE_SHIFT * 8))
5984                 batch = PAGE_SHIFT * 8;
5985 
5986         pageset_update(&p->pcp, high, batch);
5987 }
5988 
5989 static void pageset_set_high_and_batch(struct zone *zone,
5990                                        struct per_cpu_pageset *pcp)
5991 {
5992         if (percpu_pagelist_fraction)
5993                 pageset_set_high(pcp,
5994                         (zone_managed_pages(zone) /
5995                                 percpu_pagelist_fraction));
5996         else
5997                 pageset_set_batch(pcp, zone_batchsize(zone));
5998 }
5999 
6000 static void __meminit zone_pageset_init(struct zone *zone, int cpu)
6001 {
6002         struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
6003 
6004         pageset_init(pcp);
6005         pageset_set_high_and_batch(zone, pcp);
6006 }
6007 
6008 void __meminit setup_zone_pageset(struct zone *zone)
6009 {
6010         int cpu;
6011         zone->pageset = alloc_percpu(struct per_cpu_pageset);
6012         for_each_possible_cpu(cpu)
6013                 zone_pageset_init(zone, cpu);
6014 }
6015 
6016 /*
6017  * Allocate per cpu pagesets and initialize them.
6018  * Before this call only boot pagesets were available.
6019  */
6020 void __init setup_per_cpu_pageset(void)
6021 {
6022         struct pglist_data *pgdat;
6023         struct zone *zone;
6024 
6025         for_each_populated_zone(zone)
6026                 setup_zone_pageset(zone);
6027 
6028         for_each_online_pgdat(pgdat)
6029                 pgdat->per_cpu_nodestats =
6030                         alloc_percpu(struct per_cpu_nodestat);
6031 }
6032 
6033 static __meminit void zone_pcp_init(struct zone *zone)
6034 {
6035         /*
6036          * per cpu subsystem is not up at this point. The following code
6037          * relies on the ability of the linker to provide the
6038          * offset of a (static) per cpu variable into the per cpu area.
6039          */
6040         zone->pageset = &boot_pageset;
6041 
6042         if (populated_zone(zone))
6043                 printk(KERN_DEBUG "  %s zone: %lu pages, LIFO batch:%u\n",
6044                         zone->name, zone->present_pages,
6045                                          zone_batchsize(zone));
6046 }
6047 
6048 void __meminit init_currently_empty_zone(struct zone *zone,
6049                                         unsigned long zone_start_pfn,
6050                                         unsigned long size)
6051 {
6052         struct pglist_data *pgdat = zone->zone_pgdat;
6053         int zone_idx = zone_idx(zone) + 1;
6054 
6055         if (zone_idx > pgdat->nr_zones)
6056                 pgdat->nr_zones = zone_idx;
6057 
6058         zone->zone_start_pfn = zone_start_pfn;
6059 
6060         mminit_dprintk(MMINIT_TRACE, "memmap_init",
6061                         "Initialising map node %d zone %lu pfns %lu -> %lu\n",
6062                         pgdat->node_id,
6063                         (unsigned long)zone_idx(zone),
6064                         zone_start_pfn, (zone_start_pfn + size));
6065 
6066         zone_init_free_lists(zone);
6067         zone->initialized = 1;
6068 }
6069 
6070 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
6071 #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
6072 
6073 /*
6074  * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
6075  */
6076 int __meminit __early_pfn_to_nid(unsigned long pfn,
6077                                         struct mminit_pfnnid_cache *state)
6078 {
6079         unsigned long start_pfn, end_pfn;
6080         int nid;
6081 
6082         if (state->last_start <= pfn && pfn < state->last_end)
6083                 return state->last_nid;
6084 
6085         nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
6086         if (nid != NUMA_NO_NODE) {
6087                 state->last_start = start_pfn;
6088                 state->last_end = end_pfn;
6089                 state->last_nid = nid;
6090         }
6091 
6092         return nid;
6093 }
6094 #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
6095 
6096 /**
6097  * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range
6098  * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed.
6099  * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid
6100  *
6101  * If an architecture guarantees that all ranges registered contain no holes
6102  * and may be freed, this this function may be used instead of calling
6103  * memblock_free_early_nid() manually.
6104  */
6105 void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn)
6106 {
6107         unsigned long start_pfn, end_pfn;
6108         int i, this_nid;
6109 
6110         for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) {
6111                 start_pfn = min(start_pfn, max_low_pfn);
6112                 end_pfn = min(end_pfn, max_low_pfn);
6113 
6114                 if (start_pfn < end_pfn)
6115                         memblock_free_early_nid(PFN_PHYS(start_pfn),
6116                                         (end_pfn - start_pfn) << PAGE_SHIFT,
6117                                         this_nid);
6118         }
6119 }
6120 
6121 /**
6122  * sparse_memory_present_with_active_regions - Call memory_present for each active range
6123  * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used.
6124  *
6125  * If an architecture guarantees that all ranges registered contain no holes and may
6126  * be freed, this function may be used instead of calling memory_present() manually.
6127  */
6128 void __init sparse_memory_present_with_active_regions(int nid)
6129 {
6130         unsigned long start_pfn, end_pfn;
6131         int i, this_nid;
6132 
6133         for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid)
6134                 memory_present(this_nid, start_pfn, end_pfn);
6135 }
6136 
6137 /**
6138  * get_pfn_range_for_nid - Return the start and end page frames for a node
6139  * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
6140  * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
6141  * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
6142  *
6143  * It returns the start and end page frame of a node based on information
6144  * provided by memblock_set_node(). If called for a node
6145  * with no available memory, a warning is printed and the start and end
6146  * PFNs will be 0.
6147  */
6148 void __init get_pfn_range_for_nid(unsigned int nid,
6149                         unsigned long *start_pfn, unsigned long *end_pfn)
6150 {
6151         unsigned long this_start_pfn, this_end_pfn;
6152         int i;
6153 
6154         *start_pfn = -1UL;
6155         *end_pfn = 0;
6156 
6157         for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
6158                 *start_pfn = min(*start_pfn, this_start_pfn);
6159                 *end_pfn = max(*end_pfn, this_end_pfn);
6160         }
6161 
6162         if (*start_pfn == -1UL)
6163                 *start_pfn = 0;
6164 }
6165 
6166 /*
6167  * This finds a zone that can be used for ZONE_MOVABLE pages. The
6168  * assumption is made that zones within a node are ordered in monotonic
6169  * increasing memory addresses so that the "highest" populated zone is used
6170  */
6171 static void __init find_usable_zone_for_movable(void)
6172 {
6173         int zone_index;
6174         for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
6175                 if (zone_index == ZONE_MOVABLE)
6176                         continue;
6177 
6178                 if (arch_zone_highest_possible_pfn[zone_index] >
6179                                 arch_zone_lowest_possible_pfn[zone_index])
6180                         break;
6181         }
6182 
6183         VM_BUG_ON(zone_index == -1);
6184         movable_zone = zone_index;
6185 }
6186 
6187 /*
6188  * The zone ranges provided by the architecture do not include ZONE_MOVABLE
6189  * because it is sized independent of architecture. Unlike the other zones,
6190  * the starting point for ZONE_MOVABLE is not fixed. It may be different
6191  * in each node depending on the size of each node and how evenly kernelcore
6192  * is distributed. This helper function adjusts the zone ranges
6193  * provided by the architecture for a given node by using the end of the
6194  * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
6195  * zones within a node are in order of monotonic increases memory addresses
6196  */
6197 static void __init adjust_zone_range_for_zone_movable(int nid,
6198                                         unsigned long zone_type,
6199                                         unsigned long node_start_pfn,
6200                                         unsigned long node_end_pfn,
6201                                         unsigned long *zone_start_pfn,
6202                                         unsigned long *zone_end_pfn)
6203 {
6204         /* Only adjust if ZONE_MOVABLE is on this node */
6205         if (zone_movable_pfn[nid]) {
6206                 /* Size ZONE_MOVABLE */
6207                 if (zone_type == ZONE_MOVABLE) {
6208                         *zone_start_pfn = zone_movable_pfn[nid];
6209                         *zone_end_pfn = min(node_end_pfn,
6210                                 arch_zone_highest_possible_pfn[movable_zone]);
6211 
6212                 /* Adjust for ZONE_MOVABLE starting within this range */
6213                 } else if (!mirrored_kernelcore &&
6214                         *zone_start_pfn < zone_movable_pfn[nid] &&
6215                         *zone_end_pfn > zone_movable_pfn[nid]) {
6216                         *zone_end_pfn = zone_movable_pfn[nid];
6217 
6218                 /* Check if this whole range is within ZONE_MOVABLE */
6219                 } else if (*zone_start_pfn >= zone_movable_pfn[nid])
6220                         *zone_start_pfn = *zone_end_pfn;
6221         }
6222 }
6223 
6224 /*
6225  * Return the number of pages a zone spans in a node, including holes
6226  * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
6227  */
6228 static unsigned long __init zone_spanned_pages_in_node(int nid,
6229                                         unsigned long zone_type,
6230                                         unsigned long node_start_pfn,
6231                                         unsigned long node_end_pfn,
6232                                         unsigned long *zone_start_pfn,
6233                                         unsigned long *zone_end_pfn,
6234                                         unsigned long *ignored)
6235 {
6236         /* When hotadd a new node from cpu_up(), the node should be empty */
6237         if (!node_start_pfn && !node_end_pfn)
6238                 return 0;
6239 
6240         /* Get the start and end of the zone */
6241         *zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type];
6242         *zone_end_pfn = arch_zone_highest_possible_pfn[zone_type];
6243         adjust_zone_range_for_zone_movable(nid, zone_type,
6244                                 node_start_pfn, node_end_pfn,
6245                                 zone_start_pfn, zone_end_pfn);
6246 
6247         /* Check that this node has pages within the zone's required range */
6248         if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn)
6249                 return 0;
6250 
6251         /* Move the zone boundaries inside the node if necessary */
6252         *zone_end_pfn = min(*zone_end_pfn, node_end_pfn);
6253         *zone_start_pfn = max(*zone_start_pfn, node_start_pfn);
6254 
6255         /* Return the spanned pages */
6256         return *zone_end_pfn - *zone_start_pfn;
6257 }
6258 
6259 /*
6260  * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
6261  * then all holes in the requested range will be accounted for.
6262  */
6263 unsigned long __init __absent_pages_in_range(int nid,
6264                                 unsigned long range_start_pfn,
6265                                 unsigned long range_end_pfn)
6266 {
6267         unsigned long nr_absent = range_end_pfn - range_start_pfn;
6268         unsigned long start_pfn, end_pfn;
6269         int i;
6270 
6271         for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
6272                 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
6273                 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
6274                 nr_absent -= end_pfn - start_pfn;
6275         }
6276         return nr_absent;
6277 }
6278 
6279 /**
6280  * absent_pages_in_range - Return number of page frames in holes within a range
6281  * @start_pfn: The start PFN to start searching for holes
6282  * @end_pfn: The end PFN to stop searching for holes
6283  *
6284  * Return: the number of pages frames in memory holes within a range.
6285  */
6286 unsigned long __init absent_pages_in_range(unsigned long start_pfn,
6287                                                         unsigned long end_pfn)
6288 {
6289         return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
6290 }
6291 
6292 /* Return the number of page frames in holes in a zone on a node */
6293 static unsigned long __init zone_absent_pages_in_node(int nid,
6294                                         unsigned long zone_type,
6295                                         unsigned long node_start_pfn,
6296                                         unsigned long node_end_pfn,
6297                                         unsigned long *ignored)
6298 {
6299         unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
6300         unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
6301         unsigned long zone_start_pfn, zone_end_pfn;
6302         unsigned long nr_absent;
6303 
6304         /* When hotadd a new node from cpu_up(), the node should be empty */
6305         if (!node_start_pfn && !node_end_pfn)
6306                 return 0;
6307 
6308         zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
6309         zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
6310 
6311         adjust_zone_range_for_zone_movable(nid, zone_type,
6312                         node_start_pfn, node_end_pfn,
6313                         &zone_start_pfn, &zone_end_pfn);
6314         nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
6315 
6316         /*
6317          * ZONE_MOVABLE handling.
6318          * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages
6319          * and vice versa.
6320          */
6321         if (mirrored_kernelcore && zone_movable_pfn[nid]) {
6322                 unsigned long start_pfn, end_pfn;
6323                 struct memblock_region *r;
6324 
6325                 for_each_memblock(memory, r) {
6326                         start_pfn = clamp(memblock_region_memory_base_pfn(r),
6327                                           zone_start_pfn, zone_end_pfn);
6328                         end_pfn = clamp(memblock_region_memory_end_pfn(r),
6329                                         zone_start_pfn, zone_end_pfn);
6330 
6331                         if (zone_type == ZONE_MOVABLE &&
6332                             memblock_is_mirror(r))
6333                                 nr_absent += end_pfn - start_pfn;
6334 
6335                         if (zone_type == ZONE_NORMAL &&
6336                             !memblock_is_mirror(r))
6337                                 nr_absent += end_pfn - start_pfn;
6338                 }
6339         }
6340 
6341         return nr_absent;
6342 }
6343 
6344 #else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
6345 static inline unsigned long __init zone_spanned_pages_in_node(int nid,
6346                                         unsigned long zone_type,
6347                                         unsigned long node_start_pfn,
6348                                         unsigned long node_end_pfn,
6349                                         unsigned long *zone_start_pfn,
6350                                         unsigned long *zone_end_pfn,
6351                                         unsigned long *zones_size)
6352 {
6353         unsigned int zone;
6354 
6355         *zone_start_pfn = node_start_pfn;
6356         for (zone = 0; zone < zone_type; zone++)
6357                 *zone_start_pfn += zones_size[zone];
6358 
6359         *zone_end_pfn = *zone_start_pfn + zones_size[zone_type];
6360 
6361         return zones_size[zone_type];
6362 }
6363 
6364 static inline unsigned long __init zone_absent_pages_in_node(int nid,
6365                                                 unsigned long zone_type,
6366                                                 unsigned long node_start_pfn,
6367                                                 unsigned long node_end_pfn,
6368                                                 unsigned long *zholes_size)
6369 {
6370         if (!zholes_size)
6371                 return 0;
6372 
6373         return zholes_size[zone_type];
6374 }
6375 
6376 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
6377 
6378 static void __init calculate_node_totalpages(struct pglist_data *pgdat,
6379                                                 unsigned long node_start_pfn,
6380                                                 unsigned long node_end_pfn,
6381                                                 unsigned long *zones_size,
6382                                                 unsigned long *zholes_size)
6383 {
6384         unsigned long realtotalpages = 0, totalpages = 0;
6385         enum zone_type i;
6386 
6387         for (i = 0; i < MAX_NR_ZONES; i++) {
6388                 struct zone *zone = pgdat->node_zones + i;
6389                 unsigned long zone_start_pfn, zone_end_pfn;
6390                 unsigned long size, real_size;
6391 
6392                 size = zone_spanned_pages_in_node(pgdat->node_id, i,
6393                                                   node_start_pfn,
6394                                                   node_end_pfn,
6395                                                   &zone_start_pfn,
6396                                                   &zone_end_pfn,
6397                                                   zones_size);
6398                 real_size = size - zone_absent_pages_in_node(pgdat->node_id, i,
6399                                                   node_start_pfn, node_end_pfn,
6400                                                   zholes_size);
6401                 if (size)
6402                         zone->zone_start_pfn = zone_start_pfn;
6403                 else
6404                         zone->zone_start_pfn = 0;
6405                 zone->spanned_pages = size;
6406                 zone->present_pages = real_size;
6407 
6408                 totalpages += size;
6409                 realtotalpages += real_size;
6410         }
6411 
6412         pgdat->node_spanned_pages = totalpages;
6413         pgdat->node_present_pages = realtotalpages;
6414         printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
6415                                                         realtotalpages);
6416 }
6417 
6418 #ifndef CONFIG_SPARSEMEM
6419 /*
6420  * Calculate the size of the zone->blockflags rounded to an unsigned long
6421  * Start by making sure zonesize is a multiple of pageblock_order by rounding
6422  * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
6423  * round what is now in bits to nearest long in bits, then return it in
6424  * bytes.
6425  */
6426 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
6427 {
6428         unsigned long usemapsize;
6429 
6430         zonesize += zone_start_pfn & (pageblock_nr_pages-1);
6431         usemapsize = roundup(zonesize, pageblock_nr_pages);
6432         usemapsize = usemapsize >> pageblock_order;
6433         usemapsize *= NR_PAGEBLOCK_BITS;
6434         usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
6435 
6436         return usemapsize / 8;
6437 }
6438 
6439 static void __ref setup_usemap(struct pglist_data *pgdat,
6440                                 struct zone *zone,
6441                                 unsigned long zone_start_pfn,
6442                                 unsigned long zonesize)
6443 {
6444         unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize);
6445         zone->pageblock_flags = NULL;
6446         if (usemapsize) {
6447                 zone->pageblock_flags =
6448                         memblock_alloc_node(usemapsize, SMP_CACHE_BYTES,
6449                                             pgdat->node_id);
6450                 if (!zone->pageblock_flags)
6451                         panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n",
6452                               usemapsize, zone->name, pgdat->node_id);
6453         }
6454 }
6455 #else
6456 static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone,
6457                                 unsigned long zone_start_pfn, unsigned long zonesize) {}
6458 #endif /* CONFIG_SPARSEMEM */
6459 
6460 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
6461 
6462 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
6463 void __init set_pageblock_order(void)
6464 {
6465         unsigned int order;
6466 
6467         /* Check that pageblock_nr_pages has not already been setup */
6468         if (pageblock_order)
6469                 return;
6470 
6471         if (HPAGE_SHIFT > PAGE_SHIFT)
6472                 order = HUGETLB_PAGE_ORDER;
6473         else
6474                 order = MAX_ORDER - 1;
6475 
6476         /*
6477          * Assume the largest contiguous order of interest is a huge page.
6478          * This value may be variable depending on boot parameters on IA64 and
6479          * powerpc.
6480          */
6481         pageblock_order = order;
6482 }
6483 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
6484 
6485 /*
6486  * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
6487  * is unused as pageblock_order is set at compile-time. See
6488  * include/linux/pageblock-flags.h for the values of pageblock_order based on
6489  * the kernel config
6490  */
6491 void __init set_pageblock_order(void)
6492 {
6493 }
6494 
6495 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
6496 
6497 static unsigned long __init calc_memmap_size(unsigned long spanned_pages,
6498                                                 unsigned long present_pages)
6499 {
6500         unsigned long pages = spanned_pages;
6501 
6502         /*
6503          * Provide a more accurate estimation if there are holes within
6504          * the zone and SPARSEMEM is in use. If there are holes within the
6505          * zone, each populated memory region may cost us one or two extra
6506          * memmap pages due to alignment because memmap pages for each
6507          * populated regions may not be naturally aligned on page boundary.
6508          * So the (present_pages >> 4) heuristic is a tradeoff for that.
6509          */
6510         if (spanned_pages > present_pages + (present_pages >> 4) &&
6511             IS_ENABLED(CONFIG_SPARSEMEM))
6512                 pages = present_pages;
6513 
6514         return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
6515 }
6516 
6517 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
6518 static void pgdat_init_split_queue(struct pglist_data *pgdat)
6519 {
6520         spin_lock_init(&pgdat->split_queue_lock);
6521         INIT_LIST_HEAD(&pgdat->split_queue);
6522         pgdat->split_queue_len = 0;
6523 }
6524 #else
6525 static void pgdat_init_split_queue(struct pglist_data *pgdat) {}
6526 #endif
6527 
6528 #ifdef CONFIG_COMPACTION
6529 static void pgdat_init_kcompactd(struct pglist_data *pgdat)
6530 {
6531         init_waitqueue_head(&pgdat->kcompactd_wait);
6532 }
6533 #else
6534 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {}
6535 #endif
6536 
6537 static void __meminit pgdat_init_internals(struct pglist_data *pgdat)
6538 {
6539         pgdat_resize_init(pgdat);
6540 
6541         pgdat_init_split_queue(pgdat);
6542         pgdat_init_kcompactd(pgdat);
6543 
6544         init_waitqueue_head(&pgdat->kswapd_wait);
6545         init_waitqueue_head(&pgdat->pfmemalloc_wait);
6546 
6547         pgdat_page_ext_init(pgdat);
6548         spin_lock_init(&pgdat->lru_lock);
6549         lruvec_init(node_lruvec(pgdat));
6550 }
6551 
6552 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,
6553                                                         unsigned long remaining_pages)
6554 {
6555         atomic_long_set(&zone->managed_pages, remaining_pages);
6556         zone_set_nid(zone, nid);
6557         zone->name = zone_names[idx];
6558         zone->zone_pgdat = NODE_DATA(nid);
6559         spin_lock_init(&zone->lock);
6560         zone_seqlock_init(zone);
6561         zone_pcp_init(zone);
6562 }
6563 
6564 /*
6565  * Set up the zone data structures
6566  * - init pgdat internals
6567  * - init all zones belonging to this node
6568  *
6569  * NOTE: this function is only called during memory hotplug
6570  */
6571 #ifdef CONFIG_MEMORY_HOTPLUG
6572 void __ref free_area_init_core_hotplug(int nid)
6573 {
6574         enum zone_type z;
6575         pg_data_t *pgdat = NODE_DATA(nid);
6576 
6577         pgdat_init_internals(pgdat);
6578         for (z = 0; z < MAX_NR_ZONES; z++)
6579                 zone_init_internals(&pgdat->node_zones[z], z, nid, 0);
6580 }
6581 #endif
6582 
6583 /*
6584  * Set up the zone data structures:
6585  *   - mark all pages reserved
6586  *   - mark all memory queues empty
6587  *   - clear the memory bitmaps
6588  *
6589  * NOTE: pgdat should get zeroed by caller.
6590  * NOTE: this function is only called during early init.
6591  */
6592 static void __init free_area_init_core(struct pglist_data *pgdat)
6593 {
6594         enum zone_type j;
6595         int nid = pgdat->node_id;
6596 
6597         pgdat_init_internals(pgdat);
6598         pgdat->per_cpu_nodestats = &boot_nodestats;
6599 
6600         for (j = 0; j < MAX_NR_ZONES; j++) {
6601                 struct zone *zone = pgdat->node_zones + j;
6602                 unsigned long size, freesize, memmap_pages;
6603                 unsigned long zone_start_pfn = zone->zone_start_pfn;
6604 
6605                 size = zone->spanned_pages;
6606                 freesize = zone->present_pages;
6607 
6608                 /*
6609                  * Adjust freesize so that it accounts for how much memory
6610                  * is used by this zone for memmap. This affects the watermark
6611                  * and per-cpu initialisations
6612                  */
6613                 memmap_pages = calc_memmap_size(size, freesize);
6614                 if (!is_highmem_idx(j)) {
6615                         if (freesize >= memmap_pages) {
6616                                 freesize -= memmap_pages;
6617                                 if (memmap_pages)
6618                                         printk(KERN_DEBUG
6619                                                "  %s zone: %lu pages used for memmap\n",
6620                                                zone_names[j], memmap_pages);
6621                         } else
6622                                 pr_warn("  %s zone: %lu pages exceeds freesize %lu\n",
6623                                         zone_names[j], memmap_pages, freesize);
6624                 }
6625 
6626                 /* Account for reserved pages */
6627                 if (j == 0 && freesize > dma_reserve) {
6628                         freesize -= dma_reserve;
6629                         printk(KERN_DEBUG "  %s zone: %lu pages reserved\n",
6630                                         zone_names[0], dma_reserve);
6631                 }
6632 
6633                 if (!is_highmem_idx(j))
6634                         nr_kernel_pages += freesize;
6635                 /* Charge for highmem memmap if there are enough kernel pages */
6636                 else if (nr_kernel_pages > memmap_pages * 2)
6637                         nr_kernel_pages -= memmap_pages;
6638                 nr_all_pages += freesize;
6639 
6640                 /*
6641                  * Set an approximate value for lowmem here, it will be adjusted
6642                  * when the bootmem allocator frees pages into the buddy system.
6643                  * And all highmem pages will be managed by the buddy system.
6644                  */
6645                 zone_init_internals(zone, j, nid, freesize);
6646 
6647                 if (!size)
6648                         continue;
6649 
6650                 set_pageblock_order();
6651                 setup_usemap(pgdat, zone, zone_start_pfn, size);
6652                 init_currently_empty_zone(zone, zone_start_pfn, size);
6653                 memmap_init(size, nid, j, zone_start_pfn);
6654         }
6655 }
6656 
6657 #ifdef CONFIG_FLAT_NODE_MEM_MAP
6658 static void __ref alloc_node_mem_map(struct pglist_data *pgdat)
6659 {
6660         unsigned long __maybe_unused start = 0;
6661         unsigned long __maybe_unused offset = 0;
6662 
6663         /* Skip empty nodes */
6664         if (!pgdat->node_spanned_pages)
6665                 return;
6666 
6667         start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
6668         offset = pgdat->node_start_pfn - start;
6669         /* ia64 gets its own node_mem_map, before this, without bootmem */
6670         if (!pgdat->node_mem_map) {
6671                 unsigned long size, end;
6672                 struct page *map;
6673 
6674                 /*
6675                  * The zone's endpoints aren't required to be MAX_ORDER
6676                  * aligned but the node_mem_map endpoints must be in order
6677                  * for the buddy allocator to function correctly.
6678                  */
6679                 end = pgdat_end_pfn(pgdat);
6680                 end = ALIGN(end, MAX_ORDER_NR_PAGES);
6681                 size =  (end - start) * sizeof(struct page);
6682                 map = memblock_alloc_node(size, SMP_CACHE_BYTES,
6683                                           pgdat->node_id);
6684                 if (!map)
6685                         panic("Failed to allocate %ld bytes for node %d memory map\n",
6686                               size, pgdat->node_id);
6687                 pgdat->node_mem_map = map + offset;
6688         }
6689         pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",
6690                                 __func__, pgdat->node_id, (unsigned long)pgdat,
6691                                 (unsigned long)pgdat->node_mem_map);
6692 #ifndef CONFIG_NEED_MULTIPLE_NODES
6693         /*
6694          * With no DISCONTIG, the global mem_map is just set as node 0's
6695          */
6696         if (pgdat == NODE_DATA(0)) {
6697                 mem_map = NODE_DATA(0)->node_mem_map;
6698 #if defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) || defined(CONFIG_FLATMEM)
6699                 if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
6700                         mem_map -= offset;
6701 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
6702         }
6703 #endif
6704 }
6705 #else
6706 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) { }
6707 #endif /* CONFIG_FLAT_NODE_MEM_MAP */
6708 
6709 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
6710 static inline void pgdat_set_deferred_range(pg_data_t *pgdat)
6711 {
6712         pgdat->first_deferred_pfn = ULONG_MAX;
6713 }
6714 #else
6715 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
6716 #endif
6717 
6718 void __init free_area_init_node(int nid, unsigned long *zones_size,
6719                                    unsigned long node_start_pfn,
6720                                    unsigned long *zholes_size)
6721 {
6722         pg_data_t *pgdat = NODE_DATA(nid);
6723         unsigned long start_pfn = 0;
6724         unsigned long end_pfn = 0;
6725 
6726         /* pg_data_t should be reset to zero when it's allocated */
6727         WARN_ON(pgdat->nr_zones || pgdat->kswapd_classzone_idx);
6728 
6729         pgdat->node_id = nid;
6730         pgdat->node_start_pfn = node_start_pfn;
6731         pgdat->per_cpu_nodestats = NULL;
6732 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
6733         get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
6734         pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,
6735                 (u64)start_pfn << PAGE_SHIFT,
6736                 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);
6737 #else
6738         start_pfn = node_start_pfn;
6739 #endif
6740         calculate_node_totalpages(pgdat, start_pfn, end_pfn,
6741                                   zones_size, zholes_size);
6742 
6743         alloc_node_mem_map(pgdat);
6744         pgdat_set_deferred_range(pgdat);
6745 
6746         free_area_init_core(pgdat);
6747 }
6748 
6749 #if !defined(CONFIG_FLAT_NODE_MEM_MAP)
6750 /*
6751  * Zero all valid struct pages in range [spfn, epfn), return number of struct
6752  * pages zeroed
6753  */
6754 static u64 zero_pfn_range(unsigned long spfn, unsigned long epfn)
6755 {
6756         unsigned long pfn;
6757         u64 pgcnt = 0;
6758 
6759         for (pfn = spfn; pfn < epfn; pfn++) {
6760                 if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) {
6761                         pfn = ALIGN_DOWN(pfn, pageblock_nr_pages)
6762                                 + pageblock_nr_pages - 1;
6763                         continue;
6764                 }
6765                 mm_zero_struct_page(pfn_to_page(pfn));
6766                 pgcnt++;
6767         }
6768 
6769         return pgcnt;
6770 }
6771 
6772 /*
6773  * Only struct pages that are backed by physical memory are zeroed and
6774  * initialized by going through __init_single_page(). But, there are some
6775  * struct pages which are reserved in memblock allocator and their fields
6776  * may be accessed (for example page_to_pfn() on some configuration accesses
6777  * flags). We must explicitly zero those struct pages.
6778  *
6779  * This function also addresses a similar issue where struct pages are left
6780  * uninitialized because the physical address range is not covered by
6781  * memblock.memory or memblock.reserved. That could happen when memblock
6782  * layout is manually configured via memmap=.
6783  */
6784 void __init zero_resv_unavail(void)
6785 {
6786         phys_addr_t start, end;
6787         u64 i, pgcnt;
6788         phys_addr_t next = 0;
6789 
6790         /*
6791          * Loop through unavailable ranges not covered by memblock.memory.
6792          */
6793         pgcnt = 0;
6794         for_each_mem_range(i, &memblock.memory, NULL,
6795                         NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end, NULL) {
6796                 if (next < start)
6797                         pgcnt += zero_pfn_range(PFN_DOWN(next), PFN_UP(start));
6798                 next = end;
6799         }
6800         pgcnt += zero_pfn_range(PFN_DOWN(next), max_pfn);
6801 
6802         /*
6803          * Struct pages that do not have backing memory. This could be because
6804          * firmware is using some of this memory, or for some other reasons.
6805          */
6806         if (pgcnt)
6807                 pr_info("Zeroed struct page in unavailable ranges: %lld pages", pgcnt);
6808 }
6809 #endif /* !CONFIG_FLAT_NODE_MEM_MAP */
6810 
6811 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
6812 
6813 #if MAX_NUMNODES > 1
6814 /*
6815  * Figure out the number of possible node ids.
6816  */
6817 void __init setup_nr_node_ids(void)
6818 {
6819         unsigned int highest;
6820 
6821         highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES);
6822         nr_node_ids = highest + 1;
6823 }
6824 #endif
6825 
6826 /**
6827  * node_map_pfn_alignment - determine the maximum internode alignment
6828  *
6829  * This function should be called after node map is populated and sorted.
6830  * It calculates the maximum power of two alignment which can distinguish
6831  * all the nodes.
6832  *
6833  * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
6834  * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)).  If the
6835  * nodes are shifted by 256MiB, 256MiB.  Note that if only the last node is
6836  * shifted, 1GiB is enough and this function will indicate so.
6837  *
6838  * This is used to test whether pfn -> nid mapping of the chosen memory
6839  * model has fine enough granularity to avoid incorrect mapping for the
6840  * populated node map.
6841  *
6842  * Return: the determined alignment in pfn's.  0 if there is no alignment
6843  * requirement (single node).
6844  */
6845 unsigned long __init node_map_pfn_alignment(void)
6846 {
6847         unsigned long accl_mask = 0, last_end = 0;
6848         unsigned long start, end, mask;
6849         int last_nid = NUMA_NO_NODE;
6850         int i, nid;
6851 
6852         for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
6853                 if (!start || last_nid < 0 || last_nid == nid) {
6854                         last_nid = nid;
6855                         last_end = end;
6856                         continue;
6857                 }
6858 
6859                 /*
6860                  * Start with a mask granular enough to pin-point to the
6861                  * start pfn and tick off bits one-by-one until it becomes
6862                  * too coarse to separate the current node from the last.
6863                  */
6864                 mask = ~((1 << __ffs(start)) - 1);
6865                 while (mask && last_end <= (start & (mask << 1)))
6866                         mask <<= 1;
6867 
6868                 /* accumulate all internode masks */
6869                 accl_mask |= mask;
6870         }
6871 
6872         /* convert mask to number of pages */
6873         return ~accl_mask + 1;
6874 }
6875 
6876 /* Find the lowest pfn for a node */
6877 static unsigned long __init find_min_pfn_for_node(int nid)
6878 {
6879         unsigned long min_pfn = ULONG_MAX;
6880         unsigned long start_pfn;
6881         int i;
6882 
6883         for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL)
6884                 min_pfn = min(min_pfn, start_pfn);
6885 
6886         if (min_pfn == ULONG_MAX) {
6887                 pr_warn("Could not find start_pfn for node %d\n", nid);
6888                 return 0;
6889         }
6890 
6891         return min_pfn;
6892 }
6893 
6894 /**
6895  * find_min_pfn_with_active_regions - Find the minimum PFN registered
6896  *
6897  * Return: the minimum PFN based on information provided via
6898  * memblock_set_node().
6899  */
6900 unsigned long __init find_min_pfn_with_active_regions(void)
6901 {
6902         return find_min_pfn_for_node(MAX_NUMNODES);
6903 }
6904 
6905 /*
6906  * early_calculate_totalpages()
6907  * Sum pages in active regions for movable zone.
6908  * Populate N_MEMORY for calculating usable_nodes.
6909  */
6910 static unsigned long __init early_calculate_totalpages(void)
6911 {
6912         unsigned long totalpages = 0;
6913         unsigned long start_pfn, end_pfn;
6914         int i, nid;
6915 
6916         for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
6917                 unsigned long pages = end_pfn - start_pfn;
6918 
6919                 totalpages += pages;
6920                 if (pages)
6921                         node_set_state(nid, N_MEMORY);
6922         }
6923         return totalpages;
6924 }
6925 
6926 /*
6927  * Find the PFN the Movable zone begins in each node. Kernel memory
6928  * is spread evenly between nodes as long as the nodes have enough
6929  * memory. When they don't, some nodes will have more kernelcore than
6930  * others
6931  */
6932 static void __init find_zone_movable_pfns_for_nodes(void)
6933 {
6934         int i, nid;
6935         unsigned long usable_startpfn;
6936         unsigned long kernelcore_node, kernelcore_remaining;
6937         /* save the state before borrow the nodemask */
6938         nodemask_t saved_node_state = node_states[N_MEMORY];
6939         unsigned long totalpages = early_calculate_totalpages();
6940         int usable_nodes = nodes_weight(node_states[N_MEMORY]);
6941         struct memblock_region *r;
6942 
6943         /* Need to find movable_zone earlier when movable_node is specified. */
6944         find_usable_zone_for_movable();
6945 
6946         /*
6947          * If movable_node is specified, ignore kernelcore and movablecore
6948          * options.
6949          */
6950         if (movable_node_is_enabled()) {
6951                 for_each_memblock(memory, r) {
6952                         if (!memblock_is_hotpluggable(r))
6953                                 continue;
6954 
6955                         nid = r->nid;
6956 
6957                         usable_startpfn = PFN_DOWN(r->base);
6958                         zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
6959                                 min(usable_startpfn, zone_movable_pfn[nid]) :
6960                                 usable_startpfn;
6961                 }
6962 
6963                 goto out2;
6964         }
6965 
6966         /*
6967          * If kernelcore=mirror is specified, ignore movablecore option
6968          */
6969         if (mirrored_kernelcore) {
6970                 bool mem_below_4gb_not_mirrored = false;
6971 
6972                 for_each_memblock(memory, r) {
6973                         if (memblock_is_mirror(r))
6974                                 continue;
6975 
6976                         nid = r->nid;
6977 
6978                         usable_startpfn = memblock_region_memory_base_pfn(r);
6979 
6980                         if (usable_startpfn < 0x100000) {
6981                                 mem_below_4gb_not_mirrored = true;
6982                                 continue;
6983                         }
6984 
6985                         zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
6986                                 min(usable_startpfn, zone_movable_pfn[nid]) :
6987                                 usable_startpfn;
6988                 }
6989 
6990                 if (mem_below_4gb_not_mirrored)
6991                         pr_warn("This configuration results in unmirrored kernel memory.");
6992 
6993                 goto out2;
6994         }
6995 
6996         /*
6997          * If kernelcore=nn% or movablecore=nn% was specified, calculate the
6998          * amount of necessary memory.
6999          */
7000         if (required_kernelcore_percent)
7001                 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) /
7002                                        10000UL;
7003         if (required_movablecore_percent)
7004                 required_movablecore = (totalpages * 100 * required_movablecore_percent) /
7005                                         10000UL;
7006 
7007         /*
7008          * If movablecore= was specified, calculate what size of
7009          * kernelcore that corresponds so that memory usable for
7010          * any allocation type is evenly spread. If both kernelcore
7011          * and movablecore are specified, then the value of kernelcore
7012          * will be used for required_kernelcore if it's greater than
7013          * what movablecore would have allowed.
7014          */
7015         if (required_movablecore) {
7016                 unsigned long corepages;
7017 
7018                 /*
7019                  * Round-up so that ZONE_MOVABLE is at least as large as what
7020                  * was requested by the user
7021                  */
7022                 required_movablecore =
7023                         roundup(required_movablecore, MAX_ORDER_NR_PAGES);
7024                 required_movablecore = min(totalpages, required_movablecore);
7025                 corepages = totalpages - required_movablecore;
7026 
7027                 required_kernelcore = max(required_kernelcore, corepages);
7028         }
7029 
7030         /*
7031          * If kernelcore was not specified or kernelcore size is larger
7032          * than totalpages, there is no ZONE_MOVABLE.
7033          */
7034         if (!required_kernelcore || required_kernelcore >= totalpages)
7035                 goto out;
7036 
7037         /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
7038         usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
7039 
7040 restart:
7041         /* Spread kernelcore memory as evenly as possible throughout nodes */
7042         kernelcore_node = required_kernelcore / usable_nodes;
7043         for_each_node_state(nid, N_MEMORY) {
7044                 unsigned long start_pfn, end_pfn;
7045 
7046                 /*
7047                  * Recalculate kernelcore_node if the division per node
7048                  * now exceeds what is necessary to satisfy the requested
7049                  * amount of memory for the kernel
7050                  */
7051                 if (required_kernelcore < kernelcore_node)
7052                         kernelcore_node = required_kernelcore / usable_nodes;
7053 
7054                 /*
7055                  * As the map is walked, we track how much memory is usable
7056                  * by the kernel using kernelcore_remaining. When it is
7057                  * 0, the rest of the node is usable by ZONE_MOVABLE
7058                  */
7059                 kernelcore_remaining = kernelcore_node;
7060 
7061                 /* Go through each range of PFNs within this node */
7062                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
7063                         unsigned long size_pages;
7064 
7065                         start_pfn = max(start_pfn, zone_movable_pfn[nid]);
7066                         if (start_pfn >= end_pfn)
7067                                 continue;
7068 
7069                         /* Account for what is only usable for kernelcore */
7070                         if (start_pfn < usable_startpfn) {
7071                                 unsigned long kernel_pages;
7072                                 kernel_pages = min(end_pfn, usable_startpfn)
7073                                                                 - start_pfn;
7074 
7075                                 kernelcore_remaining -= min(kernel_pages,
7076                                                         kernelcore_remaining);
7077                                 required_kernelcore -= min(kernel_pages,
7078                                                         required_kernelcore);
7079 
7080                                 /* Continue if range is now fully accounted */
7081                                 if (end_pfn <= usable_startpfn) {
7082 
7083                                         /*
7084                                          * Push zone_movable_pfn to the end so
7085                                          * that if we have to rebalance
7086                                          * kernelcore across nodes, we will
7087                                          * not double account here
7088                                          */
7089                                         zone_movable_pfn[nid] = end_pfn;
7090                                         continue;
7091                                 }
7092                                 start_pfn = usable_startpfn;
7093                         }
7094 
7095                         /*
7096                          * The usable PFN range for ZONE_MOVABLE is from
7097                          * start_pfn->end_pfn. Calculate size_pages as the
7098                          * number of pages used as kernelcore
7099                          */
7100                         size_pages = end_pfn - start_pfn;
7101                         if (size_pages > kernelcore_remaining)
7102                                 size_pages = kernelcore_remaining;
7103                         zone_movable_pfn[nid] = start_pfn + size_pages;
7104 
7105                         /*
7106                          * Some kernelcore has been met, update counts and
7107                          * break if the kernelcore for this node has been
7108                          * satisfied
7109                          */
7110                         required_kernelcore -= min(required_kernelcore,
7111                                                                 size_pages);
7112                         kernelcore_remaining -= size_pages;
7113                         if (!kernelcore_remaining)
7114                                 break;
7115                 }
7116         }
7117 
7118         /*
7119          * If there is still required_kernelcore, we do another pass with one
7120          * less node in the count. This will push zone_movable_pfn[nid] further
7121          * along on the nodes that still have memory until kernelcore is
7122          * satisfied
7123          */
7124         usable_nodes--;
7125         if (usable_nodes && required_kernelcore > usable_nodes)
7126                 goto restart;
7127 
7128 out2:
7129         /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
7130         for (nid = 0; nid < MAX_NUMNODES; nid++)
7131                 zone_movable_pfn[nid] =
7132                         roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
7133 
7134 out:
7135         /* restore the node_state */
7136         node_states[N_MEMORY] = saved_node_state;
7137 }
7138 
7139 /* Any regular or high memory on that node ? */
7140 static void check_for_memory(pg_data_t *pgdat, int nid)
7141 {
7142         enum zone_type zone_type;
7143 
7144         for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
7145                 struct zone *zone = &pgdat->node_zones[zone_type];
7146                 if (populated_zone(zone)) {
7147                         if (IS_ENABLED(CONFIG_HIGHMEM))
7148                                 node_set_state(nid, N_HIGH_MEMORY);
7149                         if (zone_type <= ZONE_NORMAL)
7150                                 node_set_state(nid, N_NORMAL_MEMORY);
7151                         break;
7152                 }
7153         }
7154 }
7155 
7156 /**
7157  * free_area_init_nodes - Initialise all pg_data_t and zone data
7158  * @max_zone_pfn: an array of max PFNs for each zone
7159  *
7160  * This will call free_area_init_node() for each active node in the system.
7161  * Using the page ranges provided by memblock_set_node(), the size of each
7162  * zone in each node and their holes is calculated. If the maximum PFN
7163  * between two adjacent zones match, it is assumed that the zone is empty.
7164  * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
7165  * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
7166  * starts where the previous one ended. For example, ZONE_DMA32 starts
7167  * at arch_max_dma_pfn.
7168  */
7169 void __init free_area_init_nodes(unsigned long *max_zone_pfn)
7170 {
7171         unsigned long start_pfn, end_pfn;
7172         int i, nid;
7173 
7174         /* Record where the zone boundaries are */
7175         memset(arch_zone_lowest_possible_pfn, 0,
7176                                 sizeof(arch_zone_lowest_possible_pfn));
7177         memset(arch_zone_highest_possible_pfn, 0,
7178                                 sizeof(arch_zone_highest_possible_pfn));
7179 
7180         start_pfn = find_min_pfn_with_active_regions();
7181 
7182         for (i = 0; i < MAX_NR_ZONES; i++) {
7183                 if (i == ZONE_MOVABLE)
7184                         continue;
7185 
7186                 end_pfn = max(max_zone_pfn[i], start_pfn);
7187                 arch_zone_lowest_possible_pfn[i] = start_pfn;
7188                 arch_zone_highest_possible_pfn[i] = end_pfn;
7189 
7190                 start_pfn = end_pfn;
7191         }
7192 
7193         /* Find the PFNs that ZONE_MOVABLE begins at in each node */
7194         memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
7195         find_zone_movable_pfns_for_nodes();
7196 
7197         /* Print out the zone ranges */
7198         pr_info("Zone ranges:\n");
7199         for (i = 0; i < MAX_NR_ZONES; i++) {
7200                 if (i == ZONE_MOVABLE)
7201                         continue;
7202                 pr_info("  %-8s ", zone_names[i]);
7203                 if (arch_zone_lowest_possible_pfn[i] ==
7204                                 arch_zone_highest_possible_pfn[i])
7205                         pr_cont("empty\n");
7206                 else
7207                         pr_cont("[mem %#018Lx-%#018Lx]\n",
7208                                 (u64)arch_zone_lowest_possible_pfn[i]
7209                                         << PAGE_SHIFT,
7210                                 ((u64)arch_zone_highest_possible_pfn[i]
7211                                         << PAGE_SHIFT) - 1);
7212         }
7213 
7214         /* Print out the PFNs ZONE_MOVABLE begins at in each node */
7215         pr_info("Movable zone start for each node\n");
7216         for (i = 0; i < MAX_NUMNODES; i++) {
7217                 if (zone_movable_pfn[i])
7218                         pr_info("  Node %d: %#018Lx\n", i,
7219                                (u64)zone_movable_pfn[i] << PAGE_SHIFT);
7220         }
7221 
7222         /* Print out the early node map */
7223         pr_info("Early memory node ranges\n");
7224         for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
7225                 pr_info("  node %3d: [mem %#018Lx-%#018Lx]\n", nid,
7226                         (u64)start_pfn << PAGE_SHIFT,
7227                         ((u64)end_pfn << PAGE_SHIFT) - 1);
7228 
7229         /* Initialise every node */
7230         mminit_verify_pageflags_layout();
7231         setup_nr_node_ids();
7232         zero_resv_unavail();
7233         for_each_online_node(nid) {
7234                 pg_data_t *pgdat = NODE_DATA(nid);
7235                 free_area_init_node(nid, NULL,
7236                                 find_min_pfn_for_node(nid), NULL);
7237 
7238                 /* Any memory on that node */
7239                 if (pgdat->node_present_pages)
7240                         node_set_state(nid, N_MEMORY);
7241                 check_for_memory(pgdat, nid);
7242         }
7243 }
7244 
7245 static int __init cmdline_parse_core(char *p, unsigned long *core,
7246                                      unsigned long *percent)
7247 {
7248         unsigned long long coremem;
7249         char *endptr;
7250 
7251         if (!p)
7252                 return -EINVAL;
7253 
7254         /* Value may be a percentage of total memory, otherwise bytes */
7255         coremem = simple_strtoull(p, &endptr, 0);
7256         if (*endptr == '%') {
7257                 /* Paranoid check for percent values greater than 100 */
7258                 WARN_ON(coremem > 100);
7259 
7260                 *percent = coremem;
7261         } else {
7262                 coremem = memparse(p, &p);
7263                 /* Paranoid check that UL is enough for the coremem value */
7264                 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
7265 
7266                 *core = coremem >> PAGE_SHIFT;
7267                 *percent = 0UL;
7268         }
7269         return 0;
7270 }
7271 
7272 /*
7273  * kernelcore=size sets the amount of memory for use for allocations that
7274  * cannot be reclaimed or migrated.
7275  */
7276 static int __init cmdline_parse_kernelcore(char *p)
7277 {
7278         /* parse kernelcore=mirror */
7279         if (parse_option_str(p, "mirror")) {
7280                 mirrored_kernelcore = true;
7281                 return 0;
7282         }
7283 
7284         return cmdline_parse_core(p, &required_kernelcore,
7285                                   &required_kernelcore_percent);
7286 }
7287 
7288 /*
7289  * movablecore=size sets the amount of memory for use for allocations that
7290  * can be reclaimed or migrated.
7291  */
7292 static int __init cmdline_parse_movablecore(char *p)
7293 {
7294         return cmdline_parse_core(p, &required_movablecore,
7295                                   &required_movablecore_percent);
7296 }
7297 
7298 early_param("kernelcore", cmdline_parse_kernelcore);
7299 early_param("movablecore", cmdline_parse_movablecore);
7300 
7301 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
7302 
7303 void adjust_managed_page_count(struct page *page, long count)
7304 {
7305         atomic_long_add(count, &page_zone(page)->managed_pages);
7306         totalram_pages_add(count);
7307 #ifdef CONFIG_HIGHMEM
7308         if (PageHighMem(page))
7309                 totalhigh_pages_add(count);
7310 #endif
7311 }
7312 EXPORT_SYMBOL(adjust_managed_page_count);
7313 
7314 unsigned long free_reserved_area(void *start, void *end, int poison, const char *s)
7315 {
7316         void *pos;
7317         unsigned long pages = 0;
7318 
7319         start = (void *)PAGE_ALIGN((unsigned long)start);
7320         end = (void *)((unsigned long)end & PAGE_MASK);
7321         for (pos = start; pos < end; pos += PAGE_SIZE, pages++) {
7322                 struct page *page = virt_to_page(pos);
7323                 void *direct_map_addr;
7324 
7325                 /*
7326                  * 'direct_map_addr' might be different from 'pos'
7327                  * because some architectures' virt_to_page()
7328                  * work with aliases.  Getting the direct map
7329                  * address ensures that we get a _writeable_
7330                  * alias for the memset().
7331                  */
7332                 direct_map_addr = page_address(page);
7333                 if ((unsigned int)poison <= 0xFF)
7334                         memset(direct_map_addr, poison, PAGE_SIZE);
7335 
7336                 free_reserved_page(page);
7337         }
7338 
7339         if (pages && s)
7340                 pr_info("Freeing %s memory: %ldK\n",
7341                         s, pages << (PAGE_SHIFT - 10));
7342 
7343         return pages;
7344 }
7345 
7346 #ifdef  CONFIG_HIGHMEM
7347 void free_highmem_page(struct page *page)
7348 {
7349         __free_reserved_page(page);
7350         totalram_pages_inc();
7351         atomic_long_inc(&page_zone(page)->managed_pages);
7352         totalhigh_pages_inc();
7353 }
7354 #endif
7355 
7356 
7357 void __init mem_init_print_info(const char *str)
7358 {
7359         unsigned long physpages, codesize, datasize, rosize, bss_size;
7360         unsigned long init_code_size, init_data_size;
7361 
7362         physpages = get_num_physpages();
7363         codesize = _etext - _stext;
7364         datasize = _edata - _sdata;
7365         rosize = __end_rodata - __start_rodata;
7366         bss_size = __bss_stop - __bss_start;
7367         init_data_size = __init_end - __init_begin;
7368         init_code_size = _einittext - _sinittext;
7369 
7370         /*
7371          * Detect special cases and adjust section sizes accordingly:
7372          * 1) .init.* may be embedded into .data sections
7373          * 2) .init.text.* may be out of [__init_begin, __init_end],
7374          *    please refer to arch/tile/kernel/vmlinux.lds.S.
7375          * 3) .rodata.* may be embedded into .text or .data sections.
7376          */
7377 #define adj_init_size(start, end, size, pos, adj) \
7378         do { \
7379                 if (start <= pos && pos < end && size > adj) \
7380                         size -= adj; \
7381         } while (0)
7382 
7383         adj_init_size(__init_begin, __init_end, init_data_size,
7384                      _sinittext, init_code_size);
7385         adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
7386         adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
7387         adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
7388         adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);
7389 
7390 #undef  adj_init_size
7391 
7392         pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
7393 #ifdef  CONFIG_HIGHMEM
7394                 ", %luK highmem"
7395 #endif
7396                 "%s%s)\n",
7397                 nr_free_pages() << (PAGE_SHIFT - 10),
7398                 physpages << (PAGE_SHIFT - 10),
7399                 codesize >> 10, datasize >> 10, rosize >> 10,
7400                 (init_data_size + init_code_size) >> 10, bss_size >> 10,
7401                 (physpages - totalram_pages() - totalcma_pages) << (PAGE_SHIFT - 10),
7402                 totalcma_pages << (PAGE_SHIFT - 10),
7403 #ifdef  CONFIG_HIGHMEM
7404                 totalhigh_pages() << (PAGE_SHIFT - 10),
7405 #endif
7406                 str ? ", " : "", str ? str : "");
7407 }
7408 
7409 /**
7410  * set_dma_reserve - set the specified number of pages reserved in the first zone
7411  * @new_dma_reserve: The number of pages to mark reserved
7412  *
7413  * The per-cpu batchsize and zone watermarks are determined by managed_pages.
7414  * In the DMA zone, a significant percentage may be consumed by kernel image
7415  * and other unfreeable allocations which can skew the watermarks badly. This
7416  * function may optionally be used to account for unfreeable pages in the
7417  * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
7418  * smaller per-cpu batchsize.
7419  */
7420 void __init set_dma_reserve(unsigned long new_dma_reserve)
7421 {
7422         dma_reserve = new_dma_reserve;
7423 }
7424 
7425 void __init free_area_init(unsigned long *zones_size)
7426 {
7427         zero_resv_unavail();
7428         free_area_init_node(0, zones_size,
7429                         __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
7430 }
7431 
7432 static int page_alloc_cpu_dead(unsigned int cpu)
7433 {
7434 
7435         lru_add_drain_cpu(cpu);
7436         drain_pages(cpu);
7437 
7438         /*
7439          * Spill the event counters of the dead processor
7440          * into the current processors event counters.
7441          * This artificially elevates the count of the current
7442          * processor.
7443          */
7444         vm_events_fold_cpu(cpu);
7445 
7446         /*
7447          * Zero the differential counters of the dead processor
7448          * so that the vm statistics are consistent.
7449          *
7450          * This is only okay since the processor is dead and cannot
7451          * race with what we are doing.
7452          */
7453         cpu_vm_stats_fold(cpu);
7454         return 0;
7455 }
7456 
7457 void __init page_alloc_init(void)
7458 {
7459         int ret;
7460 
7461         ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD,
7462                                         "mm/page_alloc:dead", NULL,
7463                                         page_alloc_cpu_dead);
7464         WARN_ON(ret < 0);
7465 }
7466 
7467 /*
7468  * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio
7469  *      or min_free_kbytes changes.
7470  */
7471 static void calculate_totalreserve_pages(void)
7472 {
7473         struct pglist_data *pgdat;
7474         unsigned long reserve_pages = 0;
7475         enum zone_type i, j;
7476 
7477         for_each_online_pgdat(pgdat) {
7478 
7479                 pgdat->totalreserve_pages = 0;
7480 
7481                 for (i = 0; i < MAX_NR_ZONES; i++) {
7482                         struct zone *zone = pgdat->node_zones + i;
7483                         long max = 0;
7484                         unsigned long managed_pages = zone_managed_pages(zone);
7485 
7486                         /* Find valid and maximum lowmem_reserve in the zone */
7487                         for (j = i; j < MAX_NR_ZONES; j++) {
7488                                 if (zone->lowmem_reserve[j] > max)
7489                                         max = zone->lowmem_reserve[j];
7490                         }
7491 
7492                         /* we treat the high watermark as reserved pages. */
7493                         max += high_wmark_pages(zone);
7494 
7495                         if (max > managed_pages)
7496                                 max = managed_pages;
7497 
7498                         pgdat->totalreserve_pages += max;
7499 
7500                         reserve_pages += max;
7501                 }
7502         }
7503         totalreserve_pages = reserve_pages;
7504 }
7505 
7506 /*
7507  * setup_per_zone_lowmem_reserve - called whenever
7508  *      sysctl_lowmem_reserve_ratio changes.  Ensures that each zone
7509  *      has a correct pages reserved value, so an adequate number of
7510  *      pages are left in the zone after a successful __alloc_pages().
7511  */
7512 static void setup_per_zone_lowmem_reserve(void)
7513 {
7514         struct pglist_data *pgdat;
7515         enum zone_type j, idx;
7516 
7517         for_each_online_pgdat(pgdat) {
7518                 for (j = 0; j < MAX_NR_ZONES; j++) {
7519                         struct zone *zone = pgdat->node_zones + j;
7520                         unsigned long managed_pages = zone_managed_pages(zone);
7521 
7522                         zone->lowmem_reserve[j] = 0;
7523 
7524                         idx = j;
7525                         while (idx) {
7526                                 struct zone *lower_zone;
7527 
7528                                 idx--;
7529                                 lower_zone = pgdat->node_zones + idx;
7530 
7531                                 if (sysctl_lowmem_reserve_ratio[idx] < 1) {
7532                                         sysctl_lowmem_reserve_ratio[idx] = 0;
7533                                         lower_zone->lowmem_reserve[j] = 0;
7534                                 } else {
7535                                         lower_zone->lowmem_reserve[j] =
7536                                                 managed_pages / sysctl_lowmem_reserve_ratio[idx];
7537                                 }
7538                                 managed_pages += zone_managed_pages(lower_zone);
7539                         }
7540                 }
7541         }
7542 
7543         /* update totalreserve_pages */
7544         calculate_totalreserve_pages();
7545 }
7546 
7547 static void __setup_per_zone_wmarks(void)
7548 {
7549         unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
7550         unsigned long lowmem_pages = 0;
7551         struct zone *zone;
7552         unsigned long flags;
7553 
7554         /* Calculate total number of !ZONE_HIGHMEM pages */
7555         for_each_zone(zone) {
7556                 if (!is_highmem(zone))
7557                         lowmem_pages += zone_managed_pages(zone);
7558         }
7559 
7560         for_each_zone(zone) {
7561                 u64 tmp;
7562 
7563                 spin_lock_irqsave(&zone->lock, flags);
7564                 tmp = (u64)pages_min * zone_managed_pages(zone);
7565                 do_div(tmp, lowmem_pages);
7566                 if (is_highmem(zone)) {
7567                         /*
7568                          * __GFP_HIGH and PF_MEMALLOC allocations usually don't
7569                          * need highmem pages, so cap pages_min to a small
7570                          * value here.
7571                          *
7572                          * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
7573                          * deltas control async page reclaim, and so should
7574                          * not be capped for highmem.
7575                          */
7576                         unsigned long min_pages;
7577 
7578                         min_pages = zone_managed_pages(zone) / 1024;
7579                         min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
7580                         zone->_watermark[WMARK_MIN] = min_pages;
7581                 } else {
7582                         /*
7583                          * If it's a lowmem zone, reserve a number of pages
7584                          * proportionate to the zone's size.
7585                          */
7586                         zone->_watermark[WMARK_MIN] = tmp;
7587                 }
7588 
7589                 /*
7590                  * Set the kswapd watermarks distance according to the
7591                  * scale factor in proportion to available memory, but
7592                  * ensure a minimum size on small systems.
7593                  */
7594                 tmp = max_t(u64, tmp >> 2,
7595                             mult_frac(zone_managed_pages(zone),
7596                                       watermark_scale_factor, 10000));
7597 
7598                 zone->_watermark[WMARK_LOW]  = min_wmark_pages(zone) + tmp;
7599                 zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2;
7600                 zone->watermark_boost = 0;
7601 
7602                 spin_unlock_irqrestore(&zone->lock, flags);
7603         }
7604 
7605         /* update totalreserve_pages */
7606         calculate_totalreserve_pages();
7607 }
7608 
7609 /**
7610  * setup_per_zone_wmarks - called when min_free_kbytes changes
7611  * or when memory is hot-{added|removed}
7612  *
7613  * Ensures that the watermark[min,low,high] values for each zone are set
7614  * correctly with respect to min_free_kbytes.
7615  */
7616 void setup_per_zone_wmarks(void)
7617 {
7618         static DEFINE_SPINLOCK(lock);
7619 
7620         spin_lock(&lock);
7621         __setup_per_zone_wmarks();
7622         spin_unlock(&lock);
7623 }
7624 
7625 /*
7626  * Initialise min_free_kbytes.
7627  *
7628  * For small machines we want it small (128k min).  For large machines
7629  * we want it large (64MB max).  But it is not linear, because network
7630  * bandwidth does not increase linearly with machine size.  We use
7631  *
7632  *      min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
7633  *      min_free_kbytes = sqrt(lowmem_kbytes * 16)
7634  *
7635  * which yields
7636  *
7637  * 16MB:        512k
7638  * 32MB:        724k
7639  * 64MB:        1024k
7640  * 128MB:       1448k
7641  * 256MB:       2048k
7642  * 512MB:       2896k
7643  * 1024MB:      4096k
7644  * 2048MB:      5792k
7645  * 4096MB:      8192k
7646  * 8192MB:      11584k
7647  * 16384MB:     16384k
7648  */
7649 int __meminit init_per_zone_wmark_min(void)
7650 {
7651         unsigned long lowmem_kbytes;
7652         int new_min_free_kbytes;
7653 
7654         lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
7655         new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
7656 
7657         if (new_min_free_kbytes > user_min_free_kbytes) {
7658                 min_free_kbytes = new_min_free_kbytes;
7659                 if (min_free_kbytes < 128)
7660                         min_free_kbytes = 128;
7661                 if (min_free_kbytes > 65536)
7662                         min_free_kbytes = 65536;
7663         } else {
7664                 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n",
7665                                 new_min_free_kbytes, user_min_free_kbytes);
7666         }
7667         setup_per_zone_wmarks();
7668         refresh_zone_stat_thresholds();
7669         setup_per_zone_lowmem_reserve();
7670 
7671 #ifdef CONFIG_NUMA
7672         setup_min_unmapped_ratio();
7673         setup_min_slab_ratio();
7674 #endif
7675 
7676         return 0;
7677 }
7678 core_initcall(init_per_zone_wmark_min)
7679 
7680 /*
7681  * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
7682  *      that we can call two helper functions whenever min_free_kbytes
7683  *      changes.
7684  */
7685 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write,
7686         void __user *buffer, size_t *length, loff_t *ppos)
7687 {
7688         int rc;
7689 
7690         rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7691         if (rc)
7692                 return rc;
7693 
7694         if (write) {
7695                 user_min_free_kbytes = min_free_kbytes;
7696                 setup_per_zone_wmarks();
7697         }
7698         return 0;
7699 }
7700 
7701 int watermark_boost_factor_sysctl_handler(struct ctl_table *table, int write,
7702         void __user *buffer, size_t *length, loff_t *ppos)
7703 {
7704         int rc;
7705 
7706         rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7707         if (rc)
7708                 return rc;
7709 
7710         return 0;
7711 }
7712 
7713 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write,
7714         void __user *buffer, size_t *length, loff_t *ppos)
7715 {
7716         int rc;
7717 
7718         rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7719         if (rc)
7720                 return rc;
7721 
7722         if (write)
7723                 setup_per_zone_wmarks();
7724 
7725         return 0;
7726 }
7727 
7728 #ifdef CONFIG_NUMA
7729 static void setup_min_unmapped_ratio(void)
7730 {
7731         pg_data_t *pgdat;
7732         struct zone *zone;
7733 
7734         for_each_online_pgdat(pgdat)
7735                 pgdat->min_unmapped_pages = 0;
7736 
7737         for_each_zone(zone)
7738                 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) *
7739                                                          sysctl_min_unmapped_ratio) / 100;
7740 }
7741 
7742 
7743 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write,
7744         void __user *buffer, size_t *length, loff_t *ppos)
7745 {
7746         int rc;
7747 
7748         rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7749         if (rc)
7750                 return rc;
7751 
7752         setup_min_unmapped_ratio();
7753 
7754         return 0;
7755 }
7756 
7757 static void setup_min_slab_ratio(void)
7758 {
7759         pg_data_t *pgdat;
7760         struct zone *zone;
7761 
7762         for_each_online_pgdat(pgdat)
7763                 pgdat->min_slab_pages = 0;
7764 
7765         for_each_zone(zone)
7766                 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) *
7767                                                      sysctl_min_slab_ratio) / 100;
7768 }
7769 
7770 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write,
7771         void __user *buffer, size_t *length, loff_t *ppos)
7772 {
7773         int rc;
7774 
7775         rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7776         if (rc)
7777                 return rc;
7778 
7779         setup_min_slab_ratio();
7780 
7781         return 0;
7782 }
7783 #endif
7784 
7785 /*
7786  * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
7787  *      proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
7788  *      whenever sysctl_lowmem_reserve_ratio changes.
7789  *
7790  * The reserve ratio obviously has absolutely no relation with the
7791  * minimum watermarks. The lowmem reserve ratio can only make sense
7792  * if in function of the boot time zone sizes.
7793  */
7794 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write,
7795         void __user *buffer, size_t *length, loff_t *ppos)
7796 {
7797         proc_dointvec_minmax(table, write, buffer, length, ppos);
7798         setup_per_zone_lowmem_reserve();
7799         return 0;
7800 }
7801 
7802 /*
7803  * percpu_pagelist_fraction - changes the pcp->high for each zone on each
7804  * cpu.  It is the fraction of total pages in each zone that a hot per cpu
7805  * pagelist can have before it gets flushed back to buddy allocator.
7806  */
7807 int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write,
7808         void __user *buffer, size_t *length, loff_t *ppos)
7809 {
7810         struct zone *zone;
7811         int old_percpu_pagelist_fraction;
7812         int ret;
7813 
7814         mutex_lock(&pcp_batch_high_lock);
7815         old_percpu_pagelist_fraction = percpu_pagelist_fraction;
7816 
7817         ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
7818         if (!write || ret < 0)
7819                 goto out;
7820 
7821         /* Sanity checking to avoid pcp imbalance */
7822         if (percpu_pagelist_fraction &&
7823             percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) {
7824                 percpu_pagelist_fraction = old_percpu_pagelist_fraction;
7825                 ret = -EINVAL;
7826                 goto out;
7827         }
7828 
7829         /* No change? */
7830         if (percpu_pagelist_fraction == old_percpu_pagelist_fraction)
7831                 goto out;
7832 
7833         for_each_populated_zone(zone) {
7834                 unsigned int cpu;
7835 
7836                 for_each_possible_cpu(cpu)
7837                         pageset_set_high_and_batch(zone,
7838                                         per_cpu_ptr(zone->pageset, cpu));
7839         }
7840 out:
7841         mutex_unlock(&pcp_batch_high_lock);
7842         return ret;
7843 }
7844 
7845 #ifdef CONFIG_NUMA
7846 int hashdist = HASHDIST_DEFAULT;
7847 
7848 static int __init set_hashdist(char *str)
7849 {
7850         if (!str)
7851                 return 0;
7852         hashdist = simple_strtoul(str, &str, 0);
7853         return 1;
7854 }
7855 __setup("hashdist=", set_hashdist);
7856 #endif
7857 
7858 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES
7859 /*
7860  * Returns the number of pages that arch has reserved but
7861  * is not known to alloc_large_system_hash().
7862  */
7863 static unsigned long __init arch_reserved_kernel_pages(void)
7864 {
7865         return 0;
7866 }
7867 #endif
7868 
7869 /*
7870  * Adaptive scale is meant to reduce sizes of hash tables on large memory
7871  * machines. As memory size is increased the scale is also increased but at
7872  * slower pace.  Starting from ADAPT_SCALE_BASE (64G), every time memory
7873  * quadruples the scale is increased by one, which means the size of hash table
7874  * only doubles, instead of quadrupling as well.
7875  * Because 32-bit systems cannot have large physical memory, where this scaling
7876  * makes sense, it is disabled on such platforms.
7877  */
7878 #if __BITS_PER_LONG > 32
7879 #define ADAPT_SCALE_BASE        (64ul << 30)
7880 #define ADAPT_SCALE_SHIFT       2
7881 #define ADAPT_SCALE_NPAGES      (ADAPT_SCALE_BASE >> PAGE_SHIFT)
7882 #endif
7883 
7884 /*
7885  * allocate a large system hash table from bootmem
7886  * - it is assumed that the hash table must contain an exact power-of-2
7887  *   quantity of entries
7888  * - limit is the number of hash buckets, not the total allocation size
7889  */
7890 void *__init alloc_large_system_hash(const char *tablename,
7891                                      unsigned long bucketsize,
7892                                      unsigned long numentries,
7893                                      int scale,
7894                                      int flags,
7895                                      unsigned int *_hash_shift,
7896                                      unsigned int *_hash_mask,
7897                                      unsigned long low_limit,
7898                                      unsigned long high_limit)
7899 {
7900         unsigned long long max = high_limit;
7901         unsigned long log2qty, size;
7902         void *table = NULL;
7903         gfp_t gfp_flags;
7904 
7905         /* allow the kernel cmdline to have a say */
7906         if (!numentries) {
7907                 /* round applicable memory size up to nearest megabyte */
7908                 numentries = nr_kernel_pages;
7909                 numentries -= arch_reserved_kernel_pages();
7910 
7911                 /* It isn't necessary when PAGE_SIZE >= 1MB */
7912                 if (PAGE_SHIFT < 20)
7913                         numentries = round_up(numentries, (1<<20)/PAGE_SIZE);
7914 
7915 #if __BITS_PER_LONG > 32
7916                 if (!high_limit) {
7917                         unsigned long adapt;
7918 
7919                         for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries;
7920                              adapt <<= ADAPT_SCALE_SHIFT)
7921                                 scale++;
7922                 }
7923 #endif
7924 
7925                 /* limit to 1 bucket per 2^scale bytes of low memory */
7926                 if (scale > PAGE_SHIFT)
7927                         numentries >>= (scale - PAGE_SHIFT);
7928                 else
7929                         numentries <<= (PAGE_SHIFT - scale);
7930 
7931                 /* Make sure we've got at least a 0-order allocation.. */
7932                 if (unlikely(flags & HASH_SMALL)) {
7933                         /* Makes no sense without HASH_EARLY */
7934                         WARN_ON(!(flags & HASH_EARLY));
7935                         if (!(numentries >> *_hash_shift)) {
7936                                 numentries = 1UL << *_hash_shift;
7937                                 BUG_ON(!numentries);
7938                         }
7939                 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
7940                         numentries = PAGE_SIZE / bucketsize;
7941         }
7942         numentries = roundup_pow_of_two(numentries);
7943 
7944         /* limit allocation size to 1/16 total memory by default */
7945         if (max == 0) {
7946                 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
7947                 do_div(max, bucketsize);
7948         }
7949         max = min(max, 0x80000000ULL);
7950 
7951         if (numentries < low_limit)
7952                 numentries = low_limit;
7953         if (numentries > max)
7954                 numentries = max;
7955 
7956         log2qty = ilog2(numentries);
7957 
7958         gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC;
7959         do {
7960                 size = bucketsize << log2qty;
7961                 if (flags & HASH_EARLY) {
7962                         if (flags & HASH_ZERO)
7963                                 table = memblock_alloc(size, SMP_CACHE_BYTES);
7964                         else
7965                                 table = memblock_alloc_raw(size,
7966                                                            SMP_CACHE_BYTES);
7967                 } else if (hashdist) {
7968                         table = __vmalloc(size, gfp_flags, PAGE_KERNEL);
7969                 } else {
7970                         /*
7971                          * If bucketsize is not a power-of-two, we may free
7972                          * some pages at the end of hash table which
7973                          * alloc_pages_exact() automatically does
7974                          */
7975                         if (get_order(size) < MAX_ORDER) {
7976                                 table = alloc_pages_exact(size, gfp_flags);
7977                                 kmemleak_alloc(table, size, 1, gfp_flags);
7978                         }
7979                 }
7980         } while (!table && size > PAGE_SIZE && --log2qty);
7981 
7982         if (!table)
7983                 panic("Failed to allocate %s hash table\n", tablename);
7984 
7985         pr_info("%s hash table entries: %ld (order: %d, %lu bytes)\n",
7986                 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size);
7987 
7988         if (_hash_shift)
7989                 *_hash_shift = log2qty;
7990         if (_hash_mask)
7991                 *_hash_mask = (1 << log2qty) - 1;
7992 
7993         return table;
7994 }
7995 
7996 /*
7997  * This function checks whether pageblock includes unmovable pages or not.
7998  * If @count is not zero, it is okay to include less @count unmovable pages
7999  *
8000  * PageLRU check without isolation or lru_lock could race so that
8001  * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable
8002  * check without lock_page also may miss some movable non-lru pages at
8003  * race condition. So you can't expect this function should be exact.
8004  */
8005 bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
8006                          int migratetype, int flags)
8007 {
8008         unsigned long pfn, iter, found;
8009 
8010         /*
8011          * TODO we could make this much more efficient by not checking every
8012          * page in the range if we know all of them are in MOVABLE_ZONE and
8013          * that the movable zone guarantees that pages are migratable but
8014          * the later is not the case right now unfortunatelly. E.g. movablecore
8015          * can still lead to having bootmem allocations in zone_movable.
8016          */
8017 
8018         /*
8019          * CMA allocations (alloc_contig_range) really need to mark isolate
8020          * CMA pageblocks even when they are not movable in fact so consider
8021          * them movable here.
8022          */
8023         if (is_migrate_cma(migratetype) &&
8024                         is_migrate_cma(get_pageblock_migratetype(page)))
8025                 return false;
8026 
8027         pfn = page_to_pfn(page);
8028         for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) {
8029                 unsigned long check = pfn + iter;
8030 
8031                 if (!pfn_valid_within(check))
8032                         continue;
8033 
8034                 page = pfn_to_page(check);
8035 
8036                 if (PageReserved(page))
8037                         goto unmovable;
8038 
8039                 /*
8040                  * If the zone is movable and we have ruled out all reserved
8041                  * pages then it should be reasonably safe to assume the rest
8042                  * is movable.
8043                  */
8044                 if (zone_idx(zone) == ZONE_MOVABLE)
8045                         continue;
8046 
8047                 /*
8048                  * Hugepages are not in LRU lists, but they're movable.
8049                  * We need not scan over tail pages because we don't
8050                  * handle each tail page individually in migration.
8051                  */
8052                 if (PageHuge(page)) {
8053                         struct page *head = compound_head(page);
8054                         unsigned int skip_pages;
8055 
8056                         if (!hugepage_migration_supported(page_hstate(head)))
8057                                 goto unmovable;
8058 
8059                         skip_pages = (1 << compound_order(head)) - (page - head);
8060                         iter += skip_pages - 1;
8061                         continue;
8062                 }
8063 
8064                 /*
8065                  * We can't use page_count without pin a page
8066                  * because another CPU can free compound page.
8067                  * This check already skips compound tails of THP
8068                  * because their page->_refcount is zero at all time.
8069                  */
8070                 if (!page_ref_count(page)) {
8071                         if (PageBuddy(page))
8072                                 iter += (1 << page_order(page)) - 1;
8073                         continue;
8074                 }
8075 
8076                 /*
8077                  * The HWPoisoned page may be not in buddy system, and
8078                  * page_count() is not 0.
8079                  */
8080                 if ((flags & SKIP_HWPOISON) && PageHWPoison(page))
8081                         continue;
8082 
8083                 if (__PageMovable(page))
8084                         continue;
8085 
8086                 if (!PageLRU(page))
8087                         found++;
8088                 /*
8089                  * If there are RECLAIMABLE pages, we need to check
8090                  * it.  But now, memory offline itself doesn't call
8091                  * shrink_node_slabs() and it still to be fixed.
8092                  */
8093                 /*
8094                  * If the page is not RAM, page_count()should be 0.
8095                  * we don't need more check. This is an _used_ not-movable page.
8096                  *
8097                  * The problematic thing here is PG_reserved pages. PG_reserved
8098                  * is set to both of a memory hole page and a _used_ kernel
8099                  * page at boot.
8100                  */
8101                 if (found > count)
8102                         goto unmovable;
8103         }
8104         return false;
8105 unmovable:
8106         WARN_ON_ONCE(zone_idx(zone) == ZONE_MOVABLE);
8107         if (flags & REPORT_FAILURE)
8108                 dump_page(pfn_to_page(pfn+iter), "unmovable page");
8109         return true;
8110 }
8111 
8112 #if (defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || defined(CONFIG_CMA)
8113 
8114 static unsigned long pfn_max_align_down(unsigned long pfn)
8115 {
8116         return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
8117                              pageblock_nr_pages) - 1);
8118 }
8119 
8120 static unsigned long pfn_max_align_up(unsigned long pfn)
8121 {
8122         return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
8123                                 pageblock_nr_pages));
8124 }
8125 
8126 /* [start, end) must belong to a single zone. */
8127 static int __alloc_contig_migrate_range(struct compact_control *cc,
8128                                         unsigned long start, unsigned long end)
8129 {
8130         /* This function is based on compact_zone() from compaction.c. */
8131         unsigned long nr_reclaimed;
8132         unsigned long pfn = start;
8133         unsigned int tries = 0;
8134         int ret = 0;
8135 
8136         migrate_prep();
8137 
8138         while (pfn < end || !list_empty(&cc->migratepages)) {
8139                 if (fatal_signal_pending(current)) {
8140                         ret = -EINTR;
8141                         break;
8142                 }
8143 
8144                 if (list_empty(&cc->migratepages)) {
8145                         cc->nr_migratepages = 0;
8146                         pfn = isolate_migratepages_range(cc, pfn, end);
8147                         if (!pfn) {
8148                                 ret = -EINTR;
8149                                 break;
8150                         }
8151                         tries = 0;
8152                 } else if (++tries == 5) {
8153                         ret = ret < 0 ? ret : -EBUSY;
8154                         break;
8155                 }
8156 
8157                 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
8158                                                         &cc->migratepages);
8159                 cc->nr_migratepages -= nr_reclaimed;
8160 
8161                 ret = migrate_pages(&cc->migratepages, alloc_migrate_target,
8162                                     NULL, 0, cc->mode, MR_CONTIG_RANGE);
8163         }
8164         if (ret < 0) {
8165                 putback_movable_pages(&cc->migratepages);
8166                 return ret;
8167         }
8168         return 0;
8169 }
8170 
8171 /**
8172  * alloc_contig_range() -- tries to allocate given range of pages
8173  * @start:      start PFN to allocate
8174  * @end:        one-past-the-last PFN to allocate
8175  * @migratetype:        migratetype of the underlaying pageblocks (either
8176  *                      #MIGRATE_MOVABLE or #MIGRATE_CMA).  All pageblocks
8177  *                      in range must have the same migratetype and it must
8178  *                      be either of the two.
8179  * @gfp_mask:   GFP mask to use during compaction
8180  *
8181  * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES
8182  * aligned.  The PFN range must belong to a single zone.
8183  *
8184  * The first thing this routine does is attempt to MIGRATE_ISOLATE all
8185  * pageblocks in the range.  Once isolated, the pageblocks should not
8186  * be modified by others.
8187  *
8188  * Return: zero on success or negative error code.  On success all
8189  * pages which PFN is in [start, end) are allocated for the caller and
8190  * need to be freed with free_contig_range().
8191  */
8192 int alloc_contig_range(unsigned long start, unsigned long end,
8193                        unsigned migratetype, gfp_t gfp_mask)
8194 {
8195         unsigned long outer_start, outer_end;
8196         unsigned int order;
8197         int ret = 0;
8198 
8199         struct compact_control cc = {
8200                 .nr_migratepages = 0,
8201                 .order = -1,
8202                 .zone = page_zone(pfn_to_page(start)),
8203                 .mode = MIGRATE_SYNC,
8204                 .ignore_skip_hint = true,
8205                 .no_set_skip_hint = true,
8206                 .gfp_mask = current_gfp_context(gfp_mask),
8207         };
8208         INIT_LIST_HEAD(&cc.migratepages);
8209 
8210         /*
8211          * What we do here is we mark all pageblocks in range as
8212          * MIGRATE_ISOLATE.  Because pageblock and max order pages may
8213          * have different sizes, and due to the way page allocator
8214          * work, we align the range to biggest of the two pages so
8215          * that page allocator won't try to merge buddies from
8216          * different pageblocks and change MIGRATE_ISOLATE to some
8217          * other migration type.
8218          *
8219          * Once the pageblocks are marked as MIGRATE_ISOLATE, we
8220          * migrate the pages from an unaligned range (ie. pages that
8221          * we are interested in).  This will put all the pages in
8222          * range back to page allocator as MIGRATE_ISOLATE.
8223          *
8224          * When this is done, we take the pages in range from page
8225          * allocator removing them from the buddy system.  This way
8226          * page allocator will never consider using them.
8227          *
8228          * This lets us mark the pageblocks back as
8229          * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
8230          * aligned range but not in the unaligned, original range are
8231          * put back to page allocator so that buddy can use them.
8232          */
8233 
8234         ret = start_isolate_page_range(pfn_max_align_down(start),
8235                                        pfn_max_align_up(end), migratetype, 0);
8236         if (ret)
8237                 return ret;
8238 
8239         /*
8240          * In case of -EBUSY, we'd like to know which page causes problem.
8241          * So, just fall through. test_pages_isolated() has a tracepoint
8242          * which will report the busy page.
8243          *
8244          * It is possible that busy pages could become available before
8245          * the call to test_pages_isolated, and the range will actually be
8246          * allocated.  So, if we fall through be sure to clear ret so that
8247          * -EBUSY is not accidentally used or returned to caller.
8248          */
8249         ret = __alloc_contig_migrate_range(&cc, start, end);
8250         if (ret && ret != -EBUSY)
8251                 goto done;
8252         ret =0;
8253 
8254         /*
8255          * Pages from [start, end) are within a MAX_ORDER_NR_PAGES
8256          * aligned blocks that are marked as MIGRATE_ISOLATE.  What's
8257          * more, all pages in [start, end) are free in page allocator.
8258          * What we are going to do is to allocate all pages from
8259          * [start, end) (that is remove them from page allocator).
8260          *
8261          * The only problem is that pages at the beginning and at the
8262          * end of interesting range may be not aligned with pages that
8263          * page allocator holds, ie. they can be part of higher order
8264          * pages.  Because of this, we reserve the bigger range and
8265          * once this is done free the pages we are not interested in.
8266          *
8267          * We don't have to hold zone->lock here because the pages are
8268          * isolated thus they won't get removed from buddy.
8269          */
8270 
8271         lru_add_drain_all();
8272 
8273         order = 0;
8274         outer_start = start;
8275         while (!PageBuddy(pfn_to_page(outer_start))) {
8276                 if (++order >= MAX_ORDER) {
8277                         outer_start = start;
8278                         break;
8279                 }
8280                 outer_start &= ~0UL << order;
8281         }
8282 
8283         if (outer_start != start) {
8284                 order = page_order(pfn_to_page(outer_start));
8285 
8286                 /*
8287                  * outer_start page could be small order buddy page and
8288                  * it doesn't include start page. Adjust outer_start
8289                  * in this case to report failed page properly
8290                  * on tracepoint in test_pages_isolated()
8291                  */
8292                 if (outer_start + (1UL << order) <= start)
8293                         outer_start = start;
8294         }
8295 
8296         /* Make sure the range is really isolated. */
8297         if (test_pages_isolated(outer_start, end, false)) {
8298                 pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n",
8299                         __func__, outer_start, end);
8300                 ret = -EBUSY;
8301                 goto done;
8302         }
8303 
8304         /* Grab isolated pages from freelists. */
8305         outer_end = isolate_freepages_range(&cc, outer_start, end);
8306         if (!outer_end) {
8307                 ret = -EBUSY;
8308                 goto done;
8309         }
8310 
8311         /* Free head and tail (if any) */
8312         if (start != outer_start)
8313                 free_contig_range(outer_start, start - outer_start);
8314         if (end != outer_end)
8315                 free_contig_range(end, outer_end - end);
8316 
8317 done:
8318         undo_isolate_page_range(pfn_max_align_down(start),
8319                                 pfn_max_align_up(end), migratetype);
8320         return ret;
8321 }
8322 
8323 void free_contig_range(unsigned long pfn, unsigned nr_pages)
8324 {
8325         unsigned int count = 0;
8326 
8327         for (; nr_pages--; pfn++) {
8328                 struct page *page = pfn_to_page(pfn);
8329 
8330                 count += page_count(page) != 1;
8331                 __free_page(page);
8332         }
8333         WARN(count != 0, "%d pages are still in use!\n", count);
8334 }
8335 #endif
8336 
8337 #ifdef CONFIG_MEMORY_HOTPLUG
8338 /*
8339  * The zone indicated has a new number of managed_pages; batch sizes and percpu
8340  * page high values need to be recalulated.
8341  */
8342 void __meminit zone_pcp_update(struct zone *zone)
8343 {
8344         unsigned cpu;
8345         mutex_lock(&pcp_batch_high_lock);
8346         for_each_possible_cpu(cpu)
8347                 pageset_set_high_and_batch(zone,
8348                                 per_cpu_ptr(zone->pageset, cpu));
8349         mutex_unlock(&pcp_batch_high_lock);
8350 }
8351 #endif
8352 
8353 void zone_pcp_reset(struct zone *zone)
8354 {
8355         unsigned long flags;
8356         int cpu;
8357         struct per_cpu_pageset *pset;
8358 
8359         /* avoid races with drain_pages()  */
8360         local_irq_save(flags);
8361         if (zone->pageset != &boot_pageset) {
8362                 for_each_online_cpu(cpu) {
8363                         pset = per_cpu_ptr(zone->pageset, cpu);
8364                         drain_zonestat(zone, pset);
8365                 }
8366                 free_percpu(zone->pageset);
8367                 zone->pageset = &boot_pageset;
8368         }
8369         local_irq_restore(flags);
8370 }
8371 
8372 #ifdef CONFIG_MEMORY_HOTREMOVE
8373 /*
8374  * All pages in the range must be in a single zone and isolated
8375  * before calling this.
8376  */
8377 void
8378 __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
8379 {
8380         struct page *page;
8381         struct zone *zone;
8382         unsigned int order, i;
8383         unsigned long pfn;
8384         unsigned long flags;
8385         /* find the first valid pfn */
8386         for (pfn = start_pfn; pfn < end_pfn; pfn++)
8387                 if (pfn_valid(pfn))
8388                         break;
8389         if (pfn == end_pfn)
8390                 return;
8391         offline_mem_sections(pfn, end_pfn);
8392         zone = page_zone(pfn_to_page(pfn));
8393         spin_lock_irqsave(&zone->lock, flags);
8394         pfn = start_pfn;
8395         while (pfn < end_pfn) {
8396                 if (!pfn_valid(pfn)) {
8397                         pfn++;
8398                         continue;
8399                 }
8400                 page = pfn_to_page(pfn);
8401                 /*
8402                  * The HWPoisoned page may be not in buddy system, and
8403                  * page_count() is not 0.
8404                  */
8405                 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
8406                         pfn++;
8407                         SetPageReserved(page);
8408                         continue;
8409                 }
8410 
8411                 BUG_ON(page_count(page));
8412                 BUG_ON(!PageBuddy(page));
8413                 order = page_order(page);
8414 #ifdef CONFIG_DEBUG_VM
8415                 pr_info("remove from free list %lx %d %lx\n",
8416                         pfn, 1 << order, end_pfn);
8417 #endif
8418                 list_del(&page->lru);
8419                 rmv_page_order(page);
8420                 zone->free_area[order].nr_free--;
8421                 for (i = 0; i < (1 << order); i++)
8422                         SetPageReserved((page+i));
8423                 pfn += (1 << order);
8424         }
8425         spin_unlock_irqrestore(&zone->lock, flags);
8426 }
8427 #endif
8428 
8429 bool is_free_buddy_page(struct page *page)
8430 {
8431         struct zone *zone = page_zone(page);
8432         unsigned long pfn = page_to_pfn(page);
8433         unsigned long flags;
8434         unsigned int order;
8435 
8436         spin_lock_irqsave(&zone->lock, flags);
8437         for (order = 0; order < MAX_ORDER; order++) {
8438                 struct page *page_head = page - (pfn & ((1 << order) - 1));
8439 
8440                 if (PageBuddy(page_head) && page_order(page_head) >= order)
8441                         break;
8442         }
8443         spin_unlock_irqrestore(&zone->lock, flags);
8444 
8445         return order < MAX_ORDER;
8446 }
8447 
8448 #ifdef CONFIG_MEMORY_FAILURE
8449 /*
8450  * Set PG_hwpoison flag if a given page is confirmed to be a free page.  This
8451  * test is performed under the zone lock to prevent a race against page
8452  * allocation.
8453  */
8454 bool set_hwpoison_free_buddy_page(struct page *page)
8455 {
8456         struct zone *zone = page_zone(page);
8457         unsigned long pfn = page_to_pfn(page);
8458         unsigned long flags;
8459         unsigned int order;
8460         bool hwpoisoned = false;
8461 
8462