~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/nobootmem.c

Version: ~ [ linux-5.13-rc1 ] ~ [ linux-5.12.2 ] ~ [ linux-5.11.19 ] ~ [ linux-5.10.35 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.117 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.190 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.232 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.268 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.268 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *  bootmem - A boot-time physical memory allocator and configurator
  3  *
  4  *  Copyright (C) 1999 Ingo Molnar
  5  *                1999 Kanoj Sarcar, SGI
  6  *                2008 Johannes Weiner
  7  *
  8  * Access to this subsystem has to be serialized externally (which is true
  9  * for the boot process anyway).
 10  */
 11 #include <linux/init.h>
 12 #include <linux/pfn.h>
 13 #include <linux/slab.h>
 14 #include <linux/bootmem.h>
 15 #include <linux/export.h>
 16 #include <linux/kmemleak.h>
 17 #include <linux/range.h>
 18 #include <linux/memblock.h>
 19 
 20 #include <asm/bug.h>
 21 #include <asm/io.h>
 22 #include <asm/processor.h>
 23 
 24 #include "internal.h"
 25 
 26 #ifndef CONFIG_NEED_MULTIPLE_NODES
 27 struct pglist_data __refdata contig_page_data;
 28 EXPORT_SYMBOL(contig_page_data);
 29 #endif
 30 
 31 unsigned long max_low_pfn;
 32 unsigned long min_low_pfn;
 33 unsigned long max_pfn;
 34 
 35 static void * __init __alloc_memory_core_early(int nid, u64 size, u64 align,
 36                                         u64 goal, u64 limit)
 37 {
 38         void *ptr;
 39         u64 addr;
 40         ulong flags = choose_memblock_flags();
 41 
 42         if (limit > memblock.current_limit)
 43                 limit = memblock.current_limit;
 44 
 45 again:
 46         addr = memblock_find_in_range_node(size, align, goal, limit, nid,
 47                                            flags);
 48         if (!addr && (flags & MEMBLOCK_MIRROR)) {
 49                 flags &= ~MEMBLOCK_MIRROR;
 50                 pr_warn("Could not allocate %pap bytes of mirrored memory\n",
 51                         &size);
 52                 goto again;
 53         }
 54         if (!addr)
 55                 return NULL;
 56 
 57         if (memblock_reserve(addr, size))
 58                 return NULL;
 59 
 60         ptr = phys_to_virt(addr);
 61         memset(ptr, 0, size);
 62         /*
 63          * The min_count is set to 0 so that bootmem allocated blocks
 64          * are never reported as leaks.
 65          */
 66         kmemleak_alloc(ptr, size, 0, 0);
 67         return ptr;
 68 }
 69 
 70 /*
 71  * free_bootmem_late - free bootmem pages directly to page allocator
 72  * @addr: starting address of the range
 73  * @size: size of the range in bytes
 74  *
 75  * This is only useful when the bootmem allocator has already been torn
 76  * down, but we are still initializing the system.  Pages are given directly
 77  * to the page allocator, no bootmem metadata is updated because it is gone.
 78  */
 79 void __init free_bootmem_late(unsigned long addr, unsigned long size)
 80 {
 81         unsigned long cursor, end;
 82 
 83         kmemleak_free_part(__va(addr), size);
 84 
 85         cursor = PFN_UP(addr);
 86         end = PFN_DOWN(addr + size);
 87 
 88         for (; cursor < end; cursor++) {
 89                 __free_pages_bootmem(pfn_to_page(cursor), cursor, 0);
 90                 totalram_pages++;
 91         }
 92 }
 93 
 94 static void __init __free_pages_memory(unsigned long start, unsigned long end)
 95 {
 96         int order;
 97 
 98         while (start < end) {
 99                 order = min(MAX_ORDER - 1UL, __ffs(start));
100 
101                 while (start + (1UL << order) > end)
102                         order--;
103 
104                 __free_pages_bootmem(pfn_to_page(start), start, order);
105 
106                 start += (1UL << order);
107         }
108 }
109 
110 static unsigned long __init __free_memory_core(phys_addr_t start,
111                                  phys_addr_t end)
112 {
113         unsigned long start_pfn = PFN_UP(start);
114         unsigned long end_pfn = min_t(unsigned long,
115                                       PFN_DOWN(end), max_low_pfn);
116 
117         if (start_pfn > end_pfn)
118                 return 0;
119 
120         __free_pages_memory(start_pfn, end_pfn);
121 
122         return end_pfn - start_pfn;
123 }
124 
125 static unsigned long __init free_low_memory_core_early(void)
126 {
127         unsigned long count = 0;
128         phys_addr_t start, end;
129         u64 i;
130 
131         memblock_clear_hotplug(0, -1);
132 
133         for_each_reserved_mem_region(i, &start, &end)
134                 reserve_bootmem_region(start, end);
135 
136         for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end,
137                                 NULL)
138                 count += __free_memory_core(start, end);
139 
140 #ifdef CONFIG_ARCH_DISCARD_MEMBLOCK
141         {
142                 phys_addr_t size;
143 
144                 /* Free memblock.reserved array if it was allocated */
145                 size = get_allocated_memblock_reserved_regions_info(&start);
146                 if (size)
147                         count += __free_memory_core(start, start + size);
148 
149                 /* Free memblock.memory array if it was allocated */
150                 size = get_allocated_memblock_memory_regions_info(&start);
151                 if (size)
152                         count += __free_memory_core(start, start + size);
153         }
154 #endif
155 
156         return count;
157 }
158 
159 static int reset_managed_pages_done __initdata;
160 
161 void reset_node_managed_pages(pg_data_t *pgdat)
162 {
163         struct zone *z;
164 
165         for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
166                 z->managed_pages = 0;
167 }
168 
169 void __init reset_all_zones_managed_pages(void)
170 {
171         struct pglist_data *pgdat;
172 
173         if (reset_managed_pages_done)
174                 return;
175 
176         for_each_online_pgdat(pgdat)
177                 reset_node_managed_pages(pgdat);
178 
179         reset_managed_pages_done = 1;
180 }
181 
182 /**
183  * free_all_bootmem - release free pages to the buddy allocator
184  *
185  * Returns the number of pages actually released.
186  */
187 unsigned long __init free_all_bootmem(void)
188 {
189         unsigned long pages;
190 
191         reset_all_zones_managed_pages();
192 
193         /*
194          * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id
195          *  because in some case like Node0 doesn't have RAM installed
196          *  low ram will be on Node1
197          */
198         pages = free_low_memory_core_early();
199         totalram_pages += pages;
200 
201         return pages;
202 }
203 
204 /**
205  * free_bootmem_node - mark a page range as usable
206  * @pgdat: node the range resides on
207  * @physaddr: starting address of the range
208  * @size: size of the range in bytes
209  *
210  * Partial pages will be considered reserved and left as they are.
211  *
212  * The range must reside completely on the specified node.
213  */
214 void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
215                               unsigned long size)
216 {
217         memblock_free(physaddr, size);
218 }
219 
220 /**
221  * free_bootmem - mark a page range as usable
222  * @addr: starting address of the range
223  * @size: size of the range in bytes
224  *
225  * Partial pages will be considered reserved and left as they are.
226  *
227  * The range must be contiguous but may span node boundaries.
228  */
229 void __init free_bootmem(unsigned long addr, unsigned long size)
230 {
231         memblock_free(addr, size);
232 }
233 
234 static void * __init ___alloc_bootmem_nopanic(unsigned long size,
235                                         unsigned long align,
236                                         unsigned long goal,
237                                         unsigned long limit)
238 {
239         void *ptr;
240 
241         if (WARN_ON_ONCE(slab_is_available()))
242                 return kzalloc(size, GFP_NOWAIT);
243 
244 restart:
245 
246         ptr = __alloc_memory_core_early(NUMA_NO_NODE, size, align, goal, limit);
247 
248         if (ptr)
249                 return ptr;
250 
251         if (goal != 0) {
252                 goal = 0;
253                 goto restart;
254         }
255 
256         return NULL;
257 }
258 
259 /**
260  * __alloc_bootmem_nopanic - allocate boot memory without panicking
261  * @size: size of the request in bytes
262  * @align: alignment of the region
263  * @goal: preferred starting address of the region
264  *
265  * The goal is dropped if it can not be satisfied and the allocation will
266  * fall back to memory below @goal.
267  *
268  * Allocation may happen on any node in the system.
269  *
270  * Returns NULL on failure.
271  */
272 void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
273                                         unsigned long goal)
274 {
275         unsigned long limit = -1UL;
276 
277         return ___alloc_bootmem_nopanic(size, align, goal, limit);
278 }
279 
280 static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
281                                         unsigned long goal, unsigned long limit)
282 {
283         void *mem = ___alloc_bootmem_nopanic(size, align, goal, limit);
284 
285         if (mem)
286                 return mem;
287         /*
288          * Whoops, we cannot satisfy the allocation request.
289          */
290         printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
291         panic("Out of memory");
292         return NULL;
293 }
294 
295 /**
296  * __alloc_bootmem - allocate boot memory
297  * @size: size of the request in bytes
298  * @align: alignment of the region
299  * @goal: preferred starting address of the region
300  *
301  * The goal is dropped if it can not be satisfied and the allocation will
302  * fall back to memory below @goal.
303  *
304  * Allocation may happen on any node in the system.
305  *
306  * The function panics if the request can not be satisfied.
307  */
308 void * __init __alloc_bootmem(unsigned long size, unsigned long align,
309                               unsigned long goal)
310 {
311         unsigned long limit = -1UL;
312 
313         return ___alloc_bootmem(size, align, goal, limit);
314 }
315 
316 void * __init ___alloc_bootmem_node_nopanic(pg_data_t *pgdat,
317                                                    unsigned long size,
318                                                    unsigned long align,
319                                                    unsigned long goal,
320                                                    unsigned long limit)
321 {
322         void *ptr;
323 
324 again:
325         ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
326                                         goal, limit);
327         if (ptr)
328                 return ptr;
329 
330         ptr = __alloc_memory_core_early(NUMA_NO_NODE, size, align,
331                                         goal, limit);
332         if (ptr)
333                 return ptr;
334 
335         if (goal) {
336                 goal = 0;
337                 goto again;
338         }
339 
340         return NULL;
341 }
342 
343 void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
344                                    unsigned long align, unsigned long goal)
345 {
346         if (WARN_ON_ONCE(slab_is_available()))
347                 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
348 
349         return ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, 0);
350 }
351 
352 static void * __init ___alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
353                                     unsigned long align, unsigned long goal,
354                                     unsigned long limit)
355 {
356         void *ptr;
357 
358         ptr = ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, limit);
359         if (ptr)
360                 return ptr;
361 
362         printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
363         panic("Out of memory");
364         return NULL;
365 }
366 
367 /**
368  * __alloc_bootmem_node - allocate boot memory from a specific node
369  * @pgdat: node to allocate from
370  * @size: size of the request in bytes
371  * @align: alignment of the region
372  * @goal: preferred starting address of the region
373  *
374  * The goal is dropped if it can not be satisfied and the allocation will
375  * fall back to memory below @goal.
376  *
377  * Allocation may fall back to any node in the system if the specified node
378  * can not hold the requested memory.
379  *
380  * The function panics if the request can not be satisfied.
381  */
382 void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
383                                    unsigned long align, unsigned long goal)
384 {
385         if (WARN_ON_ONCE(slab_is_available()))
386                 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
387 
388         return ___alloc_bootmem_node(pgdat, size, align, goal, 0);
389 }
390 
391 void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
392                                    unsigned long align, unsigned long goal)
393 {
394         return __alloc_bootmem_node(pgdat, size, align, goal);
395 }
396 
397 #ifndef ARCH_LOW_ADDRESS_LIMIT
398 #define ARCH_LOW_ADDRESS_LIMIT  0xffffffffUL
399 #endif
400 
401 /**
402  * __alloc_bootmem_low - allocate low boot memory
403  * @size: size of the request in bytes
404  * @align: alignment of the region
405  * @goal: preferred starting address of the region
406  *
407  * The goal is dropped if it can not be satisfied and the allocation will
408  * fall back to memory below @goal.
409  *
410  * Allocation may happen on any node in the system.
411  *
412  * The function panics if the request can not be satisfied.
413  */
414 void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
415                                   unsigned long goal)
416 {
417         return ___alloc_bootmem(size, align, goal, ARCH_LOW_ADDRESS_LIMIT);
418 }
419 
420 void * __init __alloc_bootmem_low_nopanic(unsigned long size,
421                                           unsigned long align,
422                                           unsigned long goal)
423 {
424         return ___alloc_bootmem_nopanic(size, align, goal,
425                                         ARCH_LOW_ADDRESS_LIMIT);
426 }
427 
428 /**
429  * __alloc_bootmem_low_node - allocate low boot memory from a specific node
430  * @pgdat: node to allocate from
431  * @size: size of the request in bytes
432  * @align: alignment of the region
433  * @goal: preferred starting address of the region
434  *
435  * The goal is dropped if it can not be satisfied and the allocation will
436  * fall back to memory below @goal.
437  *
438  * Allocation may fall back to any node in the system if the specified node
439  * can not hold the requested memory.
440  *
441  * The function panics if the request can not be satisfied.
442  */
443 void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
444                                        unsigned long align, unsigned long goal)
445 {
446         if (WARN_ON_ONCE(slab_is_available()))
447                 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
448 
449         return ___alloc_bootmem_node(pgdat, size, align, goal,
450                                      ARCH_LOW_ADDRESS_LIMIT);
451 }
452 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp