~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm/mm/init.c

Version: ~ [ linux-5.4-rc3 ] ~ [ linux-5.3.6 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.79 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.149 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.196 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.196 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.75 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *  linux/arch/arm/mm/init.c
  3  *
  4  *  Copyright (C) 1995-2005 Russell King
  5  *
  6  * This program is free software; you can redistribute it and/or modify
  7  * it under the terms of the GNU General Public License version 2 as
  8  * published by the Free Software Foundation.
  9  */
 10 #include <linux/kernel.h>
 11 #include <linux/errno.h>
 12 #include <linux/swap.h>
 13 #include <linux/init.h>
 14 #include <linux/bootmem.h>
 15 #include <linux/mman.h>
 16 #include <linux/export.h>
 17 #include <linux/nodemask.h>
 18 #include <linux/initrd.h>
 19 #include <linux/of_fdt.h>
 20 #include <linux/highmem.h>
 21 #include <linux/gfp.h>
 22 #include <linux/memblock.h>
 23 #include <linux/dma-contiguous.h>
 24 #include <linux/sizes.h>
 25 #include <linux/stop_machine.h>
 26 
 27 #include <asm/cp15.h>
 28 #include <asm/mach-types.h>
 29 #include <asm/memblock.h>
 30 #include <asm/prom.h>
 31 #include <asm/sections.h>
 32 #include <asm/setup.h>
 33 #include <asm/system_info.h>
 34 #include <asm/tlb.h>
 35 #include <asm/fixmap.h>
 36 
 37 #include <asm/mach/arch.h>
 38 #include <asm/mach/map.h>
 39 
 40 #include "mm.h"
 41 
 42 #ifdef CONFIG_CPU_CP15_MMU
 43 unsigned long __init __clear_cr(unsigned long mask)
 44 {
 45         cr_alignment = cr_alignment & ~mask;
 46         return cr_alignment;
 47 }
 48 #endif
 49 
 50 static phys_addr_t phys_initrd_start __initdata = 0;
 51 static unsigned long phys_initrd_size __initdata = 0;
 52 
 53 static int __init early_initrd(char *p)
 54 {
 55         phys_addr_t start;
 56         unsigned long size;
 57         char *endp;
 58 
 59         start = memparse(p, &endp);
 60         if (*endp == ',') {
 61                 size = memparse(endp + 1, NULL);
 62 
 63                 phys_initrd_start = start;
 64                 phys_initrd_size = size;
 65         }
 66         return 0;
 67 }
 68 early_param("initrd", early_initrd);
 69 
 70 static int __init parse_tag_initrd(const struct tag *tag)
 71 {
 72         pr_warn("ATAG_INITRD is deprecated; "
 73                 "please update your bootloader.\n");
 74         phys_initrd_start = __virt_to_phys(tag->u.initrd.start);
 75         phys_initrd_size = tag->u.initrd.size;
 76         return 0;
 77 }
 78 
 79 __tagtable(ATAG_INITRD, parse_tag_initrd);
 80 
 81 static int __init parse_tag_initrd2(const struct tag *tag)
 82 {
 83         phys_initrd_start = tag->u.initrd.start;
 84         phys_initrd_size = tag->u.initrd.size;
 85         return 0;
 86 }
 87 
 88 __tagtable(ATAG_INITRD2, parse_tag_initrd2);
 89 
 90 static void __init find_limits(unsigned long *min, unsigned long *max_low,
 91                                unsigned long *max_high)
 92 {
 93         *max_low = PFN_DOWN(memblock_get_current_limit());
 94         *min = PFN_UP(memblock_start_of_DRAM());
 95         *max_high = PFN_DOWN(memblock_end_of_DRAM());
 96 }
 97 
 98 #ifdef CONFIG_ZONE_DMA
 99 
100 phys_addr_t arm_dma_zone_size __read_mostly;
101 EXPORT_SYMBOL(arm_dma_zone_size);
102 
103 /*
104  * The DMA mask corresponding to the maximum bus address allocatable
105  * using GFP_DMA.  The default here places no restriction on DMA
106  * allocations.  This must be the smallest DMA mask in the system,
107  * so a successful GFP_DMA allocation will always satisfy this.
108  */
109 phys_addr_t arm_dma_limit;
110 unsigned long arm_dma_pfn_limit;
111 
112 static void __init arm_adjust_dma_zone(unsigned long *size, unsigned long *hole,
113         unsigned long dma_size)
114 {
115         if (size[0] <= dma_size)
116                 return;
117 
118         size[ZONE_NORMAL] = size[0] - dma_size;
119         size[ZONE_DMA] = dma_size;
120         hole[ZONE_NORMAL] = hole[0];
121         hole[ZONE_DMA] = 0;
122 }
123 #endif
124 
125 void __init setup_dma_zone(const struct machine_desc *mdesc)
126 {
127 #ifdef CONFIG_ZONE_DMA
128         if (mdesc->dma_zone_size) {
129                 arm_dma_zone_size = mdesc->dma_zone_size;
130                 arm_dma_limit = PHYS_OFFSET + arm_dma_zone_size - 1;
131         } else
132                 arm_dma_limit = 0xffffffff;
133         arm_dma_pfn_limit = arm_dma_limit >> PAGE_SHIFT;
134 #endif
135 }
136 
137 static void __init zone_sizes_init(unsigned long min, unsigned long max_low,
138         unsigned long max_high)
139 {
140         unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES];
141         struct memblock_region *reg;
142 
143         /*
144          * initialise the zones.
145          */
146         memset(zone_size, 0, sizeof(zone_size));
147 
148         /*
149          * The memory size has already been determined.  If we need
150          * to do anything fancy with the allocation of this memory
151          * to the zones, now is the time to do it.
152          */
153         zone_size[0] = max_low - min;
154 #ifdef CONFIG_HIGHMEM
155         zone_size[ZONE_HIGHMEM] = max_high - max_low;
156 #endif
157 
158         /*
159          * Calculate the size of the holes.
160          *  holes = node_size - sum(bank_sizes)
161          */
162         memcpy(zhole_size, zone_size, sizeof(zhole_size));
163         for_each_memblock(memory, reg) {
164                 unsigned long start = memblock_region_memory_base_pfn(reg);
165                 unsigned long end = memblock_region_memory_end_pfn(reg);
166 
167                 if (start < max_low) {
168                         unsigned long low_end = min(end, max_low);
169                         zhole_size[0] -= low_end - start;
170                 }
171 #ifdef CONFIG_HIGHMEM
172                 if (end > max_low) {
173                         unsigned long high_start = max(start, max_low);
174                         zhole_size[ZONE_HIGHMEM] -= end - high_start;
175                 }
176 #endif
177         }
178 
179 #ifdef CONFIG_ZONE_DMA
180         /*
181          * Adjust the sizes according to any special requirements for
182          * this machine type.
183          */
184         if (arm_dma_zone_size)
185                 arm_adjust_dma_zone(zone_size, zhole_size,
186                         arm_dma_zone_size >> PAGE_SHIFT);
187 #endif
188 
189         free_area_init_node(0, zone_size, min, zhole_size);
190 }
191 
192 #ifdef CONFIG_HAVE_ARCH_PFN_VALID
193 int pfn_valid(unsigned long pfn)
194 {
195         return memblock_is_map_memory(__pfn_to_phys(pfn));
196 }
197 EXPORT_SYMBOL(pfn_valid);
198 #endif
199 
200 #ifndef CONFIG_SPARSEMEM
201 static void __init arm_memory_present(void)
202 {
203 }
204 #else
205 static void __init arm_memory_present(void)
206 {
207         struct memblock_region *reg;
208 
209         for_each_memblock(memory, reg)
210                 memory_present(0, memblock_region_memory_base_pfn(reg),
211                                memblock_region_memory_end_pfn(reg));
212 }
213 #endif
214 
215 static bool arm_memblock_steal_permitted = true;
216 
217 phys_addr_t __init arm_memblock_steal(phys_addr_t size, phys_addr_t align)
218 {
219         phys_addr_t phys;
220 
221         BUG_ON(!arm_memblock_steal_permitted);
222 
223         phys = memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ANYWHERE);
224         memblock_free(phys, size);
225         memblock_remove(phys, size);
226 
227         return phys;
228 }
229 
230 void __init arm_memblock_init(const struct machine_desc *mdesc)
231 {
232         /* Register the kernel text, kernel data and initrd with memblock. */
233 #ifdef CONFIG_XIP_KERNEL
234         memblock_reserve(__pa(_sdata), _end - _sdata);
235 #else
236         memblock_reserve(__pa(_stext), _end - _stext);
237 #endif
238 #ifdef CONFIG_BLK_DEV_INITRD
239         /* FDT scan will populate initrd_start */
240         if (initrd_start && !phys_initrd_size) {
241                 phys_initrd_start = __virt_to_phys(initrd_start);
242                 phys_initrd_size = initrd_end - initrd_start;
243         }
244         initrd_start = initrd_end = 0;
245         if (phys_initrd_size &&
246             !memblock_is_region_memory(phys_initrd_start, phys_initrd_size)) {
247                 pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region - disabling initrd\n",
248                        (u64)phys_initrd_start, phys_initrd_size);
249                 phys_initrd_start = phys_initrd_size = 0;
250         }
251         if (phys_initrd_size &&
252             memblock_is_region_reserved(phys_initrd_start, phys_initrd_size)) {
253                 pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region - disabling initrd\n",
254                        (u64)phys_initrd_start, phys_initrd_size);
255                 phys_initrd_start = phys_initrd_size = 0;
256         }
257         if (phys_initrd_size) {
258                 memblock_reserve(phys_initrd_start, phys_initrd_size);
259 
260                 /* Now convert initrd to virtual addresses */
261                 initrd_start = __phys_to_virt(phys_initrd_start);
262                 initrd_end = initrd_start + phys_initrd_size;
263         }
264 #endif
265 
266         arm_mm_memblock_reserve();
267 
268         /* reserve any platform specific memblock areas */
269         if (mdesc->reserve)
270                 mdesc->reserve();
271 
272         early_init_fdt_reserve_self();
273         early_init_fdt_scan_reserved_mem();
274 
275         /* reserve memory for DMA contiguous allocations */
276         dma_contiguous_reserve(arm_dma_limit);
277 
278         arm_memblock_steal_permitted = false;
279         memblock_dump_all();
280 }
281 
282 void __init bootmem_init(void)
283 {
284         unsigned long min, max_low, max_high;
285 
286         memblock_allow_resize();
287         max_low = max_high = 0;
288 
289         find_limits(&min, &max_low, &max_high);
290 
291         early_memtest((phys_addr_t)min << PAGE_SHIFT,
292                       (phys_addr_t)max_low << PAGE_SHIFT);
293 
294         /*
295          * Sparsemem tries to allocate bootmem in memory_present(),
296          * so must be done after the fixed reservations
297          */
298         arm_memory_present();
299 
300         /*
301          * sparse_init() needs the bootmem allocator up and running.
302          */
303         sparse_init();
304 
305         /*
306          * Now free the memory - free_area_init_node needs
307          * the sparse mem_map arrays initialized by sparse_init()
308          * for memmap_init_zone(), otherwise all PFNs are invalid.
309          */
310         zone_sizes_init(min, max_low, max_high);
311 
312         /*
313          * This doesn't seem to be used by the Linux memory manager any
314          * more, but is used by ll_rw_block.  If we can get rid of it, we
315          * also get rid of some of the stuff above as well.
316          */
317         min_low_pfn = min;
318         max_low_pfn = max_low;
319         max_pfn = max_high;
320 }
321 
322 /*
323  * Poison init memory with an undefined instruction (ARM) or a branch to an
324  * undefined instruction (Thumb).
325  */
326 static inline void poison_init_mem(void *s, size_t count)
327 {
328         u32 *p = (u32 *)s;
329         for (; count != 0; count -= 4)
330                 *p++ = 0xe7fddef0;
331 }
332 
333 static inline void
334 free_memmap(unsigned long start_pfn, unsigned long end_pfn)
335 {
336         struct page *start_pg, *end_pg;
337         phys_addr_t pg, pgend;
338 
339         /*
340          * Convert start_pfn/end_pfn to a struct page pointer.
341          */
342         start_pg = pfn_to_page(start_pfn - 1) + 1;
343         end_pg = pfn_to_page(end_pfn - 1) + 1;
344 
345         /*
346          * Convert to physical addresses, and
347          * round start upwards and end downwards.
348          */
349         pg = PAGE_ALIGN(__pa(start_pg));
350         pgend = __pa(end_pg) & PAGE_MASK;
351 
352         /*
353          * If there are free pages between these,
354          * free the section of the memmap array.
355          */
356         if (pg < pgend)
357                 memblock_free_early(pg, pgend - pg);
358 }
359 
360 /*
361  * The mem_map array can get very big.  Free the unused area of the memory map.
362  */
363 static void __init free_unused_memmap(void)
364 {
365         unsigned long start, prev_end = 0;
366         struct memblock_region *reg;
367 
368         /*
369          * This relies on each bank being in address order.
370          * The banks are sorted previously in bootmem_init().
371          */
372         for_each_memblock(memory, reg) {
373                 start = memblock_region_memory_base_pfn(reg);
374 
375 #ifdef CONFIG_SPARSEMEM
376                 /*
377                  * Take care not to free memmap entries that don't exist
378                  * due to SPARSEMEM sections which aren't present.
379                  */
380                 start = min(start,
381                                  ALIGN(prev_end, PAGES_PER_SECTION));
382 #else
383                 /*
384                  * Align down here since the VM subsystem insists that the
385                  * memmap entries are valid from the bank start aligned to
386                  * MAX_ORDER_NR_PAGES.
387                  */
388                 start = round_down(start, MAX_ORDER_NR_PAGES);
389 #endif
390                 /*
391                  * If we had a previous bank, and there is a space
392                  * between the current bank and the previous, free it.
393                  */
394                 if (prev_end && prev_end < start)
395                         free_memmap(prev_end, start);
396 
397                 /*
398                  * Align up here since the VM subsystem insists that the
399                  * memmap entries are valid from the bank end aligned to
400                  * MAX_ORDER_NR_PAGES.
401                  */
402                 prev_end = ALIGN(memblock_region_memory_end_pfn(reg),
403                                  MAX_ORDER_NR_PAGES);
404         }
405 
406 #ifdef CONFIG_SPARSEMEM
407         if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION))
408                 free_memmap(prev_end,
409                             ALIGN(prev_end, PAGES_PER_SECTION));
410 #endif
411 }
412 
413 #ifdef CONFIG_HIGHMEM
414 static inline void free_area_high(unsigned long pfn, unsigned long end)
415 {
416         for (; pfn < end; pfn++)
417                 free_highmem_page(pfn_to_page(pfn));
418 }
419 #endif
420 
421 static void __init free_highpages(void)
422 {
423 #ifdef CONFIG_HIGHMEM
424         unsigned long max_low = max_low_pfn;
425         struct memblock_region *mem, *res;
426 
427         /* set highmem page free */
428         for_each_memblock(memory, mem) {
429                 unsigned long start = memblock_region_memory_base_pfn(mem);
430                 unsigned long end = memblock_region_memory_end_pfn(mem);
431 
432                 /* Ignore complete lowmem entries */
433                 if (end <= max_low)
434                         continue;
435 
436                 if (memblock_is_nomap(mem))
437                         continue;
438 
439                 /* Truncate partial highmem entries */
440                 if (start < max_low)
441                         start = max_low;
442 
443                 /* Find and exclude any reserved regions */
444                 for_each_memblock(reserved, res) {
445                         unsigned long res_start, res_end;
446 
447                         res_start = memblock_region_reserved_base_pfn(res);
448                         res_end = memblock_region_reserved_end_pfn(res);
449 
450                         if (res_end < start)
451                                 continue;
452                         if (res_start < start)
453                                 res_start = start;
454                         if (res_start > end)
455                                 res_start = end;
456                         if (res_end > end)
457                                 res_end = end;
458                         if (res_start != start)
459                                 free_area_high(start, res_start);
460                         start = res_end;
461                         if (start == end)
462                                 break;
463                 }
464 
465                 /* And now free anything which remains */
466                 if (start < end)
467                         free_area_high(start, end);
468         }
469 #endif
470 }
471 
472 /*
473  * mem_init() marks the free areas in the mem_map and tells us how much
474  * memory is free.  This is done after various parts of the system have
475  * claimed their memory after the kernel image.
476  */
477 void __init mem_init(void)
478 {
479 #ifdef CONFIG_HAVE_TCM
480         /* These pointers are filled in on TCM detection */
481         extern u32 dtcm_end;
482         extern u32 itcm_end;
483 #endif
484 
485         set_max_mapnr(pfn_to_page(max_pfn) - mem_map);
486 
487         /* this will put all unused low memory onto the freelists */
488         free_unused_memmap();
489         free_all_bootmem();
490 
491 #ifdef CONFIG_SA1111
492         /* now that our DMA memory is actually so designated, we can free it */
493         free_reserved_area(__va(PHYS_OFFSET), swapper_pg_dir, -1, NULL);
494 #endif
495 
496         free_highpages();
497 
498         mem_init_print_info(NULL);
499 
500 #define MLK(b, t) b, t, ((t) - (b)) >> 10
501 #define MLM(b, t) b, t, ((t) - (b)) >> 20
502 #define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K)
503 
504         pr_notice("Virtual kernel memory layout:\n"
505                         "    vector  : 0x%08lx - 0x%08lx   (%4ld kB)\n"
506 #ifdef CONFIG_HAVE_TCM
507                         "    DTCM    : 0x%08lx - 0x%08lx   (%4ld kB)\n"
508                         "    ITCM    : 0x%08lx - 0x%08lx   (%4ld kB)\n"
509 #endif
510                         "    fixmap  : 0x%08lx - 0x%08lx   (%4ld kB)\n"
511                         "    vmalloc : 0x%08lx - 0x%08lx   (%4ld MB)\n"
512                         "    lowmem  : 0x%08lx - 0x%08lx   (%4ld MB)\n"
513 #ifdef CONFIG_HIGHMEM
514                         "    pkmap   : 0x%08lx - 0x%08lx   (%4ld MB)\n"
515 #endif
516 #ifdef CONFIG_MODULES
517                         "    modules : 0x%08lx - 0x%08lx   (%4ld MB)\n"
518 #endif
519                         "      .text : 0x%p" " - 0x%p" "   (%4td kB)\n"
520                         "      .init : 0x%p" " - 0x%p" "   (%4td kB)\n"
521                         "      .data : 0x%p" " - 0x%p" "   (%4td kB)\n"
522                         "       .bss : 0x%p" " - 0x%p" "   (%4td kB)\n",
523 
524                         MLK(UL(CONFIG_VECTORS_BASE), UL(CONFIG_VECTORS_BASE) +
525                                 (PAGE_SIZE)),
526 #ifdef CONFIG_HAVE_TCM
527                         MLK(DTCM_OFFSET, (unsigned long) dtcm_end),
528                         MLK(ITCM_OFFSET, (unsigned long) itcm_end),
529 #endif
530                         MLK(FIXADDR_START, FIXADDR_END),
531                         MLM(VMALLOC_START, VMALLOC_END),
532                         MLM(PAGE_OFFSET, (unsigned long)high_memory),
533 #ifdef CONFIG_HIGHMEM
534                         MLM(PKMAP_BASE, (PKMAP_BASE) + (LAST_PKMAP) *
535                                 (PAGE_SIZE)),
536 #endif
537 #ifdef CONFIG_MODULES
538                         MLM(MODULES_VADDR, MODULES_END),
539 #endif
540 
541                         MLK_ROUNDUP(_text, _etext),
542                         MLK_ROUNDUP(__init_begin, __init_end),
543                         MLK_ROUNDUP(_sdata, _edata),
544                         MLK_ROUNDUP(__bss_start, __bss_stop));
545 
546 #undef MLK
547 #undef MLM
548 #undef MLK_ROUNDUP
549 
550         /*
551          * Check boundaries twice: Some fundamental inconsistencies can
552          * be detected at build time already.
553          */
554 #ifdef CONFIG_MMU
555         BUILD_BUG_ON(TASK_SIZE                          > MODULES_VADDR);
556         BUG_ON(TASK_SIZE                                > MODULES_VADDR);
557 #endif
558 
559 #ifdef CONFIG_HIGHMEM
560         BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET);
561         BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE      > PAGE_OFFSET);
562 #endif
563 
564         if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) {
565                 extern int sysctl_overcommit_memory;
566                 /*
567                  * On a machine this small we won't get
568                  * anywhere without overcommit, so turn
569                  * it on by default.
570                  */
571                 sysctl_overcommit_memory = OVERCOMMIT_ALWAYS;
572         }
573 }
574 
575 #ifdef CONFIG_DEBUG_RODATA
576 struct section_perm {
577         const char *name;
578         unsigned long start;
579         unsigned long end;
580         pmdval_t mask;
581         pmdval_t prot;
582         pmdval_t clear;
583 };
584 
585 /* First section-aligned location at or after __start_rodata. */
586 extern char __start_rodata_section_aligned[];
587 
588 static struct section_perm nx_perms[] = {
589         /* Make pages tables, etc before _stext RW (set NX). */
590         {
591                 .name   = "pre-text NX",
592                 .start  = PAGE_OFFSET,
593                 .end    = (unsigned long)_stext,
594                 .mask   = ~PMD_SECT_XN,
595                 .prot   = PMD_SECT_XN,
596         },
597         /* Make init RW (set NX). */
598         {
599                 .name   = "init NX",
600                 .start  = (unsigned long)__init_begin,
601                 .end    = (unsigned long)_sdata,
602                 .mask   = ~PMD_SECT_XN,
603                 .prot   = PMD_SECT_XN,
604         },
605         /* Make rodata NX (set RO in ro_perms below). */
606         {
607                 .name   = "rodata NX",
608                 .start  = (unsigned long)__start_rodata_section_aligned,
609                 .end    = (unsigned long)__init_begin,
610                 .mask   = ~PMD_SECT_XN,
611                 .prot   = PMD_SECT_XN,
612         },
613 };
614 
615 static struct section_perm ro_perms[] = {
616         /* Make kernel code and rodata RX (set RO). */
617         {
618                 .name   = "text/rodata RO",
619                 .start  = (unsigned long)_stext,
620                 .end    = (unsigned long)__init_begin,
621 #ifdef CONFIG_ARM_LPAE
622                 .mask   = ~L_PMD_SECT_RDONLY,
623                 .prot   = L_PMD_SECT_RDONLY,
624 #else
625                 .mask   = ~(PMD_SECT_APX | PMD_SECT_AP_WRITE),
626                 .prot   = PMD_SECT_APX | PMD_SECT_AP_WRITE,
627                 .clear  = PMD_SECT_AP_WRITE,
628 #endif
629         },
630 };
631 
632 /*
633  * Updates section permissions only for the current mm (sections are
634  * copied into each mm). During startup, this is the init_mm. Is only
635  * safe to be called with preemption disabled, as under stop_machine().
636  */
637 static inline void section_update(unsigned long addr, pmdval_t mask,
638                                   pmdval_t prot, struct mm_struct *mm)
639 {
640         pmd_t *pmd;
641 
642         pmd = pmd_offset(pud_offset(pgd_offset(mm, addr), addr), addr);
643 
644 #ifdef CONFIG_ARM_LPAE
645         pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot);
646 #else
647         if (addr & SECTION_SIZE)
648                 pmd[1] = __pmd((pmd_val(pmd[1]) & mask) | prot);
649         else
650                 pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot);
651 #endif
652         flush_pmd_entry(pmd);
653         local_flush_tlb_kernel_range(addr, addr + SECTION_SIZE);
654 }
655 
656 /* Make sure extended page tables are in use. */
657 static inline bool arch_has_strict_perms(void)
658 {
659         if (cpu_architecture() < CPU_ARCH_ARMv6)
660                 return false;
661 
662         return !!(get_cr() & CR_XP);
663 }
664 
665 void set_section_perms(struct section_perm *perms, int n, bool set,
666                         struct mm_struct *mm)
667 {
668         size_t i;
669         unsigned long addr;
670 
671         if (!arch_has_strict_perms())
672                 return;
673 
674         for (i = 0; i < n; i++) {
675                 if (!IS_ALIGNED(perms[i].start, SECTION_SIZE) ||
676                     !IS_ALIGNED(perms[i].end, SECTION_SIZE)) {
677                         pr_err("BUG: %s section %lx-%lx not aligned to %lx\n",
678                                 perms[i].name, perms[i].start, perms[i].end,
679                                 SECTION_SIZE);
680                         continue;
681                 }
682 
683                 for (addr = perms[i].start;
684                      addr < perms[i].end;
685                      addr += SECTION_SIZE)
686                         section_update(addr, perms[i].mask,
687                                 set ? perms[i].prot : perms[i].clear, mm);
688         }
689 
690 }
691 
692 static void update_sections_early(struct section_perm perms[], int n)
693 {
694         struct task_struct *t, *s;
695 
696         read_lock(&tasklist_lock);
697         for_each_process(t) {
698                 if (t->flags & PF_KTHREAD)
699                         continue;
700                 for_each_thread(t, s)
701                         set_section_perms(perms, n, true, s->mm);
702         }
703         read_unlock(&tasklist_lock);
704         set_section_perms(perms, n, true, current->active_mm);
705         set_section_perms(perms, n, true, &init_mm);
706 }
707 
708 int __fix_kernmem_perms(void *unused)
709 {
710         update_sections_early(nx_perms, ARRAY_SIZE(nx_perms));
711         return 0;
712 }
713 
714 void fix_kernmem_perms(void)
715 {
716         stop_machine(__fix_kernmem_perms, NULL, NULL);
717 }
718 
719 int __mark_rodata_ro(void *unused)
720 {
721         update_sections_early(ro_perms, ARRAY_SIZE(ro_perms));
722         return 0;
723 }
724 
725 void mark_rodata_ro(void)
726 {
727         stop_machine(__mark_rodata_ro, NULL, NULL);
728 }
729 
730 void set_kernel_text_rw(void)
731 {
732         set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), false,
733                                 current->active_mm);
734 }
735 
736 void set_kernel_text_ro(void)
737 {
738         set_section_perms(ro_perms, ARRAY_SIZE(ro_perms), true,
739                                 current->active_mm);
740 }
741 
742 #else
743 static inline void fix_kernmem_perms(void) { }
744 #endif /* CONFIG_DEBUG_RODATA */
745 
746 void free_tcmmem(void)
747 {
748 #ifdef CONFIG_HAVE_TCM
749         extern char __tcm_start, __tcm_end;
750 
751         poison_init_mem(&__tcm_start, &__tcm_end - &__tcm_start);
752         free_reserved_area(&__tcm_start, &__tcm_end, -1, "TCM link");
753 #endif
754 }
755 
756 void free_initmem(void)
757 {
758         fix_kernmem_perms();
759         free_tcmmem();
760 
761         poison_init_mem(__init_begin, __init_end - __init_begin);
762         if (!machine_is_integrator() && !machine_is_cintegrator())
763                 free_initmem_default(-1);
764 }
765 
766 #ifdef CONFIG_BLK_DEV_INITRD
767 
768 static int keep_initrd;
769 
770 void free_initrd_mem(unsigned long start, unsigned long end)
771 {
772         if (!keep_initrd) {
773                 if (start == initrd_start)
774                         start = round_down(start, PAGE_SIZE);
775                 if (end == initrd_end)
776                         end = round_up(end, PAGE_SIZE);
777 
778                 poison_init_mem((void *)start, PAGE_ALIGN(end) - start);
779                 free_reserved_area((void *)start, (void *)end, -1, "initrd");
780         }
781 }
782 
783 static int __init keepinitrd_setup(char *__unused)
784 {
785         keep_initrd = 1;
786         return 1;
787 }
788 
789 __setup("keepinitrd", keepinitrd_setup);
790 #endif
791 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp