~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/powerpc/mm/pgtable-radix.c

Version: ~ [ linux-5.9-rc6 ] ~ [ linux-5.8.10 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.66 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.146 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.198 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.236 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.236 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Page table handling routines for radix page table.
  3  *
  4  * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
  5  *
  6  * This program is free software; you can redistribute it and/or
  7  * modify it under the terms of the GNU General Public License
  8  * as published by the Free Software Foundation; either version
  9  * 2 of the License, or (at your option) any later version.
 10  */
 11 
 12 #define pr_fmt(fmt) "radix-mmu: " fmt
 13 
 14 #include <linux/kernel.h>
 15 #include <linux/sched/mm.h>
 16 #include <linux/memblock.h>
 17 #include <linux/of_fdt.h>
 18 #include <linux/mm.h>
 19 #include <linux/string_helpers.h>
 20 #include <linux/stop_machine.h>
 21 
 22 #include <asm/pgtable.h>
 23 #include <asm/pgalloc.h>
 24 #include <asm/mmu_context.h>
 25 #include <asm/dma.h>
 26 #include <asm/machdep.h>
 27 #include <asm/mmu.h>
 28 #include <asm/firmware.h>
 29 #include <asm/powernv.h>
 30 #include <asm/sections.h>
 31 #include <asm/trace.h>
 32 
 33 #include <trace/events/thp.h>
 34 
 35 unsigned int mmu_pid_bits;
 36 unsigned int mmu_base_pid;
 37 
 38 static int native_register_process_table(unsigned long base, unsigned long pg_sz,
 39                                          unsigned long table_size)
 40 {
 41         unsigned long patb0, patb1;
 42 
 43         patb0 = be64_to_cpu(partition_tb[0].patb0);
 44         patb1 = base | table_size | PATB_GR;
 45 
 46         mmu_partition_table_set_entry(0, patb0, patb1);
 47 
 48         return 0;
 49 }
 50 
 51 static __ref void *early_alloc_pgtable(unsigned long size, int nid,
 52                         unsigned long region_start, unsigned long region_end)
 53 {
 54         unsigned long pa = 0;
 55         void *pt;
 56 
 57         if (region_start || region_end) /* has region hint */
 58                 pa = memblock_alloc_range(size, size, region_start, region_end,
 59                                                 MEMBLOCK_NONE);
 60         else if (nid != -1) /* has node hint */
 61                 pa = memblock_alloc_base_nid(size, size,
 62                                                 MEMBLOCK_ALLOC_ANYWHERE,
 63                                                 nid, MEMBLOCK_NONE);
 64 
 65         if (!pa)
 66                 pa = memblock_alloc_base(size, size, MEMBLOCK_ALLOC_ANYWHERE);
 67 
 68         BUG_ON(!pa);
 69 
 70         pt = __va(pa);
 71         memset(pt, 0, size);
 72 
 73         return pt;
 74 }
 75 
 76 static int early_map_kernel_page(unsigned long ea, unsigned long pa,
 77                           pgprot_t flags,
 78                           unsigned int map_page_size,
 79                           int nid,
 80                           unsigned long region_start, unsigned long region_end)
 81 {
 82         unsigned long pfn = pa >> PAGE_SHIFT;
 83         pgd_t *pgdp;
 84         pud_t *pudp;
 85         pmd_t *pmdp;
 86         pte_t *ptep;
 87 
 88         pgdp = pgd_offset_k(ea);
 89         if (pgd_none(*pgdp)) {
 90                 pudp = early_alloc_pgtable(PUD_TABLE_SIZE, nid,
 91                                                 region_start, region_end);
 92                 pgd_populate(&init_mm, pgdp, pudp);
 93         }
 94         pudp = pud_offset(pgdp, ea);
 95         if (map_page_size == PUD_SIZE) {
 96                 ptep = (pte_t *)pudp;
 97                 goto set_the_pte;
 98         }
 99         if (pud_none(*pudp)) {
100                 pmdp = early_alloc_pgtable(PMD_TABLE_SIZE, nid,
101                                                 region_start, region_end);
102                 pud_populate(&init_mm, pudp, pmdp);
103         }
104         pmdp = pmd_offset(pudp, ea);
105         if (map_page_size == PMD_SIZE) {
106                 ptep = pmdp_ptep(pmdp);
107                 goto set_the_pte;
108         }
109         if (!pmd_present(*pmdp)) {
110                 ptep = early_alloc_pgtable(PAGE_SIZE, nid,
111                                                 region_start, region_end);
112                 pmd_populate_kernel(&init_mm, pmdp, ptep);
113         }
114         ptep = pte_offset_kernel(pmdp, ea);
115 
116 set_the_pte:
117         set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
118         smp_wmb();
119         return 0;
120 }
121 
122 /*
123  * nid, region_start, and region_end are hints to try to place the page
124  * table memory in the same node or region.
125  */
126 static int __map_kernel_page(unsigned long ea, unsigned long pa,
127                           pgprot_t flags,
128                           unsigned int map_page_size,
129                           int nid,
130                           unsigned long region_start, unsigned long region_end)
131 {
132         unsigned long pfn = pa >> PAGE_SHIFT;
133         pgd_t *pgdp;
134         pud_t *pudp;
135         pmd_t *pmdp;
136         pte_t *ptep;
137         /*
138          * Make sure task size is correct as per the max adddr
139          */
140         BUILD_BUG_ON(TASK_SIZE_USER64 > RADIX_PGTABLE_RANGE);
141 
142         if (unlikely(!slab_is_available()))
143                 return early_map_kernel_page(ea, pa, flags, map_page_size,
144                                                 nid, region_start, region_end);
145 
146         /*
147          * Should make page table allocation functions be able to take a
148          * node, so we can place kernel page tables on the right nodes after
149          * boot.
150          */
151         pgdp = pgd_offset_k(ea);
152         pudp = pud_alloc(&init_mm, pgdp, ea);
153         if (!pudp)
154                 return -ENOMEM;
155         if (map_page_size == PUD_SIZE) {
156                 ptep = (pte_t *)pudp;
157                 goto set_the_pte;
158         }
159         pmdp = pmd_alloc(&init_mm, pudp, ea);
160         if (!pmdp)
161                 return -ENOMEM;
162         if (map_page_size == PMD_SIZE) {
163                 ptep = pmdp_ptep(pmdp);
164                 goto set_the_pte;
165         }
166         ptep = pte_alloc_kernel(pmdp, ea);
167         if (!ptep)
168                 return -ENOMEM;
169 
170 set_the_pte:
171         set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
172         smp_wmb();
173         return 0;
174 }
175 
176 int radix__map_kernel_page(unsigned long ea, unsigned long pa,
177                           pgprot_t flags,
178                           unsigned int map_page_size)
179 {
180         return __map_kernel_page(ea, pa, flags, map_page_size, -1, 0, 0);
181 }
182 
183 #ifdef CONFIG_STRICT_KERNEL_RWX
184 void radix__change_memory_range(unsigned long start, unsigned long end,
185                                 unsigned long clear)
186 {
187         unsigned long idx;
188         pgd_t *pgdp;
189         pud_t *pudp;
190         pmd_t *pmdp;
191         pte_t *ptep;
192 
193         start = ALIGN_DOWN(start, PAGE_SIZE);
194         end = PAGE_ALIGN(end); // aligns up
195 
196         pr_debug("Changing flags on range %lx-%lx removing 0x%lx\n",
197                  start, end, clear);
198 
199         for (idx = start; idx < end; idx += PAGE_SIZE) {
200                 pgdp = pgd_offset_k(idx);
201                 pudp = pud_alloc(&init_mm, pgdp, idx);
202                 if (!pudp)
203                         continue;
204                 if (pud_huge(*pudp)) {
205                         ptep = (pte_t *)pudp;
206                         goto update_the_pte;
207                 }
208                 pmdp = pmd_alloc(&init_mm, pudp, idx);
209                 if (!pmdp)
210                         continue;
211                 if (pmd_huge(*pmdp)) {
212                         ptep = pmdp_ptep(pmdp);
213                         goto update_the_pte;
214                 }
215                 ptep = pte_alloc_kernel(pmdp, idx);
216                 if (!ptep)
217                         continue;
218 update_the_pte:
219                 radix__pte_update(&init_mm, idx, ptep, clear, 0, 0);
220         }
221 
222         radix__flush_tlb_kernel_range(start, end);
223 }
224 
225 void radix__mark_rodata_ro(void)
226 {
227         unsigned long start, end;
228 
229         /*
230          * mark_rodata_ro() will mark itself as !writable at some point.
231          * Due to DD1 workaround in radix__pte_update(), we'll end up with
232          * an invalid pte and the system will crash quite severly.
233          */
234         if (cpu_has_feature(CPU_FTR_POWER9_DD1)) {
235                 pr_warn("Warning: Unable to mark rodata read only on P9 DD1\n");
236                 return;
237         }
238 
239         start = (unsigned long)_stext;
240         end = (unsigned long)__init_begin;
241 
242         radix__change_memory_range(start, end, _PAGE_WRITE);
243 }
244 
245 void radix__mark_initmem_nx(void)
246 {
247         unsigned long start = (unsigned long)__init_begin;
248         unsigned long end = (unsigned long)__init_end;
249 
250         radix__change_memory_range(start, end, _PAGE_EXEC);
251 }
252 #endif /* CONFIG_STRICT_KERNEL_RWX */
253 
254 static inline void __meminit print_mapping(unsigned long start,
255                                            unsigned long end,
256                                            unsigned long size)
257 {
258         char buf[10];
259 
260         if (end <= start)
261                 return;
262 
263         string_get_size(size, 1, STRING_UNITS_2, buf, sizeof(buf));
264 
265         pr_info("Mapped 0x%016lx-0x%016lx with %s pages\n", start, end, buf);
266 }
267 
268 static int __meminit create_physical_mapping(unsigned long start,
269                                              unsigned long end,
270                                              int nid)
271 {
272         unsigned long vaddr, addr, mapping_size = 0;
273         pgprot_t prot;
274         unsigned long max_mapping_size;
275 #ifdef CONFIG_STRICT_KERNEL_RWX
276         int split_text_mapping = 1;
277 #else
278         int split_text_mapping = 0;
279 #endif
280 
281         start = _ALIGN_UP(start, PAGE_SIZE);
282         for (addr = start; addr < end; addr += mapping_size) {
283                 unsigned long gap, previous_size;
284                 int rc;
285 
286                 gap = end - addr;
287                 previous_size = mapping_size;
288                 max_mapping_size = PUD_SIZE;
289 
290 retry:
291                 if (IS_ALIGNED(addr, PUD_SIZE) && gap >= PUD_SIZE &&
292                     mmu_psize_defs[MMU_PAGE_1G].shift &&
293                     PUD_SIZE <= max_mapping_size)
294                         mapping_size = PUD_SIZE;
295                 else if (IS_ALIGNED(addr, PMD_SIZE) && gap >= PMD_SIZE &&
296                          mmu_psize_defs[MMU_PAGE_2M].shift)
297                         mapping_size = PMD_SIZE;
298                 else
299                         mapping_size = PAGE_SIZE;
300 
301                 if (split_text_mapping && (mapping_size == PUD_SIZE) &&
302                         (addr <= __pa_symbol(__init_begin)) &&
303                         (addr + mapping_size) >= __pa_symbol(_stext)) {
304                         max_mapping_size = PMD_SIZE;
305                         goto retry;
306                 }
307 
308                 if (split_text_mapping && (mapping_size == PMD_SIZE) &&
309                     (addr <= __pa_symbol(__init_begin)) &&
310                     (addr + mapping_size) >= __pa_symbol(_stext))
311                         mapping_size = PAGE_SIZE;
312 
313                 if (mapping_size != previous_size) {
314                         print_mapping(start, addr, previous_size);
315                         start = addr;
316                 }
317 
318                 vaddr = (unsigned long)__va(addr);
319 
320                 if (overlaps_kernel_text(vaddr, vaddr + mapping_size) ||
321                     overlaps_interrupt_vector_text(vaddr, vaddr + mapping_size))
322                         prot = PAGE_KERNEL_X;
323                 else
324                         prot = PAGE_KERNEL;
325 
326                 rc = __map_kernel_page(vaddr, addr, prot, mapping_size, nid, start, end);
327                 if (rc)
328                         return rc;
329         }
330 
331         print_mapping(start, addr, mapping_size);
332         return 0;
333 }
334 
335 void __init radix_init_pgtable(void)
336 {
337         unsigned long rts_field;
338         struct memblock_region *reg;
339 
340         /* We don't support slb for radix */
341         mmu_slb_size = 0;
342         /*
343          * Create the linear mapping, using standard page size for now
344          */
345         for_each_memblock(memory, reg) {
346                 /*
347                  * The memblock allocator  is up at this point, so the
348                  * page tables will be allocated within the range. No
349                  * need or a node (which we don't have yet).
350                  */
351                 WARN_ON(create_physical_mapping(reg->base,
352                                                 reg->base + reg->size,
353                                                 -1));
354         }
355 
356         /* Find out how many PID bits are supported */
357         if (cpu_has_feature(CPU_FTR_HVMODE)) {
358                 if (!mmu_pid_bits)
359                         mmu_pid_bits = 20;
360 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
361                 /*
362                  * When KVM is possible, we only use the top half of the
363                  * PID space to avoid collisions between host and guest PIDs
364                  * which can cause problems due to prefetch when exiting the
365                  * guest with AIL=3
366                  */
367                 mmu_base_pid = 1 << (mmu_pid_bits - 1);
368 #else
369                 mmu_base_pid = 1;
370 #endif
371         } else {
372                 /* The guest uses the bottom half of the PID space */
373                 if (!mmu_pid_bits)
374                         mmu_pid_bits = 19;
375                 mmu_base_pid = 1;
376         }
377 
378         /*
379          * Allocate Partition table and process table for the
380          * host.
381          */
382         BUG_ON(PRTB_SIZE_SHIFT > 36);
383         process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT, -1, 0, 0);
384         /*
385          * Fill in the process table.
386          */
387         rts_field = radix__get_tree_size();
388         process_tb->prtb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE);
389         /*
390          * Fill in the partition table. We are suppose to use effective address
391          * of process table here. But our linear mapping also enable us to use
392          * physical address here.
393          */
394         register_process_table(__pa(process_tb), 0, PRTB_SIZE_SHIFT - 12);
395         pr_info("Process table %p and radix root for kernel: %p\n", process_tb, init_mm.pgd);
396         asm volatile("ptesync" : : : "memory");
397         asm volatile(PPC_TLBIE_5(%0,%1,2,1,1) : :
398                      "r" (TLBIEL_INVAL_SET_LPID), "r" (0));
399         asm volatile("eieio; tlbsync; ptesync" : : : "memory");
400         trace_tlbie(0, 0, TLBIEL_INVAL_SET_LPID, 0, 2, 1, 1);
401 
402         /*
403          * The init_mm context is given the first available (non-zero) PID,
404          * which is the "guard PID" and contains no page table. PIDR should
405          * never be set to zero because that duplicates the kernel address
406          * space at the 0x0... offset (quadrant 0)!
407          *
408          * An arbitrary PID that may later be allocated by the PID allocator
409          * for userspace processes must not be used either, because that
410          * would cause stale user mappings for that PID on CPUs outside of
411          * the TLB invalidation scheme (because it won't be in mm_cpumask).
412          *
413          * So permanently carve out one PID for the purpose of a guard PID.
414          */
415         init_mm.context.id = mmu_base_pid;
416         mmu_base_pid++;
417 }
418 
419 static void __init radix_init_partition_table(void)
420 {
421         unsigned long rts_field, dw0;
422 
423         mmu_partition_table_init();
424         rts_field = radix__get_tree_size();
425         dw0 = rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE | PATB_HR;
426         mmu_partition_table_set_entry(0, dw0, 0);
427 
428         pr_info("Initializing Radix MMU\n");
429         pr_info("Partition table %p\n", partition_tb);
430 }
431 
432 void __init radix_init_native(void)
433 {
434         register_process_table = native_register_process_table;
435 }
436 
437 static int __init get_idx_from_shift(unsigned int shift)
438 {
439         int idx = -1;
440 
441         switch (shift) {
442         case 0xc:
443                 idx = MMU_PAGE_4K;
444                 break;
445         case 0x10:
446                 idx = MMU_PAGE_64K;
447                 break;
448         case 0x15:
449                 idx = MMU_PAGE_2M;
450                 break;
451         case 0x1e:
452                 idx = MMU_PAGE_1G;
453                 break;
454         }
455         return idx;
456 }
457 
458 static int __init radix_dt_scan_page_sizes(unsigned long node,
459                                            const char *uname, int depth,
460                                            void *data)
461 {
462         int size = 0;
463         int shift, idx;
464         unsigned int ap;
465         const __be32 *prop;
466         const char *type = of_get_flat_dt_prop(node, "device_type", NULL);
467 
468         /* We are scanning "cpu" nodes only */
469         if (type == NULL || strcmp(type, "cpu") != 0)
470                 return 0;
471 
472         /* Find MMU PID size */
473         prop = of_get_flat_dt_prop(node, "ibm,mmu-pid-bits", &size);
474         if (prop && size == 4)
475                 mmu_pid_bits = be32_to_cpup(prop);
476 
477         /* Grab page size encodings */
478         prop = of_get_flat_dt_prop(node, "ibm,processor-radix-AP-encodings", &size);
479         if (!prop)
480                 return 0;
481 
482         pr_info("Page sizes from device-tree:\n");
483         for (; size >= 4; size -= 4, ++prop) {
484 
485                 struct mmu_psize_def *def;
486 
487                 /* top 3 bit is AP encoding */
488                 shift = be32_to_cpu(prop[0]) & ~(0xe << 28);
489                 ap = be32_to_cpu(prop[0]) >> 29;
490                 pr_info("Page size shift = %d AP=0x%x\n", shift, ap);
491 
492                 idx = get_idx_from_shift(shift);
493                 if (idx < 0)
494                         continue;
495 
496                 def = &mmu_psize_defs[idx];
497                 def->shift = shift;
498                 def->ap  = ap;
499         }
500 
501         /* needed ? */
502         cur_cpu_spec->mmu_features &= ~MMU_FTR_NO_SLBIE_B;
503         return 1;
504 }
505 
506 void __init radix__early_init_devtree(void)
507 {
508         int rc;
509 
510         /*
511          * Try to find the available page sizes in the device-tree
512          */
513         rc = of_scan_flat_dt(radix_dt_scan_page_sizes, NULL);
514         if (rc != 0)  /* Found */
515                 goto found;
516         /*
517          * let's assume we have page 4k and 64k support
518          */
519         mmu_psize_defs[MMU_PAGE_4K].shift = 12;
520         mmu_psize_defs[MMU_PAGE_4K].ap = 0x0;
521 
522         mmu_psize_defs[MMU_PAGE_64K].shift = 16;
523         mmu_psize_defs[MMU_PAGE_64K].ap = 0x5;
524 found:
525 #ifdef CONFIG_SPARSEMEM_VMEMMAP
526         if (mmu_psize_defs[MMU_PAGE_2M].shift) {
527                 /*
528                  * map vmemmap using 2M if available
529                  */
530                 mmu_vmemmap_psize = MMU_PAGE_2M;
531         }
532 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
533         return;
534 }
535 
536 static void update_hid_for_radix(void)
537 {
538         unsigned long hid0;
539         unsigned long rb = 3UL << PPC_BITLSHIFT(53); /* IS = 3 */
540 
541         asm volatile("ptesync": : :"memory");
542         /* prs = 0, ric = 2, rs = 0, r = 1 is = 3 */
543         asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
544                      : : "r"(rb), "i"(1), "i"(0), "i"(2), "r"(0) : "memory");
545         /* prs = 1, ric = 2, rs = 0, r = 1 is = 3 */
546         asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
547                      : : "r"(rb), "i"(1), "i"(1), "i"(2), "r"(0) : "memory");
548         asm volatile("eieio; tlbsync; ptesync; isync; slbia": : :"memory");
549         trace_tlbie(0, 0, rb, 0, 2, 0, 1);
550         trace_tlbie(0, 0, rb, 0, 2, 1, 1);
551 
552         /*
553          * now switch the HID
554          */
555         hid0  = mfspr(SPRN_HID0);
556         hid0 |= HID0_POWER9_RADIX;
557         mtspr(SPRN_HID0, hid0);
558         asm volatile("isync": : :"memory");
559 
560         /* Wait for it to happen */
561         while (!(mfspr(SPRN_HID0) & HID0_POWER9_RADIX))
562                 cpu_relax();
563 }
564 
565 static void radix_init_amor(void)
566 {
567         /*
568         * In HV mode, we init AMOR (Authority Mask Override Register) so that
569         * the hypervisor and guest can setup IAMR (Instruction Authority Mask
570         * Register), enable key 0 and set it to 1.
571         *
572         * AMOR = 0b1100 .... 0000 (Mask for key 0 is 11)
573         */
574         mtspr(SPRN_AMOR, (3ul << 62));
575 }
576 
577 static void radix_init_iamr(void)
578 {
579         unsigned long iamr;
580 
581         /*
582          * The IAMR should set to 0 on DD1.
583          */
584         if (cpu_has_feature(CPU_FTR_POWER9_DD1))
585                 iamr = 0;
586         else
587                 iamr = (1ul << 62);
588 
589         /*
590          * Radix always uses key0 of the IAMR to determine if an access is
591          * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction
592          * fetch.
593          */
594         mtspr(SPRN_IAMR, iamr);
595 }
596 
597 void __init radix__early_init_mmu(void)
598 {
599         unsigned long lpcr;
600 
601 #ifdef CONFIG_PPC_64K_PAGES
602         /* PAGE_SIZE mappings */
603         mmu_virtual_psize = MMU_PAGE_64K;
604 #else
605         mmu_virtual_psize = MMU_PAGE_4K;
606 #endif
607 
608 #ifdef CONFIG_SPARSEMEM_VMEMMAP
609         /* vmemmap mapping */
610         mmu_vmemmap_psize = mmu_virtual_psize;
611 #endif
612         /*
613          * initialize page table size
614          */
615         __pte_index_size = RADIX_PTE_INDEX_SIZE;
616         __pmd_index_size = RADIX_PMD_INDEX_SIZE;
617         __pud_index_size = RADIX_PUD_INDEX_SIZE;
618         __pgd_index_size = RADIX_PGD_INDEX_SIZE;
619         __pud_cache_index = RADIX_PUD_INDEX_SIZE;
620         __pmd_cache_index = RADIX_PMD_INDEX_SIZE;
621         __pte_table_size = RADIX_PTE_TABLE_SIZE;
622         __pmd_table_size = RADIX_PMD_TABLE_SIZE;
623         __pud_table_size = RADIX_PUD_TABLE_SIZE;
624         __pgd_table_size = RADIX_PGD_TABLE_SIZE;
625 
626         __pmd_val_bits = RADIX_PMD_VAL_BITS;
627         __pud_val_bits = RADIX_PUD_VAL_BITS;
628         __pgd_val_bits = RADIX_PGD_VAL_BITS;
629 
630         __kernel_virt_start = RADIX_KERN_VIRT_START;
631         __kernel_virt_size = RADIX_KERN_VIRT_SIZE;
632         __vmalloc_start = RADIX_VMALLOC_START;
633         __vmalloc_end = RADIX_VMALLOC_END;
634         __kernel_io_start = RADIX_KERN_IO_START;
635         vmemmap = (struct page *)RADIX_VMEMMAP_BASE;
636         ioremap_bot = IOREMAP_BASE;
637 
638 #ifdef CONFIG_PCI
639         pci_io_base = ISA_IO_BASE;
640 #endif
641         __pte_frag_nr = RADIX_PTE_FRAG_NR;
642         __pte_frag_size_shift = RADIX_PTE_FRAG_SIZE_SHIFT;
643 
644         if (!firmware_has_feature(FW_FEATURE_LPAR)) {
645                 radix_init_native();
646                 if (cpu_has_feature(CPU_FTR_POWER9_DD1))
647                         update_hid_for_radix();
648                 lpcr = mfspr(SPRN_LPCR);
649                 mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
650                 radix_init_partition_table();
651                 radix_init_amor();
652         } else {
653                 radix_init_pseries();
654         }
655 
656         memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
657 
658         radix_init_iamr();
659         radix_init_pgtable();
660         /* Switch to the guard PID before turning on MMU */
661         radix__switch_mmu_context(NULL, &init_mm);
662         if (cpu_has_feature(CPU_FTR_HVMODE))
663                 tlbiel_all();
664 }
665 
666 void radix__early_init_mmu_secondary(void)
667 {
668         unsigned long lpcr;
669         /*
670          * update partition table control register and UPRT
671          */
672         if (!firmware_has_feature(FW_FEATURE_LPAR)) {
673 
674                 if (cpu_has_feature(CPU_FTR_POWER9_DD1))
675                         update_hid_for_radix();
676 
677                 lpcr = mfspr(SPRN_LPCR);
678                 mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
679 
680                 mtspr(SPRN_PTCR,
681                       __pa(partition_tb) | (PATB_SIZE_SHIFT - 12));
682                 radix_init_amor();
683         }
684         radix_init_iamr();
685 
686         radix__switch_mmu_context(NULL, &init_mm);
687         if (cpu_has_feature(CPU_FTR_HVMODE))
688                 tlbiel_all();
689 }
690 
691 void radix__mmu_cleanup_all(void)
692 {
693         unsigned long lpcr;
694 
695         if (!firmware_has_feature(FW_FEATURE_LPAR)) {
696                 lpcr = mfspr(SPRN_LPCR);
697                 mtspr(SPRN_LPCR, lpcr & ~LPCR_UPRT);
698                 mtspr(SPRN_PTCR, 0);
699                 powernv_set_nmmu_ptcr(0);
700                 radix__flush_tlb_all();
701         }
702 }
703 
704 void radix__setup_initial_memory_limit(phys_addr_t first_memblock_base,
705                                 phys_addr_t first_memblock_size)
706 {
707         /* We don't currently support the first MEMBLOCK not mapping 0
708          * physical on those processors
709          */
710         BUG_ON(first_memblock_base != 0);
711 
712         /*
713          * Radix mode is not limited by RMA / VRMA addressing.
714          */
715         ppc64_rma_size = ULONG_MAX;
716 }
717 
718 #ifdef CONFIG_MEMORY_HOTPLUG
719 static void free_pte_table(pte_t *pte_start, pmd_t *pmd)
720 {
721         pte_t *pte;
722         int i;
723 
724         for (i = 0; i < PTRS_PER_PTE; i++) {
725                 pte = pte_start + i;
726                 if (!pte_none(*pte))
727                         return;
728         }
729 
730         pte_free_kernel(&init_mm, pte_start);
731         pmd_clear(pmd);
732 }
733 
734 static void free_pmd_table(pmd_t *pmd_start, pud_t *pud)
735 {
736         pmd_t *pmd;
737         int i;
738 
739         for (i = 0; i < PTRS_PER_PMD; i++) {
740                 pmd = pmd_start + i;
741                 if (!pmd_none(*pmd))
742                         return;
743         }
744 
745         pmd_free(&init_mm, pmd_start);
746         pud_clear(pud);
747 }
748 
749 struct change_mapping_params {
750         pte_t *pte;
751         unsigned long start;
752         unsigned long end;
753         unsigned long aligned_start;
754         unsigned long aligned_end;
755 };
756 
757 static int __meminit stop_machine_change_mapping(void *data)
758 {
759         struct change_mapping_params *params =
760                         (struct change_mapping_params *)data;
761 
762         if (!data)
763                 return -1;
764 
765         spin_unlock(&init_mm.page_table_lock);
766         pte_clear(&init_mm, params->aligned_start, params->pte);
767         create_physical_mapping(params->aligned_start, params->start, -1);
768         create_physical_mapping(params->end, params->aligned_end, -1);
769         spin_lock(&init_mm.page_table_lock);
770         return 0;
771 }
772 
773 static void remove_pte_table(pte_t *pte_start, unsigned long addr,
774                              unsigned long end)
775 {
776         unsigned long next;
777         pte_t *pte;
778 
779         pte = pte_start + pte_index(addr);
780         for (; addr < end; addr = next, pte++) {
781                 next = (addr + PAGE_SIZE) & PAGE_MASK;
782                 if (next > end)
783                         next = end;
784 
785                 if (!pte_present(*pte))
786                         continue;
787 
788                 if (!PAGE_ALIGNED(addr) || !PAGE_ALIGNED(next)) {
789                         /*
790                          * The vmemmap_free() and remove_section_mapping()
791                          * codepaths call us with aligned addresses.
792                          */
793                         WARN_ONCE(1, "%s: unaligned range\n", __func__);
794                         continue;
795                 }
796 
797                 pte_clear(&init_mm, addr, pte);
798         }
799 }
800 
801 /*
802  * clear the pte and potentially split the mapping helper
803  */
804 static void __meminit split_kernel_mapping(unsigned long addr, unsigned long end,
805                                 unsigned long size, pte_t *pte)
806 {
807         unsigned long mask = ~(size - 1);
808         unsigned long aligned_start = addr & mask;
809         unsigned long aligned_end = addr + size;
810         struct change_mapping_params params;
811         bool split_region = false;
812 
813         if ((end - addr) < size) {
814                 /*
815                  * We're going to clear the PTE, but not flushed
816                  * the mapping, time to remap and flush. The
817                  * effects if visible outside the processor or
818                  * if we are running in code close to the
819                  * mapping we cleared, we are in trouble.
820                  */
821                 if (overlaps_kernel_text(aligned_start, addr) ||
822                         overlaps_kernel_text(end, aligned_end)) {
823                         /*
824                          * Hack, just return, don't pte_clear
825                          */
826                         WARN_ONCE(1, "Linear mapping %lx->%lx overlaps kernel "
827                                   "text, not splitting\n", addr, end);
828                         return;
829                 }
830                 split_region = true;
831         }
832 
833         if (split_region) {
834                 params.pte = pte;
835                 params.start = addr;
836                 params.end = end;
837                 params.aligned_start = addr & ~(size - 1);
838                 params.aligned_end = min_t(unsigned long, aligned_end,
839                                 (unsigned long)__va(memblock_end_of_DRAM()));
840                 stop_machine(stop_machine_change_mapping, &params, NULL);
841                 return;
842         }
843 
844         pte_clear(&init_mm, addr, pte);
845 }
846 
847 static void remove_pmd_table(pmd_t *pmd_start, unsigned long addr,
848                              unsigned long end)
849 {
850         unsigned long next;
851         pte_t *pte_base;
852         pmd_t *pmd;
853 
854         pmd = pmd_start + pmd_index(addr);
855         for (; addr < end; addr = next, pmd++) {
856                 next = pmd_addr_end(addr, end);
857 
858                 if (!pmd_present(*pmd))
859                         continue;
860 
861                 if (pmd_huge(*pmd)) {
862                         split_kernel_mapping(addr, end, PMD_SIZE, (pte_t *)pmd);
863                         continue;
864                 }
865 
866                 pte_base = (pte_t *)pmd_page_vaddr(*pmd);
867                 remove_pte_table(pte_base, addr, next);
868                 free_pte_table(pte_base, pmd);
869         }
870 }
871 
872 static void remove_pud_table(pud_t *pud_start, unsigned long addr,
873                              unsigned long end)
874 {
875         unsigned long next;
876         pmd_t *pmd_base;
877         pud_t *pud;
878 
879         pud = pud_start + pud_index(addr);
880         for (; addr < end; addr = next, pud++) {
881                 next = pud_addr_end(addr, end);
882 
883                 if (!pud_present(*pud))
884                         continue;
885 
886                 if (pud_huge(*pud)) {
887                         split_kernel_mapping(addr, end, PUD_SIZE, (pte_t *)pud);
888                         continue;
889                 }
890 
891                 pmd_base = (pmd_t *)pud_page_vaddr(*pud);
892                 remove_pmd_table(pmd_base, addr, next);
893                 free_pmd_table(pmd_base, pud);
894         }
895 }
896 
897 static void __meminit remove_pagetable(unsigned long start, unsigned long end)
898 {
899         unsigned long addr, next;
900         pud_t *pud_base;
901         pgd_t *pgd;
902 
903         spin_lock(&init_mm.page_table_lock);
904 
905         for (addr = start; addr < end; addr = next) {
906                 next = pgd_addr_end(addr, end);
907 
908                 pgd = pgd_offset_k(addr);
909                 if (!pgd_present(*pgd))
910                         continue;
911 
912                 if (pgd_huge(*pgd)) {
913                         split_kernel_mapping(addr, end, PGDIR_SIZE, (pte_t *)pgd);
914                         continue;
915                 }
916 
917                 pud_base = (pud_t *)pgd_page_vaddr(*pgd);
918                 remove_pud_table(pud_base, addr, next);
919         }
920 
921         spin_unlock(&init_mm.page_table_lock);
922         radix__flush_tlb_kernel_range(start, end);
923 }
924 
925 int __meminit radix__create_section_mapping(unsigned long start, unsigned long end, int nid)
926 {
927         return create_physical_mapping(start, end, nid);
928 }
929 
930 int __meminit radix__remove_section_mapping(unsigned long start, unsigned long end)
931 {
932         remove_pagetable(start, end);
933         return 0;
934 }
935 #endif /* CONFIG_MEMORY_HOTPLUG */
936 
937 #ifdef CONFIG_SPARSEMEM_VMEMMAP
938 static int __map_kernel_page_nid(unsigned long ea, unsigned long pa,
939                                  pgprot_t flags, unsigned int map_page_size,
940                                  int nid)
941 {
942         return __map_kernel_page(ea, pa, flags, map_page_size, nid, 0, 0);
943 }
944 
945 int __meminit radix__vmemmap_create_mapping(unsigned long start,
946                                       unsigned long page_size,
947                                       unsigned long phys)
948 {
949         /* Create a PTE encoding */
950         unsigned long flags = _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_KERNEL_RW;
951         int nid = early_pfn_to_nid(phys >> PAGE_SHIFT);
952         int ret;
953 
954         ret = __map_kernel_page_nid(start, phys, __pgprot(flags), page_size, nid);
955         BUG_ON(ret);
956 
957         return 0;
958 }
959 
960 #ifdef CONFIG_MEMORY_HOTPLUG
961 void __meminit radix__vmemmap_remove_mapping(unsigned long start, unsigned long page_size)
962 {
963         remove_pagetable(start, start + page_size);
964 }
965 #endif
966 #endif
967 
968 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
969 
970 unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
971                                   pmd_t *pmdp, unsigned long clr,
972                                   unsigned long set)
973 {
974         unsigned long old;
975 
976 #ifdef CONFIG_DEBUG_VM
977         WARN_ON(!radix__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
978         assert_spin_locked(&mm->page_table_lock);
979 #endif
980 
981         old = radix__pte_update(mm, addr, (pte_t *)pmdp, clr, set, 1);
982         trace_hugepage_update(addr, old, clr, set);
983 
984         return old;
985 }
986 
987 pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
988                         pmd_t *pmdp)
989 
990 {
991         pmd_t pmd;
992 
993         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
994         VM_BUG_ON(radix__pmd_trans_huge(*pmdp));
995         VM_BUG_ON(pmd_devmap(*pmdp));
996         /*
997          * khugepaged calls this for normal pmd
998          */
999         pmd = *pmdp;
1000         pmd_clear(pmdp);
1001 
1002         /*FIXME!!  Verify whether we need this kick below */
1003         serialize_against_pte_lookup(vma->vm_mm);
1004 
1005         radix__flush_tlb_collapsed_pmd(vma->vm_mm, address);
1006 
1007         return pmd;
1008 }
1009 
1010 /*
1011  * For us pgtable_t is pte_t *. Inorder to save the deposisted
1012  * page table, we consider the allocated page table as a list
1013  * head. On withdraw we need to make sure we zero out the used
1014  * list_head memory area.
1015  */
1016 void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
1017                                  pgtable_t pgtable)
1018 {
1019         struct list_head *lh = (struct list_head *) pgtable;
1020 
1021         assert_spin_locked(pmd_lockptr(mm, pmdp));
1022 
1023         /* FIFO */
1024         if (!pmd_huge_pte(mm, pmdp))
1025                 INIT_LIST_HEAD(lh);
1026         else
1027                 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
1028         pmd_huge_pte(mm, pmdp) = pgtable;
1029 }
1030 
1031 pgtable_t radix__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
1032 {
1033         pte_t *ptep;
1034         pgtable_t pgtable;
1035         struct list_head *lh;
1036 
1037         assert_spin_locked(pmd_lockptr(mm, pmdp));
1038 
1039         /* FIFO */
1040         pgtable = pmd_huge_pte(mm, pmdp);
1041         lh = (struct list_head *) pgtable;
1042         if (list_empty(lh))
1043                 pmd_huge_pte(mm, pmdp) = NULL;
1044         else {
1045                 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
1046                 list_del(lh);
1047         }
1048         ptep = (pte_t *) pgtable;
1049         *ptep = __pte(0);
1050         ptep++;
1051         *ptep = __pte(0);
1052         return pgtable;
1053 }
1054 
1055 
1056 pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm,
1057                                unsigned long addr, pmd_t *pmdp)
1058 {
1059         pmd_t old_pmd;
1060         unsigned long old;
1061 
1062         old = radix__pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
1063         old_pmd = __pmd(old);
1064         /*
1065          * Serialize against find_current_mm_pte which does lock-less
1066          * lookup in page tables with local interrupts disabled. For huge pages
1067          * it casts pmd_t to pte_t. Since format of pte_t is different from
1068          * pmd_t we want to prevent transit from pmd pointing to page table
1069          * to pmd pointing to huge page (and back) while interrupts are disabled.
1070          * We clear pmd to possibly replace it with page table pointer in
1071          * different code paths. So make sure we wait for the parallel
1072          * find_current_mm_pte to finish.
1073          */
1074         serialize_against_pte_lookup(mm);
1075         return old_pmd;
1076 }
1077 
1078 int radix__has_transparent_hugepage(void)
1079 {
1080         /* For radix 2M at PMD level means thp */
1081         if (mmu_psize_defs[MMU_PAGE_2M].shift == PMD_SHIFT)
1082                 return 1;
1083         return 0;
1084 }
1085 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1086 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp