~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/powerpc/mm/hugetlbpage.c

Version: ~ [ linux-5.8-rc5 ] ~ [ linux-5.7.8 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.51 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.132 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.188 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.230 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.230 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * PPC Huge TLB Page Support for Kernel.
  3  *
  4  * Copyright (C) 2003 David Gibson, IBM Corporation.
  5  * Copyright (C) 2011 Becky Bruce, Freescale Semiconductor
  6  *
  7  * Based on the IA-32 version:
  8  * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
  9  */
 10 
 11 #include <linux/mm.h>
 12 #include <linux/io.h>
 13 #include <linux/slab.h>
 14 #include <linux/hugetlb.h>
 15 #include <linux/export.h>
 16 #include <linux/of_fdt.h>
 17 #include <linux/memblock.h>
 18 #include <linux/bootmem.h>
 19 #include <linux/moduleparam.h>
 20 #include <asm/pgtable.h>
 21 #include <asm/pgalloc.h>
 22 #include <asm/tlb.h>
 23 #include <asm/setup.h>
 24 #include <asm/hugetlb.h>
 25 
 26 #ifdef CONFIG_HUGETLB_PAGE
 27 
 28 #define PAGE_SHIFT_64K  16
 29 #define PAGE_SHIFT_16M  24
 30 #define PAGE_SHIFT_16G  34
 31 
 32 unsigned int HPAGE_SHIFT;
 33 
 34 /*
 35  * Tracks gpages after the device tree is scanned and before the
 36  * huge_boot_pages list is ready.  On non-Freescale implementations, this is
 37  * just used to track 16G pages and so is a single array.  FSL-based
 38  * implementations may have more than one gpage size, so we need multiple
 39  * arrays
 40  */
 41 #ifdef CONFIG_PPC_FSL_BOOK3E
 42 #define MAX_NUMBER_GPAGES       128
 43 struct psize_gpages {
 44         u64 gpage_list[MAX_NUMBER_GPAGES];
 45         unsigned int nr_gpages;
 46 };
 47 static struct psize_gpages gpage_freearray[MMU_PAGE_COUNT];
 48 #else
 49 #define MAX_NUMBER_GPAGES       1024
 50 static u64 gpage_freearray[MAX_NUMBER_GPAGES];
 51 static unsigned nr_gpages;
 52 #endif
 53 
 54 #define hugepd_none(hpd)        ((hpd).pd == 0)
 55 
 56 #ifdef CONFIG_PPC_BOOK3S_64
 57 /*
 58  * At this point we do the placement change only for BOOK3S 64. This would
 59  * possibly work on other subarchs.
 60  */
 61 
 62 /*
 63  * We have PGD_INDEX_SIZ = 12 and PTE_INDEX_SIZE = 8, so that we can have
 64  * 16GB hugepage pte in PGD and 16MB hugepage pte at PMD;
 65  */
 66 int pmd_huge(pmd_t pmd)
 67 {
 68         /*
 69          * leaf pte for huge page, bottom two bits != 00
 70          */
 71         return ((pmd_val(pmd) & 0x3) != 0x0);
 72 }
 73 
 74 int pud_huge(pud_t pud)
 75 {
 76         /*
 77          * leaf pte for huge page, bottom two bits != 00
 78          */
 79         return ((pud_val(pud) & 0x3) != 0x0);
 80 }
 81 
 82 int pgd_huge(pgd_t pgd)
 83 {
 84         /*
 85          * leaf pte for huge page, bottom two bits != 00
 86          */
 87         return ((pgd_val(pgd) & 0x3) != 0x0);
 88 }
 89 #else
 90 int pmd_huge(pmd_t pmd)
 91 {
 92         return 0;
 93 }
 94 
 95 int pud_huge(pud_t pud)
 96 {
 97         return 0;
 98 }
 99 
100 int pgd_huge(pgd_t pgd)
101 {
102         return 0;
103 }
104 #endif
105 
106 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
107 {
108         /* Only called for hugetlbfs pages, hence can ignore THP */
109         return find_linux_pte_or_hugepte(mm->pgd, addr, NULL);
110 }
111 
112 static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
113                            unsigned long address, unsigned pdshift, unsigned pshift)
114 {
115         struct kmem_cache *cachep;
116         pte_t *new;
117 
118 #ifdef CONFIG_PPC_FSL_BOOK3E
119         int i;
120         int num_hugepd = 1 << (pshift - pdshift);
121         cachep = hugepte_cache;
122 #else
123         cachep = PGT_CACHE(pdshift - pshift);
124 #endif
125 
126         new = kmem_cache_zalloc(cachep, GFP_KERNEL|__GFP_REPEAT);
127 
128         BUG_ON(pshift > HUGEPD_SHIFT_MASK);
129         BUG_ON((unsigned long)new & HUGEPD_SHIFT_MASK);
130 
131         if (! new)
132                 return -ENOMEM;
133 
134         spin_lock(&mm->page_table_lock);
135 #ifdef CONFIG_PPC_FSL_BOOK3E
136         /*
137          * We have multiple higher-level entries that point to the same
138          * actual pte location.  Fill in each as we go and backtrack on error.
139          * We need all of these so the DTLB pgtable walk code can find the
140          * right higher-level entry without knowing if it's a hugepage or not.
141          */
142         for (i = 0; i < num_hugepd; i++, hpdp++) {
143                 if (unlikely(!hugepd_none(*hpdp)))
144                         break;
145                 else
146                         /* We use the old format for PPC_FSL_BOOK3E */
147                         hpdp->pd = ((unsigned long)new & ~PD_HUGE) | pshift;
148         }
149         /* If we bailed from the for loop early, an error occurred, clean up */
150         if (i < num_hugepd) {
151                 for (i = i - 1 ; i >= 0; i--, hpdp--)
152                         hpdp->pd = 0;
153                 kmem_cache_free(cachep, new);
154         }
155 #else
156         if (!hugepd_none(*hpdp))
157                 kmem_cache_free(cachep, new);
158         else {
159 #ifdef CONFIG_PPC_BOOK3S_64
160                 hpdp->pd = (unsigned long)new |
161                             (shift_to_mmu_psize(pshift) << 2);
162 #else
163                 hpdp->pd = ((unsigned long)new & ~PD_HUGE) | pshift;
164 #endif
165         }
166 #endif
167         spin_unlock(&mm->page_table_lock);
168         return 0;
169 }
170 
171 /*
172  * These macros define how to determine which level of the page table holds
173  * the hpdp.
174  */
175 #ifdef CONFIG_PPC_FSL_BOOK3E
176 #define HUGEPD_PGD_SHIFT PGDIR_SHIFT
177 #define HUGEPD_PUD_SHIFT PUD_SHIFT
178 #else
179 #define HUGEPD_PGD_SHIFT PUD_SHIFT
180 #define HUGEPD_PUD_SHIFT PMD_SHIFT
181 #endif
182 
183 #ifdef CONFIG_PPC_BOOK3S_64
184 /*
185  * At this point we do the placement change only for BOOK3S 64. This would
186  * possibly work on other subarchs.
187  */
188 pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
189 {
190         pgd_t *pg;
191         pud_t *pu;
192         pmd_t *pm;
193         hugepd_t *hpdp = NULL;
194         unsigned pshift = __ffs(sz);
195         unsigned pdshift = PGDIR_SHIFT;
196 
197         addr &= ~(sz-1);
198         pg = pgd_offset(mm, addr);
199 
200         if (pshift == PGDIR_SHIFT)
201                 /* 16GB huge page */
202                 return (pte_t *) pg;
203         else if (pshift > PUD_SHIFT)
204                 /*
205                  * We need to use hugepd table
206                  */
207                 hpdp = (hugepd_t *)pg;
208         else {
209                 pdshift = PUD_SHIFT;
210                 pu = pud_alloc(mm, pg, addr);
211                 if (pshift == PUD_SHIFT)
212                         return (pte_t *)pu;
213                 else if (pshift > PMD_SHIFT)
214                         hpdp = (hugepd_t *)pu;
215                 else {
216                         pdshift = PMD_SHIFT;
217                         pm = pmd_alloc(mm, pu, addr);
218                         if (pshift == PMD_SHIFT)
219                                 /* 16MB hugepage */
220                                 return (pte_t *)pm;
221                         else
222                                 hpdp = (hugepd_t *)pm;
223                 }
224         }
225         if (!hpdp)
226                 return NULL;
227 
228         BUG_ON(!hugepd_none(*hpdp) && !hugepd_ok(*hpdp));
229 
230         if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr, pdshift, pshift))
231                 return NULL;
232 
233         return hugepte_offset(hpdp, addr, pdshift);
234 }
235 
236 #else
237 
238 pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
239 {
240         pgd_t *pg;
241         pud_t *pu;
242         pmd_t *pm;
243         hugepd_t *hpdp = NULL;
244         unsigned pshift = __ffs(sz);
245         unsigned pdshift = PGDIR_SHIFT;
246 
247         addr &= ~(sz-1);
248 
249         pg = pgd_offset(mm, addr);
250 
251         if (pshift >= HUGEPD_PGD_SHIFT) {
252                 hpdp = (hugepd_t *)pg;
253         } else {
254                 pdshift = PUD_SHIFT;
255                 pu = pud_alloc(mm, pg, addr);
256                 if (pshift >= HUGEPD_PUD_SHIFT) {
257                         hpdp = (hugepd_t *)pu;
258                 } else {
259                         pdshift = PMD_SHIFT;
260                         pm = pmd_alloc(mm, pu, addr);
261                         hpdp = (hugepd_t *)pm;
262                 }
263         }
264 
265         if (!hpdp)
266                 return NULL;
267 
268         BUG_ON(!hugepd_none(*hpdp) && !hugepd_ok(*hpdp));
269 
270         if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr, pdshift, pshift))
271                 return NULL;
272 
273         return hugepte_offset(hpdp, addr, pdshift);
274 }
275 #endif
276 
277 #ifdef CONFIG_PPC_FSL_BOOK3E
278 /* Build list of addresses of gigantic pages.  This function is used in early
279  * boot before the buddy or bootmem allocator is setup.
280  */
281 void add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages)
282 {
283         unsigned int idx = shift_to_mmu_psize(__ffs(page_size));
284         int i;
285 
286         if (addr == 0)
287                 return;
288 
289         gpage_freearray[idx].nr_gpages = number_of_pages;
290 
291         for (i = 0; i < number_of_pages; i++) {
292                 gpage_freearray[idx].gpage_list[i] = addr;
293                 addr += page_size;
294         }
295 }
296 
297 /*
298  * Moves the gigantic page addresses from the temporary list to the
299  * huge_boot_pages list.
300  */
301 int alloc_bootmem_huge_page(struct hstate *hstate)
302 {
303         struct huge_bootmem_page *m;
304         int idx = shift_to_mmu_psize(huge_page_shift(hstate));
305         int nr_gpages = gpage_freearray[idx].nr_gpages;
306 
307         if (nr_gpages == 0)
308                 return 0;
309 
310 #ifdef CONFIG_HIGHMEM
311         /*
312          * If gpages can be in highmem we can't use the trick of storing the
313          * data structure in the page; allocate space for this
314          */
315         m = alloc_bootmem(sizeof(struct huge_bootmem_page));
316         m->phys = gpage_freearray[idx].gpage_list[--nr_gpages];
317 #else
318         m = phys_to_virt(gpage_freearray[idx].gpage_list[--nr_gpages]);
319 #endif
320 
321         list_add(&m->list, &huge_boot_pages);
322         gpage_freearray[idx].nr_gpages = nr_gpages;
323         gpage_freearray[idx].gpage_list[nr_gpages] = 0;
324         m->hstate = hstate;
325 
326         return 1;
327 }
328 /*
329  * Scan the command line hugepagesz= options for gigantic pages; store those in
330  * a list that we use to allocate the memory once all options are parsed.
331  */
332 
333 unsigned long gpage_npages[MMU_PAGE_COUNT];
334 
335 static int __init do_gpage_early_setup(char *param, char *val,
336                                        const char *unused)
337 {
338         static phys_addr_t size;
339         unsigned long npages;
340 
341         /*
342          * The hugepagesz and hugepages cmdline options are interleaved.  We
343          * use the size variable to keep track of whether or not this was done
344          * properly and skip over instances where it is incorrect.  Other
345          * command-line parsing code will issue warnings, so we don't need to.
346          *
347          */
348         if ((strcmp(param, "default_hugepagesz") == 0) ||
349             (strcmp(param, "hugepagesz") == 0)) {
350                 size = memparse(val, NULL);
351         } else if (strcmp(param, "hugepages") == 0) {
352                 if (size != 0) {
353                         if (sscanf(val, "%lu", &npages) <= 0)
354                                 npages = 0;
355                         gpage_npages[shift_to_mmu_psize(__ffs(size))] = npages;
356                         size = 0;
357                 }
358         }
359         return 0;
360 }
361 
362 
363 /*
364  * This function allocates physical space for pages that are larger than the
365  * buddy allocator can handle.  We want to allocate these in highmem because
366  * the amount of lowmem is limited.  This means that this function MUST be
367  * called before lowmem_end_addr is set up in MMU_init() in order for the lmb
368  * allocate to grab highmem.
369  */
370 void __init reserve_hugetlb_gpages(void)
371 {
372         static __initdata char cmdline[COMMAND_LINE_SIZE];
373         phys_addr_t size, base;
374         int i;
375 
376         strlcpy(cmdline, boot_command_line, COMMAND_LINE_SIZE);
377         parse_args("hugetlb gpages", cmdline, NULL, 0, 0, 0,
378                         &do_gpage_early_setup);
379 
380         /*
381          * Walk gpage list in reverse, allocating larger page sizes first.
382          * Skip over unsupported sizes, or sizes that have 0 gpages allocated.
383          * When we reach the point in the list where pages are no longer
384          * considered gpages, we're done.
385          */
386         for (i = MMU_PAGE_COUNT-1; i >= 0; i--) {
387                 if (mmu_psize_defs[i].shift == 0 || gpage_npages[i] == 0)
388                         continue;
389                 else if (mmu_psize_to_shift(i) < (MAX_ORDER + PAGE_SHIFT))
390                         break;
391 
392                 size = (phys_addr_t)(1ULL << mmu_psize_to_shift(i));
393                 base = memblock_alloc_base(size * gpage_npages[i], size,
394                                            MEMBLOCK_ALLOC_ANYWHERE);
395                 add_gpage(base, size, gpage_npages[i]);
396         }
397 }
398 
399 #else /* !PPC_FSL_BOOK3E */
400 
401 /* Build list of addresses of gigantic pages.  This function is used in early
402  * boot before the buddy or bootmem allocator is setup.
403  */
404 void add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages)
405 {
406         if (!addr)
407                 return;
408         while (number_of_pages > 0) {
409                 gpage_freearray[nr_gpages] = addr;
410                 nr_gpages++;
411                 number_of_pages--;
412                 addr += page_size;
413         }
414 }
415 
416 /* Moves the gigantic page addresses from the temporary list to the
417  * huge_boot_pages list.
418  */
419 int alloc_bootmem_huge_page(struct hstate *hstate)
420 {
421         struct huge_bootmem_page *m;
422         if (nr_gpages == 0)
423                 return 0;
424         m = phys_to_virt(gpage_freearray[--nr_gpages]);
425         gpage_freearray[nr_gpages] = 0;
426         list_add(&m->list, &huge_boot_pages);
427         m->hstate = hstate;
428         return 1;
429 }
430 #endif
431 
432 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
433 {
434         return 0;
435 }
436 
437 #ifdef CONFIG_PPC_FSL_BOOK3E
438 #define HUGEPD_FREELIST_SIZE \
439         ((PAGE_SIZE - sizeof(struct hugepd_freelist)) / sizeof(pte_t))
440 
441 struct hugepd_freelist {
442         struct rcu_head rcu;
443         unsigned int index;
444         void *ptes[0];
445 };
446 
447 static DEFINE_PER_CPU(struct hugepd_freelist *, hugepd_freelist_cur);
448 
449 static void hugepd_free_rcu_callback(struct rcu_head *head)
450 {
451         struct hugepd_freelist *batch =
452                 container_of(head, struct hugepd_freelist, rcu);
453         unsigned int i;
454 
455         for (i = 0; i < batch->index; i++)
456                 kmem_cache_free(hugepte_cache, batch->ptes[i]);
457 
458         free_page((unsigned long)batch);
459 }
460 
461 static void hugepd_free(struct mmu_gather *tlb, void *hugepte)
462 {
463         struct hugepd_freelist **batchp;
464 
465         batchp = &get_cpu_var(hugepd_freelist_cur);
466 
467         if (atomic_read(&tlb->mm->mm_users) < 2 ||
468             cpumask_equal(mm_cpumask(tlb->mm),
469                           cpumask_of(smp_processor_id()))) {
470                 kmem_cache_free(hugepte_cache, hugepte);
471         put_cpu_var(hugepd_freelist_cur);
472                 return;
473         }
474 
475         if (*batchp == NULL) {
476                 *batchp = (struct hugepd_freelist *)__get_free_page(GFP_ATOMIC);
477                 (*batchp)->index = 0;
478         }
479 
480         (*batchp)->ptes[(*batchp)->index++] = hugepte;
481         if ((*batchp)->index == HUGEPD_FREELIST_SIZE) {
482                 call_rcu_sched(&(*batchp)->rcu, hugepd_free_rcu_callback);
483                 *batchp = NULL;
484         }
485         put_cpu_var(hugepd_freelist_cur);
486 }
487 #endif
488 
489 static void free_hugepd_range(struct mmu_gather *tlb, hugepd_t *hpdp, int pdshift,
490                               unsigned long start, unsigned long end,
491                               unsigned long floor, unsigned long ceiling)
492 {
493         pte_t *hugepte = hugepd_page(*hpdp);
494         int i;
495 
496         unsigned long pdmask = ~((1UL << pdshift) - 1);
497         unsigned int num_hugepd = 1;
498 
499 #ifdef CONFIG_PPC_FSL_BOOK3E
500         /* Note: On fsl the hpdp may be the first of several */
501         num_hugepd = (1 << (hugepd_shift(*hpdp) - pdshift));
502 #else
503         unsigned int shift = hugepd_shift(*hpdp);
504 #endif
505 
506         start &= pdmask;
507         if (start < floor)
508                 return;
509         if (ceiling) {
510                 ceiling &= pdmask;
511                 if (! ceiling)
512                         return;
513         }
514         if (end - 1 > ceiling - 1)
515                 return;
516 
517         for (i = 0; i < num_hugepd; i++, hpdp++)
518                 hpdp->pd = 0;
519 
520         tlb->need_flush = 1;
521 
522 #ifdef CONFIG_PPC_FSL_BOOK3E
523         hugepd_free(tlb, hugepte);
524 #else
525         pgtable_free_tlb(tlb, hugepte, pdshift - shift);
526 #endif
527 }
528 
529 static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
530                                    unsigned long addr, unsigned long end,
531                                    unsigned long floor, unsigned long ceiling)
532 {
533         pmd_t *pmd;
534         unsigned long next;
535         unsigned long start;
536 
537         start = addr;
538         do {
539                 pmd = pmd_offset(pud, addr);
540                 next = pmd_addr_end(addr, end);
541                 if (!is_hugepd(pmd)) {
542                         /*
543                          * if it is not hugepd pointer, we should already find
544                          * it cleared.
545                          */
546                         WARN_ON(!pmd_none_or_clear_bad(pmd));
547                         continue;
548                 }
549 #ifdef CONFIG_PPC_FSL_BOOK3E
550                 /*
551                  * Increment next by the size of the huge mapping since
552                  * there may be more than one entry at this level for a
553                  * single hugepage, but all of them point to
554                  * the same kmem cache that holds the hugepte.
555                  */
556                 next = addr + (1 << hugepd_shift(*(hugepd_t *)pmd));
557 #endif
558                 free_hugepd_range(tlb, (hugepd_t *)pmd, PMD_SHIFT,
559                                   addr, next, floor, ceiling);
560         } while (addr = next, addr != end);
561 
562         start &= PUD_MASK;
563         if (start < floor)
564                 return;
565         if (ceiling) {
566                 ceiling &= PUD_MASK;
567                 if (!ceiling)
568                         return;
569         }
570         if (end - 1 > ceiling - 1)
571                 return;
572 
573         pmd = pmd_offset(pud, start);
574         pud_clear(pud);
575         pmd_free_tlb(tlb, pmd, start);
576 }
577 
578 static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
579                                    unsigned long addr, unsigned long end,
580                                    unsigned long floor, unsigned long ceiling)
581 {
582         pud_t *pud;
583         unsigned long next;
584         unsigned long start;
585 
586         start = addr;
587         do {
588                 pud = pud_offset(pgd, addr);
589                 next = pud_addr_end(addr, end);
590                 if (!is_hugepd(pud)) {
591                         if (pud_none_or_clear_bad(pud))
592                                 continue;
593                         hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
594                                                ceiling);
595                 } else {
596 #ifdef CONFIG_PPC_FSL_BOOK3E
597                         /*
598                          * Increment next by the size of the huge mapping since
599                          * there may be more than one entry at this level for a
600                          * single hugepage, but all of them point to
601                          * the same kmem cache that holds the hugepte.
602                          */
603                         next = addr + (1 << hugepd_shift(*(hugepd_t *)pud));
604 #endif
605                         free_hugepd_range(tlb, (hugepd_t *)pud, PUD_SHIFT,
606                                           addr, next, floor, ceiling);
607                 }
608         } while (addr = next, addr != end);
609 
610         start &= PGDIR_MASK;
611         if (start < floor)
612                 return;
613         if (ceiling) {
614                 ceiling &= PGDIR_MASK;
615                 if (!ceiling)
616                         return;
617         }
618         if (end - 1 > ceiling - 1)
619                 return;
620 
621         pud = pud_offset(pgd, start);
622         pgd_clear(pgd);
623         pud_free_tlb(tlb, pud, start);
624 }
625 
626 /*
627  * This function frees user-level page tables of a process.
628  */
629 void hugetlb_free_pgd_range(struct mmu_gather *tlb,
630                             unsigned long addr, unsigned long end,
631                             unsigned long floor, unsigned long ceiling)
632 {
633         pgd_t *pgd;
634         unsigned long next;
635 
636         /*
637          * Because there are a number of different possible pagetable
638          * layouts for hugepage ranges, we limit knowledge of how
639          * things should be laid out to the allocation path
640          * (huge_pte_alloc(), above).  Everything else works out the
641          * structure as it goes from information in the hugepd
642          * pointers.  That means that we can't here use the
643          * optimization used in the normal page free_pgd_range(), of
644          * checking whether we're actually covering a large enough
645          * range to have to do anything at the top level of the walk
646          * instead of at the bottom.
647          *
648          * To make sense of this, you should probably go read the big
649          * block comment at the top of the normal free_pgd_range(),
650          * too.
651          */
652 
653         do {
654                 next = pgd_addr_end(addr, end);
655                 pgd = pgd_offset(tlb->mm, addr);
656                 if (!is_hugepd(pgd)) {
657                         if (pgd_none_or_clear_bad(pgd))
658                                 continue;
659                         hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling);
660                 } else {
661 #ifdef CONFIG_PPC_FSL_BOOK3E
662                         /*
663                          * Increment next by the size of the huge mapping since
664                          * there may be more than one entry at the pgd level
665                          * for a single hugepage, but all of them point to the
666                          * same kmem cache that holds the hugepte.
667                          */
668                         next = addr + (1 << hugepd_shift(*(hugepd_t *)pgd));
669 #endif
670                         free_hugepd_range(tlb, (hugepd_t *)pgd, PGDIR_SHIFT,
671                                           addr, next, floor, ceiling);
672                 }
673         } while (addr = next, addr != end);
674 }
675 
676 struct page *
677 follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
678 {
679         pte_t *ptep;
680         struct page *page;
681         unsigned shift;
682         unsigned long mask;
683         /*
684          * Transparent hugepages are handled by generic code. We can skip them
685          * here.
686          */
687         ptep = find_linux_pte_or_hugepte(mm->pgd, address, &shift);
688 
689         /* Verify it is a huge page else bail. */
690         if (!ptep || !shift || pmd_trans_huge(*(pmd_t *)ptep))
691                 return ERR_PTR(-EINVAL);
692 
693         mask = (1UL << shift) - 1;
694         page = pte_page(*ptep);
695         if (page)
696                 page += (address & mask) / PAGE_SIZE;
697 
698         return page;
699 }
700 
701 struct page *
702 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
703                 pmd_t *pmd, int write)
704 {
705         BUG();
706         return NULL;
707 }
708 
709 static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
710                                       unsigned long sz)
711 {
712         unsigned long __boundary = (addr + sz) & ~(sz-1);
713         return (__boundary - 1 < end - 1) ? __boundary : end;
714 }
715 
716 int gup_hugepd(hugepd_t *hugepd, unsigned pdshift,
717                unsigned long addr, unsigned long end,
718                int write, struct page **pages, int *nr)
719 {
720         pte_t *ptep;
721         unsigned long sz = 1UL << hugepd_shift(*hugepd);
722         unsigned long next;
723 
724         ptep = hugepte_offset(hugepd, addr, pdshift);
725         do {
726                 next = hugepte_addr_end(addr, end, sz);
727                 if (!gup_hugepte(ptep, sz, addr, end, write, pages, nr))
728                         return 0;
729         } while (ptep++, addr = next, addr != end);
730 
731         return 1;
732 }
733 
734 #ifdef CONFIG_PPC_MM_SLICES
735 unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
736                                         unsigned long len, unsigned long pgoff,
737                                         unsigned long flags)
738 {
739         struct hstate *hstate = hstate_file(file);
740         int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate));
741 
742         return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1);
743 }
744 #endif
745 
746 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
747 {
748 #ifdef CONFIG_PPC_MM_SLICES
749         unsigned int psize = get_slice_psize(vma->vm_mm, vma->vm_start);
750 
751         return 1UL << mmu_psize_to_shift(psize);
752 #else
753         if (!is_vm_hugetlb_page(vma))
754                 return PAGE_SIZE;
755 
756         return huge_page_size(hstate_vma(vma));
757 #endif
758 }
759 
760 static inline bool is_power_of_4(unsigned long x)
761 {
762         if (is_power_of_2(x))
763                 return (__ilog2(x) % 2) ? false : true;
764         return false;
765 }
766 
767 static int __init add_huge_page_size(unsigned long long size)
768 {
769         int shift = __ffs(size);
770         int mmu_psize;
771 
772         /* Check that it is a page size supported by the hardware and
773          * that it fits within pagetable and slice limits. */
774 #ifdef CONFIG_PPC_FSL_BOOK3E
775         if ((size < PAGE_SIZE) || !is_power_of_4(size))
776                 return -EINVAL;
777 #else
778         if (!is_power_of_2(size)
779             || (shift > SLICE_HIGH_SHIFT) || (shift <= PAGE_SHIFT))
780                 return -EINVAL;
781 #endif
782 
783         if ((mmu_psize = shift_to_mmu_psize(shift)) < 0)
784                 return -EINVAL;
785 
786 #ifdef CONFIG_SPU_FS_64K_LS
787         /* Disable support for 64K huge pages when 64K SPU local store
788          * support is enabled as the current implementation conflicts.
789          */
790         if (shift == PAGE_SHIFT_64K)
791                 return -EINVAL;
792 #endif /* CONFIG_SPU_FS_64K_LS */
793 
794         BUG_ON(mmu_psize_defs[mmu_psize].shift != shift);
795 
796         /* Return if huge page size has already been setup */
797         if (size_to_hstate(size))
798                 return 0;
799 
800         hugetlb_add_hstate(shift - PAGE_SHIFT);
801 
802         return 0;
803 }
804 
805 static int __init hugepage_setup_sz(char *str)
806 {
807         unsigned long long size;
808 
809         size = memparse(str, &str);
810 
811         if (add_huge_page_size(size) != 0)
812                 printk(KERN_WARNING "Invalid huge page size specified(%llu)\n", size);
813 
814         return 1;
815 }
816 __setup("hugepagesz=", hugepage_setup_sz);
817 
818 #ifdef CONFIG_PPC_FSL_BOOK3E
819 struct kmem_cache *hugepte_cache;
820 static int __init hugetlbpage_init(void)
821 {
822         int psize;
823 
824         for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
825                 unsigned shift;
826 
827                 if (!mmu_psize_defs[psize].shift)
828                         continue;
829 
830                 shift = mmu_psize_to_shift(psize);
831 
832                 /* Don't treat normal page sizes as huge... */
833                 if (shift != PAGE_SHIFT)
834                         if (add_huge_page_size(1ULL << shift) < 0)
835                                 continue;
836         }
837 
838         /*
839          * Create a kmem cache for hugeptes.  The bottom bits in the pte have
840          * size information encoded in them, so align them to allow this
841          */
842         hugepte_cache =  kmem_cache_create("hugepte-cache", sizeof(pte_t),
843                                            HUGEPD_SHIFT_MASK + 1, 0, NULL);
844         if (hugepte_cache == NULL)
845                 panic("%s: Unable to create kmem cache for hugeptes\n",
846                       __func__);
847 
848         /* Default hpage size = 4M */
849         if (mmu_psize_defs[MMU_PAGE_4M].shift)
850                 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_4M].shift;
851         else
852                 panic("%s: Unable to set default huge page size\n", __func__);
853 
854 
855         return 0;
856 }
857 #else
858 static int __init hugetlbpage_init(void)
859 {
860         int psize;
861 
862         if (!mmu_has_feature(MMU_FTR_16M_PAGE))
863                 return -ENODEV;
864 
865         for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
866                 unsigned shift;
867                 unsigned pdshift;
868 
869                 if (!mmu_psize_defs[psize].shift)
870                         continue;
871 
872                 shift = mmu_psize_to_shift(psize);
873 
874                 if (add_huge_page_size(1ULL << shift) < 0)
875                         continue;
876 
877                 if (shift < PMD_SHIFT)
878                         pdshift = PMD_SHIFT;
879                 else if (shift < PUD_SHIFT)
880                         pdshift = PUD_SHIFT;
881                 else
882                         pdshift = PGDIR_SHIFT;
883                 /*
884                  * if we have pdshift and shift value same, we don't
885                  * use pgt cache for hugepd.
886                  */
887                 if (pdshift != shift) {
888                         pgtable_cache_add(pdshift - shift, NULL);
889                         if (!PGT_CACHE(pdshift - shift))
890                                 panic("hugetlbpage_init(): could not create "
891                                       "pgtable cache for %d bit pagesize\n", shift);
892                 }
893         }
894 
895         /* Set default large page size. Currently, we pick 16M or 1M
896          * depending on what is available
897          */
898         if (mmu_psize_defs[MMU_PAGE_16M].shift)
899                 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_16M].shift;
900         else if (mmu_psize_defs[MMU_PAGE_1M].shift)
901                 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_1M].shift;
902 
903         return 0;
904 }
905 #endif
906 module_init(hugetlbpage_init);
907 
908 void flush_dcache_icache_hugepage(struct page *page)
909 {
910         int i;
911         void *start;
912 
913         BUG_ON(!PageCompound(page));
914 
915         for (i = 0; i < (1UL << compound_order(page)); i++) {
916                 if (!PageHighMem(page)) {
917                         __flush_dcache_icache(page_address(page+i));
918                 } else {
919                         start = kmap_atomic(page+i);
920                         __flush_dcache_icache(start);
921                         kunmap_atomic(start);
922                 }
923         }
924 }
925 
926 #endif /* CONFIG_HUGETLB_PAGE */
927 
928 /*
929  * We have 4 cases for pgds and pmds:
930  * (1) invalid (all zeroes)
931  * (2) pointer to next table, as normal; bottom 6 bits == 0
932  * (3) leaf pte for huge page, bottom two bits != 00
933  * (4) hugepd pointer, bottom two bits == 00, next 4 bits indicate size of table
934  *
935  * So long as we atomically load page table pointers we are safe against teardown,
936  * we can follow the address down to the the page and take a ref on it.
937  */
938 
939 pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, unsigned *shift)
940 {
941         pgd_t pgd, *pgdp;
942         pud_t pud, *pudp;
943         pmd_t pmd, *pmdp;
944         pte_t *ret_pte;
945         hugepd_t *hpdp = NULL;
946         unsigned pdshift = PGDIR_SHIFT;
947 
948         if (shift)
949                 *shift = 0;
950 
951         pgdp = pgdir + pgd_index(ea);
952         pgd  = ACCESS_ONCE(*pgdp);
953         /*
954          * Always operate on the local stack value. This make sure the
955          * value don't get updated by a parallel THP split/collapse,
956          * page fault or a page unmap. The return pte_t * is still not
957          * stable. So should be checked there for above conditions.
958          */
959         if (pgd_none(pgd))
960                 return NULL;
961         else if (pgd_huge(pgd)) {
962                 ret_pte = (pte_t *) pgdp;
963                 goto out;
964         } else if (is_hugepd(&pgd))
965                 hpdp = (hugepd_t *)&pgd;
966         else {
967                 /*
968                  * Even if we end up with an unmap, the pgtable will not
969                  * be freed, because we do an rcu free and here we are
970                  * irq disabled
971                  */
972                 pdshift = PUD_SHIFT;
973                 pudp = pud_offset(&pgd, ea);
974                 pud  = ACCESS_ONCE(*pudp);
975 
976                 if (pud_none(pud))
977                         return NULL;
978                 else if (pud_huge(pud)) {
979                         ret_pte = (pte_t *) pudp;
980                         goto out;
981                 } else if (is_hugepd(&pud))
982                         hpdp = (hugepd_t *)&pud;
983                 else {
984                         pdshift = PMD_SHIFT;
985                         pmdp = pmd_offset(&pud, ea);
986                         pmd  = ACCESS_ONCE(*pmdp);
987                         /*
988                          * A hugepage collapse is captured by pmd_none, because
989                          * it mark the pmd none and do a hpte invalidate.
990                          *
991                          * A hugepage split is captured by pmd_trans_splitting
992                          * because we mark the pmd trans splitting and do a
993                          * hpte invalidate
994                          *
995                          */
996                         if (pmd_none(pmd) || pmd_trans_splitting(pmd))
997                                 return NULL;
998 
999                         if (pmd_huge(pmd) || pmd_large(pmd)) {
1000                                 ret_pte = (pte_t *) pmdp;
1001                                 goto out;
1002                         } else if (is_hugepd(&pmd))
1003                                 hpdp = (hugepd_t *)&pmd;
1004                         else
1005                                 return pte_offset_kernel(&pmd, ea);
1006                 }
1007         }
1008         if (!hpdp)
1009                 return NULL;
1010 
1011         ret_pte = hugepte_offset(hpdp, ea, pdshift);
1012         pdshift = hugepd_shift(*hpdp);
1013 out:
1014         if (shift)
1015                 *shift = pdshift;
1016         return ret_pte;
1017 }
1018 EXPORT_SYMBOL_GPL(find_linux_pte_or_hugepte);
1019 
1020 int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
1021                 unsigned long end, int write, struct page **pages, int *nr)
1022 {
1023         unsigned long mask;
1024         unsigned long pte_end;
1025         struct page *head, *page, *tail;
1026         pte_t pte;
1027         int refs;
1028 
1029         pte_end = (addr + sz) & ~(sz-1);
1030         if (pte_end < end)
1031                 end = pte_end;
1032 
1033         pte = ACCESS_ONCE(*ptep);
1034         mask = _PAGE_PRESENT | _PAGE_USER;
1035         if (write)
1036                 mask |= _PAGE_RW;
1037 
1038         if ((pte_val(pte) & mask) != mask)
1039                 return 0;
1040 
1041 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1042         /*
1043          * check for splitting here
1044          */
1045         if (pmd_trans_splitting(pte_pmd(pte)))
1046                 return 0;
1047 #endif
1048 
1049         /* hugepages are never "special" */
1050         VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
1051 
1052         refs = 0;
1053         head = pte_page(pte);
1054 
1055         page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
1056         tail = page;
1057         do {
1058                 VM_BUG_ON(compound_head(page) != head);
1059                 pages[*nr] = page;
1060                 (*nr)++;
1061                 page++;
1062                 refs++;
1063         } while (addr += PAGE_SIZE, addr != end);
1064 
1065         if (!page_cache_add_speculative(head, refs)) {
1066                 *nr -= refs;
1067                 return 0;
1068         }
1069 
1070         if (unlikely(pte_val(pte) != pte_val(*ptep))) {
1071                 /* Could be optimized better */
1072                 *nr -= refs;
1073                 while (refs--)
1074                         put_page(head);
1075                 return 0;
1076         }
1077 
1078         /*
1079          * Any tail page need their mapcount reference taken before we
1080          * return.
1081          */
1082         while (refs--) {
1083                 if (PageTail(tail))
1084                         get_huge_page_tail(tail);
1085                 tail++;
1086         }
1087 
1088         return 1;
1089 }
1090 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp