~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/huge_memory.c

Version: ~ [ linux-5.18 ] ~ [ linux-5.17.9 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.41 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.117 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.195 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.244 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.280 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.315 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *  Copyright (C) 2009  Red Hat, Inc.
  3  *
  4  *  This work is licensed under the terms of the GNU GPL, version 2. See
  5  *  the COPYING file in the top-level directory.
  6  */
  7 
  8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  9 
 10 #include <linux/mm.h>
 11 #include <linux/sched.h>
 12 #include <linux/sched/coredump.h>
 13 #include <linux/sched/numa_balancing.h>
 14 #include <linux/highmem.h>
 15 #include <linux/hugetlb.h>
 16 #include <linux/mmu_notifier.h>
 17 #include <linux/rmap.h>
 18 #include <linux/swap.h>
 19 #include <linux/shrinker.h>
 20 #include <linux/mm_inline.h>
 21 #include <linux/swapops.h>
 22 #include <linux/dax.h>
 23 #include <linux/khugepaged.h>
 24 #include <linux/freezer.h>
 25 #include <linux/pfn_t.h>
 26 #include <linux/mman.h>
 27 #include <linux/memremap.h>
 28 #include <linux/pagemap.h>
 29 #include <linux/debugfs.h>
 30 #include <linux/migrate.h>
 31 #include <linux/hashtable.h>
 32 #include <linux/userfaultfd_k.h>
 33 #include <linux/page_idle.h>
 34 #include <linux/shmem_fs.h>
 35 #include <linux/oom.h>
 36 
 37 #include <asm/tlb.h>
 38 #include <asm/pgalloc.h>
 39 #include "internal.h"
 40 
 41 /*
 42  * By default, transparent hugepage support is disabled in order to avoid
 43  * risking an increased memory footprint for applications that are not
 44  * guaranteed to benefit from it. When transparent hugepage support is
 45  * enabled, it is for all mappings, and khugepaged scans all mappings.
 46  * Defrag is invoked by khugepaged hugepage allocations and by page faults
 47  * for all hugepage allocations.
 48  */
 49 unsigned long transparent_hugepage_flags __read_mostly =
 50 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS
 51         (1<<TRANSPARENT_HUGEPAGE_FLAG)|
 52 #endif
 53 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE
 54         (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)|
 55 #endif
 56         (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)|
 57         (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|
 58         (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
 59 
 60 static struct shrinker deferred_split_shrinker;
 61 
 62 static atomic_t huge_zero_refcount;
 63 struct page *huge_zero_page __read_mostly;
 64 
 65 bool transparent_hugepage_enabled(struct vm_area_struct *vma)
 66 {
 67         if (vma_is_anonymous(vma))
 68                 return __transparent_hugepage_enabled(vma);
 69         if (vma_is_shmem(vma) && shmem_huge_enabled(vma))
 70                 return __transparent_hugepage_enabled(vma);
 71 
 72         return false;
 73 }
 74 
 75 static struct page *get_huge_zero_page(void)
 76 {
 77         struct page *zero_page;
 78 retry:
 79         if (likely(atomic_inc_not_zero(&huge_zero_refcount)))
 80                 return READ_ONCE(huge_zero_page);
 81 
 82         zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE,
 83                         HPAGE_PMD_ORDER);
 84         if (!zero_page) {
 85                 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED);
 86                 return NULL;
 87         }
 88         count_vm_event(THP_ZERO_PAGE_ALLOC);
 89         preempt_disable();
 90         if (cmpxchg(&huge_zero_page, NULL, zero_page)) {
 91                 preempt_enable();
 92                 __free_pages(zero_page, compound_order(zero_page));
 93                 goto retry;
 94         }
 95 
 96         /* We take additional reference here. It will be put back by shrinker */
 97         atomic_set(&huge_zero_refcount, 2);
 98         preempt_enable();
 99         return READ_ONCE(huge_zero_page);
100 }
101 
102 static void put_huge_zero_page(void)
103 {
104         /*
105          * Counter should never go to zero here. Only shrinker can put
106          * last reference.
107          */
108         BUG_ON(atomic_dec_and_test(&huge_zero_refcount));
109 }
110 
111 struct page *mm_get_huge_zero_page(struct mm_struct *mm)
112 {
113         if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
114                 return READ_ONCE(huge_zero_page);
115 
116         if (!get_huge_zero_page())
117                 return NULL;
118 
119         if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
120                 put_huge_zero_page();
121 
122         return READ_ONCE(huge_zero_page);
123 }
124 
125 void mm_put_huge_zero_page(struct mm_struct *mm)
126 {
127         if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
128                 put_huge_zero_page();
129 }
130 
131 static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink,
132                                         struct shrink_control *sc)
133 {
134         /* we can free zero page only if last reference remains */
135         return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
136 }
137 
138 static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink,
139                                        struct shrink_control *sc)
140 {
141         if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) {
142                 struct page *zero_page = xchg(&huge_zero_page, NULL);
143                 BUG_ON(zero_page == NULL);
144                 __free_pages(zero_page, compound_order(zero_page));
145                 return HPAGE_PMD_NR;
146         }
147 
148         return 0;
149 }
150 
151 static struct shrinker huge_zero_page_shrinker = {
152         .count_objects = shrink_huge_zero_page_count,
153         .scan_objects = shrink_huge_zero_page_scan,
154         .seeks = DEFAULT_SEEKS,
155 };
156 
157 #ifdef CONFIG_SYSFS
158 static ssize_t enabled_show(struct kobject *kobj,
159                             struct kobj_attribute *attr, char *buf)
160 {
161         if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags))
162                 return sprintf(buf, "[always] madvise never\n");
163         else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags))
164                 return sprintf(buf, "always [madvise] never\n");
165         else
166                 return sprintf(buf, "always madvise [never]\n");
167 }
168 
169 static ssize_t enabled_store(struct kobject *kobj,
170                              struct kobj_attribute *attr,
171                              const char *buf, size_t count)
172 {
173         ssize_t ret = count;
174 
175         if (!memcmp("always", buf,
176                     min(sizeof("always")-1, count))) {
177                 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
178                 set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
179         } else if (!memcmp("madvise", buf,
180                            min(sizeof("madvise")-1, count))) {
181                 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
182                 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
183         } else if (!memcmp("never", buf,
184                            min(sizeof("never")-1, count))) {
185                 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
186                 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
187         } else
188                 ret = -EINVAL;
189 
190         if (ret > 0) {
191                 int err = start_stop_khugepaged();
192                 if (err)
193                         ret = err;
194         }
195         return ret;
196 }
197 static struct kobj_attribute enabled_attr =
198         __ATTR(enabled, 0644, enabled_show, enabled_store);
199 
200 ssize_t single_hugepage_flag_show(struct kobject *kobj,
201                                 struct kobj_attribute *attr, char *buf,
202                                 enum transparent_hugepage_flag flag)
203 {
204         return sprintf(buf, "%d\n",
205                        !!test_bit(flag, &transparent_hugepage_flags));
206 }
207 
208 ssize_t single_hugepage_flag_store(struct kobject *kobj,
209                                  struct kobj_attribute *attr,
210                                  const char *buf, size_t count,
211                                  enum transparent_hugepage_flag flag)
212 {
213         unsigned long value;
214         int ret;
215 
216         ret = kstrtoul(buf, 10, &value);
217         if (ret < 0)
218                 return ret;
219         if (value > 1)
220                 return -EINVAL;
221 
222         if (value)
223                 set_bit(flag, &transparent_hugepage_flags);
224         else
225                 clear_bit(flag, &transparent_hugepage_flags);
226 
227         return count;
228 }
229 
230 static ssize_t defrag_show(struct kobject *kobj,
231                            struct kobj_attribute *attr, char *buf)
232 {
233         if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
234                 return sprintf(buf, "[always] defer defer+madvise madvise never\n");
235         if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
236                 return sprintf(buf, "always [defer] defer+madvise madvise never\n");
237         if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags))
238                 return sprintf(buf, "always defer [defer+madvise] madvise never\n");
239         if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
240                 return sprintf(buf, "always defer defer+madvise [madvise] never\n");
241         return sprintf(buf, "always defer defer+madvise madvise [never]\n");
242 }
243 
244 static ssize_t defrag_store(struct kobject *kobj,
245                             struct kobj_attribute *attr,
246                             const char *buf, size_t count)
247 {
248         if (!memcmp("always", buf,
249                     min(sizeof("always")-1, count))) {
250                 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
251                 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
252                 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
253                 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
254         } else if (!memcmp("defer+madvise", buf,
255                     min(sizeof("defer+madvise")-1, count))) {
256                 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
257                 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
258                 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
259                 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
260         } else if (!memcmp("defer", buf,
261                     min(sizeof("defer")-1, count))) {
262                 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
263                 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
264                 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
265                 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
266         } else if (!memcmp("madvise", buf,
267                            min(sizeof("madvise")-1, count))) {
268                 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
269                 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
270                 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
271                 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
272         } else if (!memcmp("never", buf,
273                            min(sizeof("never")-1, count))) {
274                 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
275                 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
276                 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
277                 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
278         } else
279                 return -EINVAL;
280 
281         return count;
282 }
283 static struct kobj_attribute defrag_attr =
284         __ATTR(defrag, 0644, defrag_show, defrag_store);
285 
286 static ssize_t use_zero_page_show(struct kobject *kobj,
287                 struct kobj_attribute *attr, char *buf)
288 {
289         return single_hugepage_flag_show(kobj, attr, buf,
290                                 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
291 }
292 static ssize_t use_zero_page_store(struct kobject *kobj,
293                 struct kobj_attribute *attr, const char *buf, size_t count)
294 {
295         return single_hugepage_flag_store(kobj, attr, buf, count,
296                                  TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
297 }
298 static struct kobj_attribute use_zero_page_attr =
299         __ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store);
300 
301 static ssize_t hpage_pmd_size_show(struct kobject *kobj,
302                 struct kobj_attribute *attr, char *buf)
303 {
304         return sprintf(buf, "%lu\n", HPAGE_PMD_SIZE);
305 }
306 static struct kobj_attribute hpage_pmd_size_attr =
307         __ATTR_RO(hpage_pmd_size);
308 
309 #ifdef CONFIG_DEBUG_VM
310 static ssize_t debug_cow_show(struct kobject *kobj,
311                                 struct kobj_attribute *attr, char *buf)
312 {
313         return single_hugepage_flag_show(kobj, attr, buf,
314                                 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
315 }
316 static ssize_t debug_cow_store(struct kobject *kobj,
317                                struct kobj_attribute *attr,
318                                const char *buf, size_t count)
319 {
320         return single_hugepage_flag_store(kobj, attr, buf, count,
321                                  TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
322 }
323 static struct kobj_attribute debug_cow_attr =
324         __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store);
325 #endif /* CONFIG_DEBUG_VM */
326 
327 static struct attribute *hugepage_attr[] = {
328         &enabled_attr.attr,
329         &defrag_attr.attr,
330         &use_zero_page_attr.attr,
331         &hpage_pmd_size_attr.attr,
332 #if defined(CONFIG_SHMEM) && defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE)
333         &shmem_enabled_attr.attr,
334 #endif
335 #ifdef CONFIG_DEBUG_VM
336         &debug_cow_attr.attr,
337 #endif
338         NULL,
339 };
340 
341 static const struct attribute_group hugepage_attr_group = {
342         .attrs = hugepage_attr,
343 };
344 
345 static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj)
346 {
347         int err;
348 
349         *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj);
350         if (unlikely(!*hugepage_kobj)) {
351                 pr_err("failed to create transparent hugepage kobject\n");
352                 return -ENOMEM;
353         }
354 
355         err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group);
356         if (err) {
357                 pr_err("failed to register transparent hugepage group\n");
358                 goto delete_obj;
359         }
360 
361         err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group);
362         if (err) {
363                 pr_err("failed to register transparent hugepage group\n");
364                 goto remove_hp_group;
365         }
366 
367         return 0;
368 
369 remove_hp_group:
370         sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group);
371 delete_obj:
372         kobject_put(*hugepage_kobj);
373         return err;
374 }
375 
376 static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj)
377 {
378         sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group);
379         sysfs_remove_group(hugepage_kobj, &hugepage_attr_group);
380         kobject_put(hugepage_kobj);
381 }
382 #else
383 static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj)
384 {
385         return 0;
386 }
387 
388 static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj)
389 {
390 }
391 #endif /* CONFIG_SYSFS */
392 
393 static int __init hugepage_init(void)
394 {
395         int err;
396         struct kobject *hugepage_kobj;
397 
398         if (!has_transparent_hugepage()) {
399                 transparent_hugepage_flags = 0;
400                 return -EINVAL;
401         }
402 
403         /*
404          * hugepages can't be allocated by the buddy allocator
405          */
406         MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER >= MAX_ORDER);
407         /*
408          * we use page->mapping and page->index in second tail page
409          * as list_head: assuming THP order >= 2
410          */
411         MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER < 2);
412 
413         err = hugepage_init_sysfs(&hugepage_kobj);
414         if (err)
415                 goto err_sysfs;
416 
417         err = khugepaged_init();
418         if (err)
419                 goto err_slab;
420 
421         err = register_shrinker(&huge_zero_page_shrinker);
422         if (err)
423                 goto err_hzp_shrinker;
424         err = register_shrinker(&deferred_split_shrinker);
425         if (err)
426                 goto err_split_shrinker;
427 
428         /*
429          * By default disable transparent hugepages on smaller systems,
430          * where the extra memory used could hurt more than TLB overhead
431          * is likely to save.  The admin can still enable it through /sys.
432          */
433         if (totalram_pages() < (512 << (20 - PAGE_SHIFT))) {
434                 transparent_hugepage_flags = 0;
435                 return 0;
436         }
437 
438         err = start_stop_khugepaged();
439         if (err)
440                 goto err_khugepaged;
441 
442         return 0;
443 err_khugepaged:
444         unregister_shrinker(&deferred_split_shrinker);
445 err_split_shrinker:
446         unregister_shrinker(&huge_zero_page_shrinker);
447 err_hzp_shrinker:
448         khugepaged_destroy();
449 err_slab:
450         hugepage_exit_sysfs(hugepage_kobj);
451 err_sysfs:
452         return err;
453 }
454 subsys_initcall(hugepage_init);
455 
456 static int __init setup_transparent_hugepage(char *str)
457 {
458         int ret = 0;
459         if (!str)
460                 goto out;
461         if (!strcmp(str, "always")) {
462                 set_bit(TRANSPARENT_HUGEPAGE_FLAG,
463                         &transparent_hugepage_flags);
464                 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
465                           &transparent_hugepage_flags);
466                 ret = 1;
467         } else if (!strcmp(str, "madvise")) {
468                 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
469                           &transparent_hugepage_flags);
470                 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
471                         &transparent_hugepage_flags);
472                 ret = 1;
473         } else if (!strcmp(str, "never")) {
474                 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
475                           &transparent_hugepage_flags);
476                 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
477                           &transparent_hugepage_flags);
478                 ret = 1;
479         }
480 out:
481         if (!ret)
482                 pr_warn("transparent_hugepage= cannot parse, ignored\n");
483         return ret;
484 }
485 __setup("transparent_hugepage=", setup_transparent_hugepage);
486 
487 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
488 {
489         if (likely(vma->vm_flags & VM_WRITE))
490                 pmd = pmd_mkwrite(pmd);
491         return pmd;
492 }
493 
494 static inline struct list_head *page_deferred_list(struct page *page)
495 {
496         /* ->lru in the tail pages is occupied by compound_head. */
497         return &page[2].deferred_list;
498 }
499 
500 void prep_transhuge_page(struct page *page)
501 {
502         /*
503          * we use page->mapping and page->indexlru in second tail page
504          * as list_head: assuming THP order >= 2
505          */
506 
507         INIT_LIST_HEAD(page_deferred_list(page));
508         set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR);
509 }
510 
511 unsigned long __thp_get_unmapped_area(struct file *filp, unsigned long len,
512                 loff_t off, unsigned long flags, unsigned long size)
513 {
514         unsigned long addr;
515         loff_t off_end = off + len;
516         loff_t off_align = round_up(off, size);
517         unsigned long len_pad;
518 
519         if (off_end <= off_align || (off_end - off_align) < size)
520                 return 0;
521 
522         len_pad = len + size;
523         if (len_pad < len || (off + len_pad) < off)
524                 return 0;
525 
526         addr = current->mm->get_unmapped_area(filp, 0, len_pad,
527                                               off >> PAGE_SHIFT, flags);
528         if (IS_ERR_VALUE(addr))
529                 return 0;
530 
531         addr += (off - addr) & (size - 1);
532         return addr;
533 }
534 
535 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
536                 unsigned long len, unsigned long pgoff, unsigned long flags)
537 {
538         loff_t off = (loff_t)pgoff << PAGE_SHIFT;
539 
540         if (addr)
541                 goto out;
542         if (!IS_DAX(filp->f_mapping->host) || !IS_ENABLED(CONFIG_FS_DAX_PMD))
543                 goto out;
544 
545         addr = __thp_get_unmapped_area(filp, len, off, flags, PMD_SIZE);
546         if (addr)
547                 return addr;
548 
549  out:
550         return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
551 }
552 EXPORT_SYMBOL_GPL(thp_get_unmapped_area);
553 
554 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf,
555                         struct page *page, gfp_t gfp)
556 {
557         struct vm_area_struct *vma = vmf->vma;
558         struct mem_cgroup *memcg;
559         pgtable_t pgtable;
560         unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
561         vm_fault_t ret = 0;
562 
563         VM_BUG_ON_PAGE(!PageCompound(page), page);
564 
565         if (mem_cgroup_try_charge_delay(page, vma->vm_mm, gfp, &memcg, true)) {
566                 put_page(page);
567                 count_vm_event(THP_FAULT_FALLBACK);
568                 return VM_FAULT_FALLBACK;
569         }
570 
571         pgtable = pte_alloc_one(vma->vm_mm);
572         if (unlikely(!pgtable)) {
573                 ret = VM_FAULT_OOM;
574                 goto release;
575         }
576 
577         clear_huge_page(page, vmf->address, HPAGE_PMD_NR);
578         /*
579          * The memory barrier inside __SetPageUptodate makes sure that
580          * clear_huge_page writes become visible before the set_pmd_at()
581          * write.
582          */
583         __SetPageUptodate(page);
584 
585         vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
586         if (unlikely(!pmd_none(*vmf->pmd))) {
587                 goto unlock_release;
588         } else {
589                 pmd_t entry;
590 
591                 ret = check_stable_address_space(vma->vm_mm);
592                 if (ret)
593                         goto unlock_release;
594 
595                 /* Deliver the page fault to userland */
596                 if (userfaultfd_missing(vma)) {
597                         vm_fault_t ret2;
598 
599                         spin_unlock(vmf->ptl);
600                         mem_cgroup_cancel_charge(page, memcg, true);
601                         put_page(page);
602                         pte_free(vma->vm_mm, pgtable);
603                         ret2 = handle_userfault(vmf, VM_UFFD_MISSING);
604                         VM_BUG_ON(ret2 & VM_FAULT_FALLBACK);
605                         return ret2;
606                 }
607 
608                 entry = mk_huge_pmd(page, vma->vm_page_prot);
609                 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
610                 page_add_new_anon_rmap(page, vma, haddr, true);
611                 mem_cgroup_commit_charge(page, memcg, false, true);
612                 lru_cache_add_active_or_unevictable(page, vma);
613                 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);
614                 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
615                 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
616                 mm_inc_nr_ptes(vma->vm_mm);
617                 spin_unlock(vmf->ptl);
618                 count_vm_event(THP_FAULT_ALLOC);
619         }
620 
621         return 0;
622 unlock_release:
623         spin_unlock(vmf->ptl);
624 release:
625         if (pgtable)
626                 pte_free(vma->vm_mm, pgtable);
627         mem_cgroup_cancel_charge(page, memcg, true);
628         put_page(page);
629         return ret;
630 
631 }
632 
633 /*
634  * always: directly stall for all thp allocations
635  * defer: wake kswapd and fail if not immediately available
636  * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise
637  *                fail if not immediately available
638  * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately
639  *          available
640  * never: never stall for any thp allocation
641  */
642 static inline gfp_t alloc_hugepage_direct_gfpmask(struct vm_area_struct *vma)
643 {
644         const bool vma_madvised = !!(vma->vm_flags & VM_HUGEPAGE);
645 
646         /* Always do synchronous compaction */
647         if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
648                 return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY);
649 
650         /* Kick kcompactd and fail quickly */
651         if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
652                 return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM;
653 
654         /* Synchronous compaction if madvised, otherwise kick kcompactd */
655         if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags))
656                 return GFP_TRANSHUGE_LIGHT |
657                         (vma_madvised ? __GFP_DIRECT_RECLAIM :
658                                         __GFP_KSWAPD_RECLAIM);
659 
660         /* Only do synchronous compaction if madvised */
661         if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
662                 return GFP_TRANSHUGE_LIGHT |
663                        (vma_madvised ? __GFP_DIRECT_RECLAIM : 0);
664 
665         return GFP_TRANSHUGE_LIGHT;
666 }
667 
668 /* Caller must hold page table lock. */
669 static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
670                 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,
671                 struct page *zero_page)
672 {
673         pmd_t entry;
674         if (!pmd_none(*pmd))
675                 return false;
676         entry = mk_pmd(zero_page, vma->vm_page_prot);
677         entry = pmd_mkhuge(entry);
678         if (pgtable)
679                 pgtable_trans_huge_deposit(mm, pmd, pgtable);
680         set_pmd_at(mm, haddr, pmd, entry);
681         mm_inc_nr_ptes(mm);
682         return true;
683 }
684 
685 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf)
686 {
687         struct vm_area_struct *vma = vmf->vma;
688         gfp_t gfp;
689         struct page *page;
690         unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
691 
692         if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
693                 return VM_FAULT_FALLBACK;
694         if (unlikely(anon_vma_prepare(vma)))
695                 return VM_FAULT_OOM;
696         if (unlikely(khugepaged_enter(vma, vma->vm_flags)))
697                 return VM_FAULT_OOM;
698         if (!(vmf->flags & FAULT_FLAG_WRITE) &&
699                         !mm_forbids_zeropage(vma->vm_mm) &&
700                         transparent_hugepage_use_zero_page()) {
701                 pgtable_t pgtable;
702                 struct page *zero_page;
703                 bool set;
704                 vm_fault_t ret;
705                 pgtable = pte_alloc_one(vma->vm_mm);
706                 if (unlikely(!pgtable))
707                         return VM_FAULT_OOM;
708                 zero_page = mm_get_huge_zero_page(vma->vm_mm);
709                 if (unlikely(!zero_page)) {
710                         pte_free(vma->vm_mm, pgtable);
711                         count_vm_event(THP_FAULT_FALLBACK);
712                         return VM_FAULT_FALLBACK;
713                 }
714                 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
715                 ret = 0;
716                 set = false;
717                 if (pmd_none(*vmf->pmd)) {
718                         ret = check_stable_address_space(vma->vm_mm);
719                         if (ret) {
720                                 spin_unlock(vmf->ptl);
721                         } else if (userfaultfd_missing(vma)) {
722                                 spin_unlock(vmf->ptl);
723                                 ret = handle_userfault(vmf, VM_UFFD_MISSING);
724                                 VM_BUG_ON(ret & VM_FAULT_FALLBACK);
725                         } else {
726                                 set_huge_zero_page(pgtable, vma->vm_mm, vma,
727                                                    haddr, vmf->pmd, zero_page);
728                                 spin_unlock(vmf->ptl);
729                                 set = true;
730                         }
731                 } else
732                         spin_unlock(vmf->ptl);
733                 if (!set)
734                         pte_free(vma->vm_mm, pgtable);
735                 return ret;
736         }
737         gfp = alloc_hugepage_direct_gfpmask(vma);
738         page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER);
739         if (unlikely(!page)) {
740                 count_vm_event(THP_FAULT_FALLBACK);
741                 return VM_FAULT_FALLBACK;
742         }
743         prep_transhuge_page(page);
744         return __do_huge_pmd_anonymous_page(vmf, page, gfp);
745 }
746 
747 static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
748                 pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write,
749                 pgtable_t pgtable)
750 {
751         struct mm_struct *mm = vma->vm_mm;
752         pmd_t entry;
753         spinlock_t *ptl;
754 
755         ptl = pmd_lock(mm, pmd);
756         if (!pmd_none(*pmd)) {
757                 if (write) {
758                         if (pmd_pfn(*pmd) != pfn_t_to_pfn(pfn)) {
759                                 WARN_ON_ONCE(!is_huge_zero_pmd(*pmd));
760                                 goto out_unlock;
761                         }
762                         entry = pmd_mkyoung(*pmd);
763                         entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
764                         if (pmdp_set_access_flags(vma, addr, pmd, entry, 1))
765                                 update_mmu_cache_pmd(vma, addr, pmd);
766                 }
767 
768                 goto out_unlock;
769         }
770 
771         entry = pmd_mkhuge(pfn_t_pmd(pfn, prot));
772         if (pfn_t_devmap(pfn))
773                 entry = pmd_mkdevmap(entry);
774         if (write) {
775                 entry = pmd_mkyoung(pmd_mkdirty(entry));
776                 entry = maybe_pmd_mkwrite(entry, vma);
777         }
778 
779         if (pgtable) {
780                 pgtable_trans_huge_deposit(mm, pmd, pgtable);
781                 mm_inc_nr_ptes(mm);
782                 pgtable = NULL;
783         }
784 
785         set_pmd_at(mm, addr, pmd, entry);
786         update_mmu_cache_pmd(vma, addr, pmd);
787 
788 out_unlock:
789         spin_unlock(ptl);
790         if (pgtable)
791                 pte_free(mm, pgtable);
792 }
793 
794 vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write)
795 {
796         unsigned long addr = vmf->address & PMD_MASK;
797         struct vm_area_struct *vma = vmf->vma;
798         pgprot_t pgprot = vma->vm_page_prot;
799         pgtable_t pgtable = NULL;
800 
801         /*
802          * If we had pmd_special, we could avoid all these restrictions,
803          * but we need to be consistent with PTEs and architectures that
804          * can't support a 'special' bit.
805          */
806         BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) &&
807                         !pfn_t_devmap(pfn));
808         BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
809                                                 (VM_PFNMAP|VM_MIXEDMAP));
810         BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
811 
812         if (addr < vma->vm_start || addr >= vma->vm_end)
813                 return VM_FAULT_SIGBUS;
814 
815         if (arch_needs_pgtable_deposit()) {
816                 pgtable = pte_alloc_one(vma->vm_mm);
817                 if (!pgtable)
818                         return VM_FAULT_OOM;
819         }
820 
821         track_pfn_insert(vma, &pgprot, pfn);
822 
823         insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable);
824         return VM_FAULT_NOPAGE;
825 }
826 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd);
827 
828 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
829 static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma)
830 {
831         if (likely(vma->vm_flags & VM_WRITE))
832                 pud = pud_mkwrite(pud);
833         return pud;
834 }
835 
836 static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr,
837                 pud_t *pud, pfn_t pfn, pgprot_t prot, bool write)
838 {
839         struct mm_struct *mm = vma->vm_mm;
840         pud_t entry;
841         spinlock_t *ptl;
842 
843         ptl = pud_lock(mm, pud);
844         if (!pud_none(*pud)) {
845                 if (write) {
846                         if (pud_pfn(*pud) != pfn_t_to_pfn(pfn)) {
847                                 WARN_ON_ONCE(!is_huge_zero_pud(*pud));
848                                 goto out_unlock;
849                         }
850                         entry = pud_mkyoung(*pud);
851                         entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma);
852                         if (pudp_set_access_flags(vma, addr, pud, entry, 1))
853                                 update_mmu_cache_pud(vma, addr, pud);
854                 }
855                 goto out_unlock;
856         }
857 
858         entry = pud_mkhuge(pfn_t_pud(pfn, prot));
859         if (pfn_t_devmap(pfn))
860                 entry = pud_mkdevmap(entry);
861         if (write) {
862                 entry = pud_mkyoung(pud_mkdirty(entry));
863                 entry = maybe_pud_mkwrite(entry, vma);
864         }
865         set_pud_at(mm, addr, pud, entry);
866         update_mmu_cache_pud(vma, addr, pud);
867 
868 out_unlock:
869         spin_unlock(ptl);
870 }
871 
872 vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write)
873 {
874         unsigned long addr = vmf->address & PUD_MASK;
875         struct vm_area_struct *vma = vmf->vma;
876         pgprot_t pgprot = vma->vm_page_prot;
877 
878         /*
879          * If we had pud_special, we could avoid all these restrictions,
880          * but we need to be consistent with PTEs and architectures that
881          * can't support a 'special' bit.
882          */
883         BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) &&
884                         !pfn_t_devmap(pfn));
885         BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
886                                                 (VM_PFNMAP|VM_MIXEDMAP));
887         BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
888 
889         if (addr < vma->vm_start || addr >= vma->vm_end)
890                 return VM_FAULT_SIGBUS;
891 
892         track_pfn_insert(vma, &pgprot, pfn);
893 
894         insert_pfn_pud(vma, addr, vmf->pud, pfn, pgprot, write);
895         return VM_FAULT_NOPAGE;
896 }
897 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud);
898 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
899 
900 static void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
901                 pmd_t *pmd, int flags)
902 {
903         pmd_t _pmd;
904 
905         _pmd = pmd_mkyoung(*pmd);
906         if (flags & FOLL_WRITE)
907                 _pmd = pmd_mkdirty(_pmd);
908         if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK,
909                                 pmd, _pmd, flags & FOLL_WRITE))
910                 update_mmu_cache_pmd(vma, addr, pmd);
911 }
912 
913 struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
914                 pmd_t *pmd, int flags, struct dev_pagemap **pgmap)
915 {
916         unsigned long pfn = pmd_pfn(*pmd);
917         struct mm_struct *mm = vma->vm_mm;
918         struct page *page;
919 
920         assert_spin_locked(pmd_lockptr(mm, pmd));
921 
922         /*
923          * When we COW a devmap PMD entry, we split it into PTEs, so we should
924          * not be in this function with `flags & FOLL_COW` set.
925          */
926         WARN_ONCE(flags & FOLL_COW, "mm: In follow_devmap_pmd with FOLL_COW set");
927 
928         if (flags & FOLL_WRITE && !pmd_write(*pmd))
929                 return NULL;
930 
931         if (pmd_present(*pmd) && pmd_devmap(*pmd))
932                 /* pass */;
933         else
934                 return NULL;
935 
936         if (flags & FOLL_TOUCH)
937                 touch_pmd(vma, addr, pmd, flags);
938 
939         /*
940          * device mapped pages can only be returned if the
941          * caller will manage the page reference count.
942          */
943         if (!(flags & FOLL_GET))
944                 return ERR_PTR(-EEXIST);
945 
946         pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT;
947         *pgmap = get_dev_pagemap(pfn, *pgmap);
948         if (!*pgmap)
949                 return ERR_PTR(-EFAULT);
950         page = pfn_to_page(pfn);
951         get_page(page);
952 
953         return page;
954 }
955 
956 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
957                   pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
958                   struct vm_area_struct *vma)
959 {
960         spinlock_t *dst_ptl, *src_ptl;
961         struct page *src_page;
962         pmd_t pmd;
963         pgtable_t pgtable = NULL;
964         int ret = -ENOMEM;
965 
966         /* Skip if can be re-fill on fault */
967         if (!vma_is_anonymous(vma))
968                 return 0;
969 
970         pgtable = pte_alloc_one(dst_mm);
971         if (unlikely(!pgtable))
972                 goto out;
973 
974         dst_ptl = pmd_lock(dst_mm, dst_pmd);
975         src_ptl = pmd_lockptr(src_mm, src_pmd);
976         spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
977 
978         ret = -EAGAIN;
979         pmd = *src_pmd;
980 
981 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
982         if (unlikely(is_swap_pmd(pmd))) {
983                 swp_entry_t entry = pmd_to_swp_entry(pmd);
984 
985                 VM_BUG_ON(!is_pmd_migration_entry(pmd));
986                 if (is_write_migration_entry(entry)) {
987                         make_migration_entry_read(&entry);
988                         pmd = swp_entry_to_pmd(entry);
989                         if (pmd_swp_soft_dirty(*src_pmd))
990                                 pmd = pmd_swp_mksoft_dirty(pmd);
991                         set_pmd_at(src_mm, addr, src_pmd, pmd);
992                 }
993                 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
994                 mm_inc_nr_ptes(dst_mm);
995                 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
996                 set_pmd_at(dst_mm, addr, dst_pmd, pmd);
997                 ret = 0;
998                 goto out_unlock;
999         }
1000 #endif
1001 
1002         if (unlikely(!pmd_trans_huge(pmd))) {
1003                 pte_free(dst_mm, pgtable);
1004                 goto out_unlock;
1005         }
1006         /*
1007          * When page table lock is held, the huge zero pmd should not be
1008          * under splitting since we don't split the page itself, only pmd to
1009          * a page table.
1010          */
1011         if (is_huge_zero_pmd(pmd)) {
1012                 struct page *zero_page;
1013                 /*
1014                  * get_huge_zero_page() will never allocate a new page here,
1015                  * since we already have a zero page to copy. It just takes a
1016                  * reference.
1017                  */
1018                 zero_page = mm_get_huge_zero_page(dst_mm);
1019                 set_huge_zero_page(pgtable, dst_mm, vma, addr, dst_pmd,
1020                                 zero_page);
1021                 ret = 0;
1022                 goto out_unlock;
1023         }
1024 
1025         src_page = pmd_page(pmd);
1026         VM_BUG_ON_PAGE(!PageHead(src_page), src_page);
1027         get_page(src_page);
1028         page_dup_rmap(src_page, true);
1029         add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
1030         mm_inc_nr_ptes(dst_mm);
1031         pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
1032 
1033         pmdp_set_wrprotect(src_mm, addr, src_pmd);
1034         pmd = pmd_mkold(pmd_wrprotect(pmd));
1035         set_pmd_at(dst_mm, addr, dst_pmd, pmd);
1036 
1037         ret = 0;
1038 out_unlock:
1039         spin_unlock(src_ptl);
1040         spin_unlock(dst_ptl);
1041 out:
1042         return ret;
1043 }
1044 
1045 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
1046 static void touch_pud(struct vm_area_struct *vma, unsigned long addr,
1047                 pud_t *pud, int flags)
1048 {
1049         pud_t _pud;
1050 
1051         _pud = pud_mkyoung(*pud);
1052         if (flags & FOLL_WRITE)
1053                 _pud = pud_mkdirty(_pud);
1054         if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK,
1055                                 pud, _pud, flags & FOLL_WRITE))
1056                 update_mmu_cache_pud(vma, addr, pud);
1057 }
1058 
1059 struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr,
1060                 pud_t *pud, int flags, struct dev_pagemap **pgmap)
1061 {
1062         unsigned long pfn = pud_pfn(*pud);
1063         struct mm_struct *mm = vma->vm_mm;
1064         struct page *page;
1065 
1066         assert_spin_locked(pud_lockptr(mm, pud));
1067 
1068         if (flags & FOLL_WRITE && !pud_write(*pud))
1069                 return NULL;
1070 
1071         if (pud_present(*pud) && pud_devmap(*pud))
1072                 /* pass */;
1073         else
1074                 return NULL;
1075 
1076         if (flags & FOLL_TOUCH)
1077                 touch_pud(vma, addr, pud, flags);
1078 
1079         /*
1080          * device mapped pages can only be returned if the
1081          * caller will manage the page reference count.
1082          */
1083         if (!(flags & FOLL_GET))
1084                 return ERR_PTR(-EEXIST);
1085 
1086         pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT;
1087         *pgmap = get_dev_pagemap(pfn, *pgmap);
1088         if (!*pgmap)
1089                 return ERR_PTR(-EFAULT);
1090         page = pfn_to_page(pfn);
1091         get_page(page);
1092 
1093         return page;
1094 }
1095 
1096 int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1097                   pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
1098                   struct vm_area_struct *vma)
1099 {
1100         spinlock_t *dst_ptl, *src_ptl;
1101         pud_t pud;
1102         int ret;
1103 
1104         dst_ptl = pud_lock(dst_mm, dst_pud);
1105         src_ptl = pud_lockptr(src_mm, src_pud);
1106         spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
1107 
1108         ret = -EAGAIN;
1109         pud = *src_pud;
1110         if (unlikely(!pud_trans_huge(pud) && !pud_devmap(pud)))
1111                 goto out_unlock;
1112 
1113         /*
1114          * When page table lock is held, the huge zero pud should not be
1115          * under splitting since we don't split the page itself, only pud to
1116          * a page table.
1117          */
1118         if (is_huge_zero_pud(pud)) {
1119                 /* No huge zero pud yet */
1120         }
1121 
1122         pudp_set_wrprotect(src_mm, addr, src_pud);
1123         pud = pud_mkold(pud_wrprotect(pud));
1124         set_pud_at(dst_mm, addr, dst_pud, pud);
1125 
1126         ret = 0;
1127 out_unlock:
1128         spin_unlock(src_ptl);
1129         spin_unlock(dst_ptl);
1130         return ret;
1131 }
1132 
1133 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)
1134 {
1135         pud_t entry;
1136         unsigned long haddr;
1137         bool write = vmf->flags & FAULT_FLAG_WRITE;
1138 
1139         vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud);
1140         if (unlikely(!pud_same(*vmf->pud, orig_pud)))
1141                 goto unlock;
1142 
1143         entry = pud_mkyoung(orig_pud);
1144         if (write)
1145                 entry = pud_mkdirty(entry);
1146         haddr = vmf->address & HPAGE_PUD_MASK;
1147         if (pudp_set_access_flags(vmf->vma, haddr, vmf->pud, entry, write))
1148                 update_mmu_cache_pud(vmf->vma, vmf->address, vmf->pud);
1149 
1150 unlock:
1151         spin_unlock(vmf->ptl);
1152 }
1153 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
1154 
1155 void huge_pmd_set_accessed(struct vm_fault *vmf, pmd_t orig_pmd)
1156 {
1157         pmd_t entry;
1158         unsigned long haddr;
1159         bool write = vmf->flags & FAULT_FLAG_WRITE;
1160 
1161         vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);
1162         if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))
1163                 goto unlock;
1164 
1165         entry = pmd_mkyoung(orig_pmd);
1166         if (write)
1167                 entry = pmd_mkdirty(entry);
1168         haddr = vmf->address & HPAGE_PMD_MASK;
1169         if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write))
1170                 update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd);
1171 
1172 unlock:
1173         spin_unlock(vmf->ptl);
1174 }
1175 
1176 static vm_fault_t do_huge_pmd_wp_page_fallback(struct vm_fault *vmf,
1177                         pmd_t orig_pmd, struct page *page)
1178 {
1179         struct vm_area_struct *vma = vmf->vma;
1180         unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
1181         struct mem_cgroup *memcg;
1182         pgtable_t pgtable;
1183         pmd_t _pmd;
1184         int i;
1185         vm_fault_t ret = 0;
1186         struct page **pages;
1187         struct mmu_notifier_range range;
1188 
1189         pages = kmalloc_array(HPAGE_PMD_NR, sizeof(struct page *),
1190                               GFP_KERNEL);
1191         if (unlikely(!pages)) {
1192                 ret |= VM_FAULT_OOM;
1193                 goto out;
1194         }
1195 
1196         for (i = 0; i < HPAGE_PMD_NR; i++) {
1197                 pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE, vma,
1198                                                vmf->address, page_to_nid(page));
1199                 if (unlikely(!pages[i] ||
1200                              mem_cgroup_try_charge_delay(pages[i], vma->vm_mm,
1201                                      GFP_KERNEL, &memcg, false))) {
1202                         if (pages[i])
1203                                 put_page(pages[i]);
1204                         while (--i >= 0) {
1205                                 memcg = (void *)page_private(pages[i]);
1206                                 set_page_private(pages[i], 0);
1207                                 mem_cgroup_cancel_charge(pages[i], memcg,
1208                                                 false);
1209                                 put_page(pages[i]);
1210                         }
1211                         kfree(pages);
1212                         ret |= VM_FAULT_OOM;
1213                         goto out;
1214                 }
1215                 set_page_private(pages[i], (unsigned long)memcg);
1216         }
1217 
1218         for (i = 0; i < HPAGE_PMD_NR; i++) {
1219                 copy_user_highpage(pages[i], page + i,
1220                                    haddr + PAGE_SIZE * i, vma);
1221                 __SetPageUptodate(pages[i]);
1222                 cond_resched();
1223         }
1224 
1225         mmu_notifier_range_init(&range, vma->vm_mm, haddr,
1226                                 haddr + HPAGE_PMD_SIZE);
1227         mmu_notifier_invalidate_range_start(&range);
1228 
1229         vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
1230         if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))
1231                 goto out_free_pages;
1232         VM_BUG_ON_PAGE(!PageHead(page), page);
1233 
1234         /*
1235          * Leave pmd empty until pte is filled note we must notify here as
1236          * concurrent CPU thread might write to new page before the call to
1237          * mmu_notifier_invalidate_range_end() happens which can lead to a
1238          * device seeing memory write in different order than CPU.
1239          *
1240          * See Documentation/vm/mmu_notifier.rst
1241          */
1242         pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);
1243 
1244         pgtable = pgtable_trans_huge_withdraw(vma->vm_mm, vmf->pmd);
1245         pmd_populate(vma->vm_mm, &_pmd, pgtable);
1246 
1247         for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
1248                 pte_t entry;
1249                 entry = mk_pte(pages[i], vma->vm_page_prot);
1250                 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
1251                 memcg = (void *)page_private(pages[i]);
1252                 set_page_private(pages[i], 0);
1253                 page_add_new_anon_rmap(pages[i], vmf->vma, haddr, false);
1254                 mem_cgroup_commit_charge(pages[i], memcg, false, false);
1255                 lru_cache_add_active_or_unevictable(pages[i], vma);
1256                 vmf->pte = pte_offset_map(&_pmd, haddr);
1257                 VM_BUG_ON(!pte_none(*vmf->pte));
1258                 set_pte_at(vma->vm_mm, haddr, vmf->pte, entry);
1259                 pte_unmap(vmf->pte);
1260         }
1261         kfree(pages);
1262 
1263         smp_wmb(); /* make pte visible before pmd */
1264         pmd_populate(vma->vm_mm, vmf->pmd, pgtable);
1265         page_remove_rmap(page, true);
1266         spin_unlock(vmf->ptl);
1267 
1268         /*
1269          * No need to double call mmu_notifier->invalidate_range() callback as
1270          * the above pmdp_huge_clear_flush_notify() did already call it.
1271          */
1272         mmu_notifier_invalidate_range_only_end(&range);
1273 
1274         ret |= VM_FAULT_WRITE;
1275         put_page(page);
1276 
1277 out:
1278         return ret;
1279 
1280 out_free_pages:
1281         spin_unlock(vmf->ptl);
1282         mmu_notifier_invalidate_range_end(&range);
1283         for (i = 0; i < HPAGE_PMD_NR; i++) {
1284                 memcg = (void *)page_private(pages[i]);
1285                 set_page_private(pages[i], 0);
1286                 mem_cgroup_cancel_charge(pages[i], memcg, false);
1287                 put_page(pages[i]);
1288         }
1289         kfree(pages);
1290         goto out;
1291 }
1292 
1293 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd)
1294 {
1295         struct vm_area_struct *vma = vmf->vma;
1296         struct page *page = NULL, *new_page;
1297         struct mem_cgroup *memcg;
1298         unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
1299         struct mmu_notifier_range range;
1300         gfp_t huge_gfp;                 /* for allocation and charge */
1301         vm_fault_t ret = 0;
1302 
1303         vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);
1304         VM_BUG_ON_VMA(!vma->anon_vma, vma);
1305         if (is_huge_zero_pmd(orig_pmd))
1306                 goto alloc;
1307         spin_lock(vmf->ptl);
1308         if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))
1309                 goto out_unlock;
1310 
1311         page = pmd_page(orig_pmd);
1312         VM_BUG_ON_PAGE(!PageCompound(page) || !PageHead(page), page);
1313         /*
1314          * We can only reuse the page if nobody else maps the huge page or it's
1315          * part.
1316          */
1317         if (!trylock_page(page)) {
1318                 get_page(page);
1319                 spin_unlock(vmf->ptl);
1320                 lock_page(page);
1321                 spin_lock(vmf->ptl);
1322                 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
1323                         unlock_page(page);
1324                         put_page(page);
1325                         goto out_unlock;
1326                 }
1327                 put_page(page);
1328         }
1329         if (reuse_swap_page(page, NULL)) {
1330                 pmd_t entry;
1331                 entry = pmd_mkyoung(orig_pmd);
1332                 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
1333                 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry,  1))
1334                         update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
1335                 ret |= VM_FAULT_WRITE;
1336                 unlock_page(page);
1337                 goto out_unlock;
1338         }
1339         unlock_page(page);
1340         get_page(page);
1341         spin_unlock(vmf->ptl);
1342 alloc:
1343         if (__transparent_hugepage_enabled(vma) &&
1344             !transparent_hugepage_debug_cow()) {
1345                 huge_gfp = alloc_hugepage_direct_gfpmask(vma);
1346                 new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER);
1347         } else
1348                 new_page = NULL;
1349 
1350         if (likely(new_page)) {
1351                 prep_transhuge_page(new_page);
1352         } else {
1353                 if (!page) {
1354                         split_huge_pmd(vma, vmf->pmd, vmf->address);
1355                         ret |= VM_FAULT_FALLBACK;
1356                 } else {
1357                         ret = do_huge_pmd_wp_page_fallback(vmf, orig_pmd, page);
1358                         if (ret & VM_FAULT_OOM) {
1359                                 split_huge_pmd(vma, vmf->pmd, vmf->address);
1360                                 ret |= VM_FAULT_FALLBACK;
1361                         }
1362                         put_page(page);
1363                 }
1364                 count_vm_event(THP_FAULT_FALLBACK);
1365                 goto out;
1366         }
1367 
1368         if (unlikely(mem_cgroup_try_charge_delay(new_page, vma->vm_mm,
1369                                         huge_gfp, &memcg, true))) {
1370                 put_page(new_page);
1371                 split_huge_pmd(vma, vmf->pmd, vmf->address);
1372                 if (page)
1373                         put_page(page);
1374                 ret |= VM_FAULT_FALLBACK;
1375                 count_vm_event(THP_FAULT_FALLBACK);
1376                 goto out;
1377         }
1378 
1379         count_vm_event(THP_FAULT_ALLOC);
1380 
1381         if (!page)
1382                 clear_huge_page(new_page, vmf->address, HPAGE_PMD_NR);
1383         else
1384                 copy_user_huge_page(new_page, page, vmf->address,
1385                                     vma, HPAGE_PMD_NR);
1386         __SetPageUptodate(new_page);
1387 
1388         mmu_notifier_range_init(&range, vma->vm_mm, haddr,
1389                                 haddr + HPAGE_PMD_SIZE);
1390         mmu_notifier_invalidate_range_start(&range);
1391 
1392         spin_lock(vmf->ptl);
1393         if (page)
1394                 put_page(page);
1395         if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
1396                 spin_unlock(vmf->ptl);
1397                 mem_cgroup_cancel_charge(new_page, memcg, true);
1398                 put_page(new_page);
1399                 goto out_mn;
1400         } else {
1401                 pmd_t entry;
1402                 entry = mk_huge_pmd(new_page, vma->vm_page_prot);
1403                 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
1404                 pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);
1405                 page_add_new_anon_rmap(new_page, vma, haddr, true);
1406                 mem_cgroup_commit_charge(new_page, memcg, false, true);
1407                 lru_cache_add_active_or_unevictable(new_page, vma);
1408                 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
1409                 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
1410                 if (!page) {
1411                         add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
1412                 } else {
1413                         VM_BUG_ON_PAGE(!PageHead(page), page);
1414                         page_remove_rmap(page, true);
1415                         put_page(page);
1416                 }
1417                 ret |= VM_FAULT_WRITE;
1418         }
1419         spin_unlock(vmf->ptl);
1420 out_mn:
1421         /*
1422          * No need to double call mmu_notifier->invalidate_range() callback as
1423          * the above pmdp_huge_clear_flush_notify() did already call it.
1424          */
1425         mmu_notifier_invalidate_range_only_end(&range);
1426 out:
1427         return ret;
1428 out_unlock:
1429         spin_unlock(vmf->ptl);
1430         return ret;
1431 }
1432 
1433 /*
1434  * FOLL_FORCE can write to even unwritable pmd's, but only
1435  * after we've gone through a COW cycle and they are dirty.
1436  */
1437 static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags)
1438 {
1439         return pmd_write(pmd) ||
1440                ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pmd_dirty(pmd));
1441 }
1442 
1443 struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
1444                                    unsigned long addr,
1445                                    pmd_t *pmd,
1446                                    unsigned int flags)
1447 {
1448         struct mm_struct *mm = vma->vm_mm;
1449         struct page *page = NULL;
1450 
1451         assert_spin_locked(pmd_lockptr(mm, pmd));
1452 
1453         if (flags & FOLL_WRITE && !can_follow_write_pmd(*pmd, flags))
1454                 goto out;
1455 
1456         /* Avoid dumping huge zero page */
1457         if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))
1458                 return ERR_PTR(-EFAULT);
1459 
1460         /* Full NUMA hinting faults to serialise migration in fault paths */
1461         if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))
1462                 goto out;
1463 
1464         page = pmd_page(*pmd);
1465         VM_BUG_ON_PAGE(!PageHead(page) && !is_zone_device_page(page), page);
1466         if (flags & FOLL_TOUCH)
1467                 touch_pmd(vma, addr, pmd, flags);
1468         if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
1469                 /*
1470                  * We don't mlock() pte-mapped THPs. This way we can avoid
1471                  * leaking mlocked pages into non-VM_LOCKED VMAs.
1472                  *
1473                  * For anon THP:
1474                  *
1475                  * In most cases the pmd is the only mapping of the page as we
1476                  * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for
1477                  * writable private mappings in populate_vma_page_range().
1478                  *
1479                  * The only scenario when we have the page shared here is if we
1480                  * mlocking read-only mapping shared over fork(). We skip
1481                  * mlocking such pages.
1482                  *
1483                  * For file THP:
1484                  *
1485                  * We can expect PageDoubleMap() to be stable under page lock:
1486                  * for file pages we set it in page_add_file_rmap(), which
1487                  * requires page to be locked.
1488                  */
1489 
1490                 if (PageAnon(page) && compound_mapcount(page) != 1)
1491                         goto skip_mlock;
1492                 if (PageDoubleMap(page) || !page->mapping)
1493                         goto skip_mlock;
1494                 if (!trylock_page(page))
1495                         goto skip_mlock;
1496                 lru_add_drain();
1497                 if (page->mapping && !PageDoubleMap(page))
1498                         mlock_vma_page(page);
1499                 unlock_page(page);
1500         }
1501 skip_mlock:
1502         page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
1503         VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page);
1504         if (flags & FOLL_GET)
1505                 get_page(page);
1506 
1507 out:
1508         return page;
1509 }
1510 
1511 /* NUMA hinting page fault entry point for trans huge pmds */
1512 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)
1513 {
1514         struct vm_area_struct *vma = vmf->vma;
1515         struct anon_vma *anon_vma = NULL;
1516         struct page *page;
1517         unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
1518         int page_nid = -1, this_nid = numa_node_id();
1519         int target_nid, last_cpupid = -1;
1520         bool page_locked;
1521         bool migrated = false;
1522         bool was_writable;
1523         int flags = 0;
1524 
1525         vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
1526         if (unlikely(!pmd_same(pmd, *vmf->pmd)))
1527                 goto out_unlock;
1528 
1529         /*
1530          * If there are potential migrations, wait for completion and retry
1531          * without disrupting NUMA hinting information. Do not relock and
1532          * check_same as the page may no longer be mapped.
1533          */
1534         if (unlikely(pmd_trans_migrating(*vmf->pmd))) {
1535                 page = pmd_page(*vmf->pmd);
1536                 if (!get_page_unless_zero(page))
1537                         goto out_unlock;
1538                 spin_unlock(vmf->ptl);
1539                 put_and_wait_on_page_locked(page);
1540                 goto out;
1541         }
1542 
1543         page = pmd_page(pmd);
1544         BUG_ON(is_huge_zero_page(page));
1545         page_nid = page_to_nid(page);
1546         last_cpupid = page_cpupid_last(page);
1547         count_vm_numa_event(NUMA_HINT_FAULTS);
1548         if (page_nid == this_nid) {
1549                 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
1550                 flags |= TNF_FAULT_LOCAL;
1551         }
1552 
1553         /* See similar comment in do_numa_page for explanation */
1554         if (!pmd_savedwrite(pmd))
1555                 flags |= TNF_NO_GROUP;
1556 
1557         /*
1558          * Acquire the page lock to serialise THP migrations but avoid dropping
1559          * page_table_lock if at all possible
1560          */
1561         page_locked = trylock_page(page);
1562         target_nid = mpol_misplaced(page, vma, haddr);
1563         if (target_nid == -1) {
1564                 /* If the page was locked, there are no parallel migrations */
1565                 if (page_locked)
1566                         goto clear_pmdnuma;
1567         }
1568 
1569         /* Migration could have started since the pmd_trans_migrating check */
1570         if (!page_locked) {
1571                 page_nid = -1;
1572                 if (!get_page_unless_zero(page))
1573                         goto out_unlock;
1574                 spin_unlock(vmf->ptl);
1575                 put_and_wait_on_page_locked(page);
1576                 goto out;
1577         }
1578 
1579         /*
1580          * Page is misplaced. Page lock serialises migrations. Acquire anon_vma
1581          * to serialises splits
1582          */
1583         get_page(page);
1584         spin_unlock(vmf->ptl);
1585         anon_vma = page_lock_anon_vma_read(page);
1586 
1587         /* Confirm the PMD did not change while page_table_lock was released */
1588         spin_lock(vmf->ptl);
1589         if (unlikely(!pmd_same(pmd, *vmf->pmd))) {
1590                 unlock_page(page);
1591                 put_page(page);
1592                 page_nid = -1;
1593                 goto out_unlock;
1594         }
1595 
1596         /* Bail if we fail to protect against THP splits for any reason */
1597         if (unlikely(!anon_vma)) {
1598                 put_page(page);
1599                 page_nid = -1;
1600                 goto clear_pmdnuma;
1601         }
1602 
1603         /*
1604          * Since we took the NUMA fault, we must have observed the !accessible
1605          * bit. Make sure all other CPUs agree with that, to avoid them
1606          * modifying the page we're about to migrate.
1607          *
1608          * Must be done under PTL such that we'll observe the relevant
1609          * inc_tlb_flush_pending().
1610          *
1611          * We are not sure a pending tlb flush here is for a huge page
1612          * mapping or not. Hence use the tlb range variant
1613          */
1614         if (mm_tlb_flush_pending(vma->vm_mm)) {
1615                 flush_tlb_range(vma, haddr, haddr + HPAGE_PMD_SIZE);
1616                 /*
1617                  * change_huge_pmd() released the pmd lock before
1618                  * invalidating the secondary MMUs sharing the primary
1619                  * MMU pagetables (with ->invalidate_range()). The
1620                  * mmu_notifier_invalidate_range_end() (which
1621                  * internally calls ->invalidate_range()) in
1622                  * change_pmd_range() will run after us, so we can't
1623                  * rely on it here and we need an explicit invalidate.
1624                  */
1625                 mmu_notifier_invalidate_range(vma->vm_mm, haddr,
1626                                               haddr + HPAGE_PMD_SIZE);
1627         }
1628 
1629         /*
1630          * Migrate the THP to the requested node, returns with page unlocked
1631          * and access rights restored.
1632          */
1633         spin_unlock(vmf->ptl);
1634 
1635         migrated = migrate_misplaced_transhuge_page(vma->vm_mm, vma,
1636                                 vmf->pmd, pmd, vmf->address, page, target_nid);
1637         if (migrated) {
1638                 flags |= TNF_MIGRATED;
1639                 page_nid = target_nid;
1640         } else
1641                 flags |= TNF_MIGRATE_FAIL;
1642 
1643         goto out;
1644 clear_pmdnuma:
1645         BUG_ON(!PageLocked(page));
1646         was_writable = pmd_savedwrite(pmd);
1647         pmd = pmd_modify(pmd, vma->vm_page_prot);
1648         pmd = pmd_mkyoung(pmd);
1649         if (was_writable)
1650                 pmd = pmd_mkwrite(pmd);
1651         set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);
1652         update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
1653         unlock_page(page);
1654 out_unlock:
1655         spin_unlock(vmf->ptl);
1656 
1657 out:
1658         if (anon_vma)
1659                 page_unlock_anon_vma_read(anon_vma);
1660 
1661         if (page_nid != -1)
1662                 task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR,
1663                                 flags);
1664 
1665         return 0;
1666 }
1667 
1668 /*
1669  * Return true if we do MADV_FREE successfully on entire pmd page.
1670  * Otherwise, return false.
1671  */
1672 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
1673                 pmd_t *pmd, unsigned long addr, unsigned long next)
1674 {
1675         spinlock_t *ptl;
1676         pmd_t orig_pmd;
1677         struct page *page;
1678         struct mm_struct *mm = tlb->mm;
1679         bool ret = false;
1680 
1681         tlb_remove_check_page_size_change(tlb, HPAGE_PMD_SIZE);
1682 
1683         ptl = pmd_trans_huge_lock(pmd, vma);
1684         if (!ptl)
1685                 goto out_unlocked;
1686 
1687         orig_pmd = *pmd;
1688         if (is_huge_zero_pmd(orig_pmd))
1689                 goto out;
1690 
1691         if (unlikely(!pmd_present(orig_pmd))) {
1692                 VM_BUG_ON(thp_migration_supported() &&
1693                                   !is_pmd_migration_entry(orig_pmd));
1694                 goto out;
1695         }
1696 
1697         page = pmd_page(orig_pmd);
1698         /*
1699          * If other processes are mapping this page, we couldn't discard
1700          * the page unless they all do MADV_FREE so let's skip the page.
1701          */
1702         if (page_mapcount(page) != 1)
1703                 goto out;
1704 
1705         if (!trylock_page(page))
1706                 goto out;
1707 
1708         /*
1709          * If user want to discard part-pages of THP, split it so MADV_FREE
1710          * will deactivate only them.
1711          */
1712         if (next - addr != HPAGE_PMD_SIZE) {
1713                 get_page(page);
1714                 spin_unlock(ptl);
1715                 split_huge_page(page);
1716                 unlock_page(page);
1717                 put_page(page);
1718                 goto out_unlocked;
1719         }
1720 
1721         if (PageDirty(page))
1722                 ClearPageDirty(page);
1723         unlock_page(page);
1724 
1725         if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) {
1726                 pmdp_invalidate(vma, addr, pmd);
1727                 orig_pmd = pmd_mkold(orig_pmd);
1728                 orig_pmd = pmd_mkclean(orig_pmd);
1729 
1730                 set_pmd_at(mm, addr, pmd, orig_pmd);
1731                 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1732         }
1733 
1734         mark_page_lazyfree(page);
1735         ret = true;
1736 out:
1737         spin_unlock(ptl);
1738 out_unlocked:
1739         return ret;
1740 }
1741 
1742 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)
1743 {
1744         pgtable_t pgtable;
1745 
1746         pgtable = pgtable_trans_huge_withdraw(mm, pmd);
1747         pte_free(mm, pgtable);
1748         mm_dec_nr_ptes(mm);
1749 }
1750 
1751 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
1752                  pmd_t *pmd, unsigned long addr)
1753 {
1754         pmd_t orig_pmd;
1755         spinlock_t *ptl;
1756 
1757         tlb_remove_check_page_size_change(tlb, HPAGE_PMD_SIZE);
1758 
1759         ptl = __pmd_trans_huge_lock(pmd, vma);
1760         if (!ptl)
1761                 return 0;
1762         /*
1763          * For architectures like ppc64 we look at deposited pgtable
1764          * when calling pmdp_huge_get_and_clear. So do the
1765          * pgtable_trans_huge_withdraw after finishing pmdp related
1766          * operations.
1767          */
1768         orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
1769                         tlb->fullmm);
1770         tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1771         if (vma_is_dax(vma)) {
1772                 if (arch_needs_pgtable_deposit())
1773                         zap_deposited_table(tlb->mm, pmd);
1774                 spin_unlock(ptl);
1775                 if (is_huge_zero_pmd(orig_pmd))
1776                         tlb_remove_page_size(tlb, pmd_page(orig_pmd), HPAGE_PMD_SIZE);
1777         } else if (is_huge_zero_pmd(orig_pmd)) {
1778                 zap_deposited_table(tlb->mm, pmd);
1779                 spin_unlock(ptl);
1780                 tlb_remove_page_size(tlb, pmd_page(orig_pmd), HPAGE_PMD_SIZE);
1781         } else {
1782                 struct page *page = NULL;
1783                 int flush_needed = 1;
1784 
1785                 if (pmd_present(orig_pmd)) {
1786                         page = pmd_page(orig_pmd);
1787                         page_remove_rmap(page, true);
1788                         VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
1789                         VM_BUG_ON_PAGE(!PageHead(page), page);
1790                 } else if (thp_migration_supported()) {
1791                         swp_entry_t entry;
1792 
1793                         VM_BUG_ON(!is_pmd_migration_entry(orig_pmd));
1794                         entry = pmd_to_swp_entry(orig_pmd);
1795                         page = pfn_to_page(swp_offset(entry));
1796                         flush_needed = 0;
1797                 } else
1798                         WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
1799 
1800                 if (PageAnon(page)) {
1801                         zap_deposited_table(tlb->mm, pmd);
1802                         add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
1803                 } else {
1804                         if (arch_needs_pgtable_deposit())
1805                                 zap_deposited_table(tlb->mm, pmd);
1806                         add_mm_counter(tlb->mm, mm_counter_file(page), -HPAGE_PMD_NR);
1807                 }
1808 
1809                 spin_unlock(ptl);
1810                 if (flush_needed)
1811                         tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE);
1812         }
1813         return 1;
1814 }
1815 
1816 #ifndef pmd_move_must_withdraw
1817 static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl,
1818                                          spinlock_t *old_pmd_ptl,
1819                                          struct vm_area_struct *vma)
1820 {
1821         /*
1822          * With split pmd lock we also need to move preallocated
1823          * PTE page table if new_pmd is on different PMD page table.
1824          *
1825          * We also don't deposit and withdraw tables for file pages.
1826          */
1827         return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma);
1828 }
1829 #endif
1830 
1831 static pmd_t move_soft_dirty_pmd(pmd_t pmd)
1832 {
1833 #ifdef CONFIG_MEM_SOFT_DIRTY
1834         if (unlikely(is_pmd_migration_entry(pmd)))
1835                 pmd = pmd_swp_mksoft_dirty(pmd);
1836         else if (pmd_present(pmd))
1837                 pmd = pmd_mksoft_dirty(pmd);
1838 #endif
1839         return pmd;
1840 }
1841 
1842 bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
1843                   unsigned long new_addr, unsigned long old_end,
1844                   pmd_t *old_pmd, pmd_t *new_pmd)
1845 {
1846         spinlock_t *old_ptl, *new_ptl;
1847         pmd_t pmd;
1848         struct mm_struct *mm = vma->vm_mm;
1849         bool force_flush = false;
1850 
1851         if ((old_addr & ~HPAGE_PMD_MASK) ||
1852             (new_addr & ~HPAGE_PMD_MASK) ||
1853             old_end - old_addr < HPAGE_PMD_SIZE)
1854                 return false;
1855 
1856         /*
1857          * The destination pmd shouldn't be established, free_pgtables()
1858          * should have release it.
1859          */
1860         if (WARN_ON(!pmd_none(*new_pmd))) {
1861                 VM_BUG_ON(pmd_trans_huge(*new_pmd));
1862                 return false;
1863         }
1864 
1865         /*
1866          * We don't have to worry about the ordering of src and dst
1867          * ptlocks because exclusive mmap_sem prevents deadlock.
1868          */
1869         old_ptl = __pmd_trans_huge_lock(old_pmd, vma);
1870         if (old_ptl) {
1871                 new_ptl = pmd_lockptr(mm, new_pmd);
1872                 if (new_ptl != old_ptl)
1873                         spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
1874                 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);
1875                 if (pmd_present(pmd))
1876                         force_flush = true;
1877                 VM_BUG_ON(!pmd_none(*new_pmd));
1878 
1879                 if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) {
1880                         pgtable_t pgtable;
1881                         pgtable = pgtable_trans_huge_withdraw(mm, old_pmd);
1882                         pgtable_trans_huge_deposit(mm, new_pmd, pgtable);
1883                 }
1884                 pmd = move_soft_dirty_pmd(pmd);
1885                 set_pmd_at(mm, new_addr, new_pmd, pmd);
1886                 if (force_flush)
1887                         flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
1888                 if (new_ptl != old_ptl)
1889                         spin_unlock(new_ptl);
1890                 spin_unlock(old_ptl);
1891                 return true;
1892         }
1893         return false;
1894 }
1895 
1896 /*
1897  * Returns
1898  *  - 0 if PMD could not be locked
1899  *  - 1 if PMD was locked but protections unchange and TLB flush unnecessary
1900  *  - HPAGE_PMD_NR is protections changed and TLB flush necessary
1901  */
1902 int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
1903                 unsigned long addr, pgprot_t newprot, int prot_numa)
1904 {
1905         struct mm_struct *mm = vma->vm_mm;
1906         spinlock_t *ptl;
1907         pmd_t entry;
1908         bool preserve_write;
1909         int ret;
1910 
1911         ptl = __pmd_trans_huge_lock(pmd, vma);
1912         if (!ptl)
1913                 return 0;
1914 
1915         preserve_write = prot_numa && pmd_write(*pmd);
1916         ret = 1;
1917 
1918 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1919         if (is_swap_pmd(*pmd)) {
1920                 swp_entry_t entry = pmd_to_swp_entry(*pmd);
1921 
1922                 VM_BUG_ON(!is_pmd_migration_entry(*pmd));
1923                 if (is_write_migration_entry(entry)) {
1924                         pmd_t newpmd;
1925                         /*
1926                          * A protection check is difficult so
1927                          * just be safe and disable write
1928                          */
1929                         make_migration_entry_read(&entry);
1930                         newpmd = swp_entry_to_pmd(entry);
1931                         if (pmd_swp_soft_dirty(*pmd))
1932                                 newpmd = pmd_swp_mksoft_dirty(newpmd);
1933                         set_pmd_at(mm, addr, pmd, newpmd);
1934                 }
1935                 goto unlock;
1936         }
1937 #endif
1938 
1939         /*
1940          * Avoid trapping faults against the zero page. The read-only
1941          * data is likely to be read-cached on the local CPU and
1942          * local/remote hits to the zero page are not interesting.
1943          */
1944         if (prot_numa && is_huge_zero_pmd(*pmd))
1945                 goto unlock;
1946 
1947         if (prot_numa && pmd_protnone(*pmd))
1948                 goto unlock;
1949 
1950         /*
1951          * In case prot_numa, we are under down_read(mmap_sem). It's critical
1952          * to not clear pmd intermittently to avoid race with MADV_DONTNEED
1953          * which is also under down_read(mmap_sem):
1954          *
1955          *      CPU0:                           CPU1:
1956          *                              change_huge_pmd(prot_numa=1)
1957          *                               pmdp_huge_get_and_clear_notify()
1958          * madvise_dontneed()
1959          *  zap_pmd_range()
1960          *   pmd_trans_huge(*pmd) == 0 (without ptl)
1961          *   // skip the pmd
1962          *                               set_pmd_at();
1963          *                               // pmd is re-established
1964          *
1965          * The race makes MADV_DONTNEED miss the huge pmd and don't clear it
1966          * which may break userspace.
1967          *
1968          * pmdp_invalidate() is required to make sure we don't miss
1969          * dirty/young flags set by hardware.
1970          */
1971         entry = pmdp_invalidate(vma, addr, pmd);
1972 
1973         entry = pmd_modify(entry, newprot);
1974         if (preserve_write)
1975                 entry = pmd_mk_savedwrite(entry);
1976         ret = HPAGE_PMD_NR;
1977         set_pmd_at(mm, addr, pmd, entry);
1978         BUG_ON(vma_is_anonymous(vma) && !preserve_write && pmd_write(entry));
1979 unlock:
1980         spin_unlock(ptl);
1981         return ret;
1982 }
1983 
1984 /*
1985  * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise.
1986  *
1987  * Note that if it returns page table lock pointer, this routine returns without
1988  * unlocking page table lock. So callers must unlock it.
1989  */
1990 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma)
1991 {
1992         spinlock_t *ptl;
1993         ptl = pmd_lock(vma->vm_mm, pmd);
1994         if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) ||
1995                         pmd_devmap(*pmd)))
1996                 return ptl;
1997         spin_unlock(ptl);
1998         return NULL;
1999 }
2000 
2001 /*
2002  * Returns true if a given pud maps a thp, false otherwise.
2003  *
2004  * Note that if it returns true, this routine returns without unlocking page
2005  * table lock. So callers must unlock it.
2006  */
2007 spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma)
2008 {
2009         spinlock_t *ptl;
2010 
2011         ptl = pud_lock(vma->vm_mm, pud);
2012         if (likely(pud_trans_huge(*pud) || pud_devmap(*pud)))
2013                 return ptl;
2014         spin_unlock(ptl);
2015         return NULL;
2016 }
2017 
2018 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
2019 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
2020                  pud_t *pud, unsigned long addr)
2021 {
2022         pud_t orig_pud;
2023         spinlock_t *ptl;
2024 
2025         ptl = __pud_trans_huge_lock(pud, vma);
2026         if (!ptl)
2027                 return 0;
2028         /*
2029          * For architectures like ppc64 we look at deposited pgtable
2030          * when calling pudp_huge_get_and_clear. So do the
2031          * pgtable_trans_huge_withdraw after finishing pudp related
2032          * operations.
2033          */
2034         orig_pud = pudp_huge_get_and_clear_full(tlb->mm, addr, pud,
2035                         tlb->fullmm);
2036         tlb_remove_pud_tlb_entry(tlb, pud, addr);
2037         if (vma_is_dax(vma)) {
2038                 spin_unlock(ptl);
2039                 /* No zero page support yet */
2040         } else {
2041                 /* No support for anonymous PUD pages yet */
2042                 BUG();
2043         }
2044         return 1;
2045 }
2046 
2047 static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud,
2048                 unsigned long haddr)
2049 {
2050         VM_BUG_ON(haddr & ~HPAGE_PUD_MASK);
2051         VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
2052         VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma);
2053         VM_BUG_ON(!pud_trans_huge(*pud) && !pud_devmap(*pud));
2054 
2055         count_vm_event(THP_SPLIT_PUD);
2056 
2057         pudp_huge_clear_flush_notify(vma, haddr, pud);
2058 }
2059 
2060 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
2061                 unsigned long address)
2062 {
2063         spinlock_t *ptl;
2064         struct mmu_notifier_range range;
2065 
2066         mmu_notifier_range_init(&range, vma->vm_mm, address & HPAGE_PUD_MASK,
2067                                 (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE);
2068         mmu_notifier_invalidate_range_start(&range);
2069         ptl = pud_lock(vma->vm_mm, pud);
2070         if (unlikely(!pud_trans_huge(*pud) && !pud_devmap(*pud)))
2071                 goto out;
2072         __split_huge_pud_locked(vma, pud, range.start);
2073 
2074 out:
2075         spin_unlock(ptl);
2076         /*
2077          * No need to double call mmu_notifier->invalidate_range() callback as
2078          * the above pudp_huge_clear_flush_notify() did already call it.
2079          */
2080         mmu_notifier_invalidate_range_only_end(&range);
2081 }
2082 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
2083 
2084 static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
2085                 unsigned long haddr, pmd_t *pmd)
2086 {
2087         struct mm_struct *mm = vma->vm_mm;
2088         pgtable_t pgtable;
2089         pmd_t _pmd;
2090         int i;
2091 
2092         /*
2093          * Leave pmd empty until pte is filled note that it is fine to delay
2094          * notification until mmu_notifier_invalidate_range_end() as we are
2095          * replacing a zero pmd write protected page with a zero pte write
2096          * protected page.
2097          *
2098          * See Documentation/vm/mmu_notifier.rst
2099          */
2100         pmdp_huge_clear_flush(vma, haddr, pmd);
2101 
2102         pgtable = pgtable_trans_huge_withdraw(mm, pmd);
2103         pmd_populate(mm, &_pmd, pgtable);
2104 
2105         for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
2106                 pte_t *pte, entry;
2107                 entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot);
2108                 entry = pte_mkspecial(entry);
2109                 pte = pte_offset_map(&_pmd, haddr);
2110                 VM_BUG_ON(!pte_none(*pte));
2111                 set_pte_at(mm, haddr, pte, entry);
2112                 pte_unmap(pte);
2113         }
2114         smp_wmb(); /* make pte visible before pmd */
2115         pmd_populate(mm, pmd, pgtable);
2116 }
2117 
2118 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
2119                 unsigned long haddr, bool freeze)
2120 {
2121         struct mm_struct *mm = vma->vm_mm;
2122         struct page *page;
2123         pgtable_t pgtable;
2124         pmd_t old_pmd, _pmd;
2125         bool young, write, soft_dirty, pmd_migration = false;
2126         unsigned long addr;
2127         int i;
2128 
2129         VM_BUG_ON(haddr & ~HPAGE_PMD_MASK);
2130         VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
2131         VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma);
2132         VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd)
2133                                 && !pmd_devmap(*pmd));
2134 
2135         count_vm_event(THP_SPLIT_PMD);
2136 
2137         if (!vma_is_anonymous(vma)) {
2138                 _pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);
2139                 /*
2140                  * We are going to unmap this huge page. So
2141                  * just go ahead and zap it
2142                  */
2143                 if (arch_needs_pgtable_deposit())
2144                         zap_deposited_table(mm, pmd);
2145                 if (vma_is_dax(vma))
2146                         return;
2147                 page = pmd_page(_pmd);
2148                 if (!PageDirty(page) && pmd_dirty(_pmd))
2149                         set_page_dirty(page);
2150                 if (!PageReferenced(page) && pmd_young(_pmd))
2151                         SetPageReferenced(page);
2152                 page_remove_rmap(page, true);
2153                 put_page(page);
2154                 add_mm_counter(mm, mm_counter_file(page), -HPAGE_PMD_NR);
2155                 return;
2156         } else if (is_huge_zero_pmd(*pmd)) {
2157                 /*
2158                  * FIXME: Do we want to invalidate secondary mmu by calling
2159                  * mmu_notifier_invalidate_range() see comments below inside
2160                  * __split_huge_pmd() ?
2161                  *
2162                  * We are going from a zero huge page write protected to zero
2163                  * small page also write protected so it does not seems useful
2164                  * to invalidate secondary mmu at this time.
2165                  */
2166                 return __split_huge_zero_page_pmd(vma, haddr, pmd);
2167         }
2168 
2169         /*
2170          * Up to this point the pmd is present and huge and userland has the
2171          * whole access to the hugepage during the split (which happens in
2172          * place). If we overwrite the pmd with the not-huge version pointing
2173          * to the pte here (which of course we could if all CPUs were bug
2174          * free), userland could trigger a small page size TLB miss on the
2175          * small sized TLB while the hugepage TLB entry is still established in
2176          * the huge TLB. Some CPU doesn't like that.
2177          * See http://support.amd.com/us/Processor_TechDocs/41322.pdf, Erratum
2178          * 383 on page 93. Intel should be safe but is also warns that it's
2179          * only safe if the permission and cache attributes of the two entries
2180          * loaded in the two TLB is identical (which should be the case here).
2181          * But it is generally safer to never allow small and huge TLB entries
2182          * for the same virtual address to be loaded simultaneously. So instead
2183          * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the
2184          * current pmd notpresent (atomically because here the pmd_trans_huge
2185          * must remain set at all times on the pmd until the split is complete
2186          * for this pmd), then we flush the SMP TLB and finally we write the
2187          * non-huge version of the pmd entry with pmd_populate.
2188          */
2189         old_pmd = pmdp_invalidate(vma, haddr, pmd);
2190 
2191         pmd_migration = is_pmd_migration_entry(old_pmd);
2192         if (unlikely(pmd_migration)) {
2193                 swp_entry_t entry;
2194 
2195                 entry = pmd_to_swp_entry(old_pmd);
2196                 page = pfn_to_page(swp_offset(entry));
2197                 write = is_write_migration_entry(entry);
2198                 young = false;
2199                 soft_dirty = pmd_swp_soft_dirty(old_pmd);
2200         } else {
2201                 page = pmd_page(old_pmd);
2202                 if (pmd_dirty(old_pmd))
2203                         SetPageDirty(page);
2204                 write = pmd_write(old_pmd);
2205                 young = pmd_young(old_pmd);
2206                 soft_dirty = pmd_soft_dirty(old_pmd);
2207         }
2208         VM_BUG_ON_PAGE(!page_count(page), page);
2209         page_ref_add(page, HPAGE_PMD_NR - 1);
2210 
2211         /*
2212          * Withdraw the table only after we mark the pmd entry invalid.
2213          * This's critical for some architectures (Power).
2214          */
2215         pgtable = pgtable_trans_huge_withdraw(mm, pmd);
2216         pmd_populate(mm, &_pmd, pgtable);
2217 
2218         for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) {
2219                 pte_t entry, *pte;
2220                 /*
2221                  * Note that NUMA hinting access restrictions are not
2222                  * transferred to avoid any possibility of altering
2223                  * permissions across VMAs.
2224                  */
2225                 if (freeze || pmd_migration) {
2226                         swp_entry_t swp_entry;
2227                         swp_entry = make_migration_entry(page + i, write);
2228                         entry = swp_entry_to_pte(swp_entry);
2229                         if (soft_dirty)
2230                                 entry = pte_swp_mksoft_dirty(entry);
2231                 } else {
2232                         entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot));
2233                         entry = maybe_mkwrite(entry, vma);
2234                         if (!write)
2235                                 entry = pte_wrprotect(entry);
2236                         if (!young)
2237                                 entry = pte_mkold(entry);
2238                         if (soft_dirty)
2239                                 entry = pte_mksoft_dirty(entry);
2240                 }
2241                 pte = pte_offset_map(&_pmd, addr);
2242                 BUG_ON(!pte_none(*pte));
2243                 set_pte_at(mm, addr, pte, entry);
2244                 atomic_inc(&page[i]._mapcount);
2245                 pte_unmap(pte);
2246         }
2247 
2248         /*
2249          * Set PG_double_map before dropping compound_mapcount to avoid
2250          * false-negative page_mapped().
2251          */
2252         if (compound_mapcount(page) > 1 && !TestSetPageDoubleMap(page)) {
2253                 for (i = 0; i < HPAGE_PMD_NR; i++)
2254                         atomic_inc(&page[i]._mapcount);
2255         }
2256 
2257         if (atomic_add_negative(-1, compound_mapcount_ptr(page))) {
2258                 /* Last compound_mapcount is gone. */
2259                 __dec_node_page_state(page, NR_ANON_THPS);
2260                 if (TestClearPageDoubleMap(page)) {
2261                         /* No need in mapcount reference anymore */
2262                         for (i = 0; i < HPAGE_PMD_NR; i++)
2263                                 atomic_dec(&page[i]._mapcount);
2264                 }
2265         }
2266 
2267         smp_wmb(); /* make pte visible before pmd */
2268         pmd_populate(mm, pmd, pgtable);
2269 
2270         if (freeze) {
2271                 for (i = 0; i < HPAGE_PMD_NR; i++) {
2272                         page_remove_rmap(page + i, false);
2273                         put_page(page + i);
2274                 }
2275         }
2276 }
2277 
2278 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
2279                 unsigned long address, bool freeze, struct page *page)
2280 {
2281         spinlock_t *ptl;
2282         struct mmu_notifier_range range;
2283 
2284         mmu_notifier_range_init(&range, vma->vm_mm, address & HPAGE_PMD_MASK,
2285                                 (address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE);
2286         mmu_notifier_invalidate_range_start(&range);
2287         ptl = pmd_lock(vma->vm_mm, pmd);
2288 
2289         /*
2290          * If caller asks to setup a migration entries, we need a page to check
2291          * pmd against. Otherwise we can end up replacing wrong page.
2292          */
2293         VM_BUG_ON(freeze && !page);
2294         if (page && page != pmd_page(*pmd))
2295                 goto out;
2296 
2297         if (pmd_trans_huge(*pmd)) {
2298                 page = pmd_page(*pmd);
2299                 if (PageMlocked(page))
2300                         clear_page_mlock(page);
2301         } else if (!(pmd_devmap(*pmd) || is_pmd_migration_entry(*pmd)))
2302                 goto out;
2303         __split_huge_pmd_locked(vma, pmd, range.start, freeze);
2304 out:
2305         spin_unlock(ptl);
2306         /*
2307          * No need to double call mmu_notifier->invalidate_range() callback.
2308          * They are 3 cases to consider inside __split_huge_pmd_locked():
2309          *  1) pmdp_huge_clear_flush_notify() call invalidate_range() obvious
2310          *  2) __split_huge_zero_page_pmd() read only zero page and any write
2311          *    fault will trigger a flush_notify before pointing to a new page
2312          *    (it is fine if the secondary mmu keeps pointing to the old zero
2313          *    page in the meantime)
2314          *  3) Split a huge pmd into pte pointing to the same page. No need
2315          *     to invalidate secondary tlb entry they are all still valid.
2316          *     any further changes to individual pte will notify. So no need
2317          *     to call mmu_notifier->invalidate_range()
2318          */
2319         mmu_notifier_invalidate_range_only_end(&range);
2320 }
2321 
2322 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
2323                 bool freeze, struct page *page)
2324 {
2325         pgd_t *pgd;
2326         p4d_t *p4d;
2327         pud_t *pud;
2328         pmd_t *pmd;
2329 
2330         pgd = pgd_offset(vma->vm_mm, address);
2331         if (!pgd_present(*pgd))
2332                 return;
2333 
2334         p4d = p4d_offset(pgd, address);
2335         if (!p4d_present(*p4d))
2336                 return;
2337 
2338         pud = pud_offset(p4d, address);
2339         if (!pud_present(*pud))
2340                 return;
2341 
2342         pmd = pmd_offset(pud, address);
2343 
2344         __split_huge_pmd(vma, pmd, address, freeze, page);
2345 }
2346 
2347 void vma_adjust_trans_huge(struct vm_area_struct *vma,
2348                              unsigned long start,
2349                              unsigned long end,
2350                              long adjust_next)
2351 {
2352         /*
2353          * If the new start address isn't hpage aligned and it could
2354          * previously contain an hugepage: check if we need to split
2355          * an huge pmd.
2356          */
2357         if (start & ~HPAGE_PMD_MASK &&
2358             (start & HPAGE_PMD_MASK) >= vma->vm_start &&
2359             (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
2360                 split_huge_pmd_address(vma, start, false, NULL);
2361 
2362         /*
2363          * If the new end address isn't hpage aligned and it could
2364          * previously contain an hugepage: check if we need to split
2365          * an huge pmd.
2366          */
2367         if (end & ~HPAGE_PMD_MASK &&
2368             (end & HPAGE_PMD_MASK) >= vma->vm_start &&
2369             (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
2370                 split_huge_pmd_address(vma, end, false, NULL);
2371 
2372         /*
2373          * If we're also updating the vma->vm_next->vm_start, if the new
2374          * vm_next->vm_start isn't page aligned and it could previously
2375          * contain an hugepage: check if we need to split an huge pmd.
2376          */
2377         if (adjust_next > 0) {
2378                 struct vm_area_struct *next = vma->vm_next;
2379                 unsigned long nstart = next->vm_start;
2380                 nstart += adjust_next << PAGE_SHIFT;
2381                 if (nstart & ~HPAGE_PMD_MASK &&
2382                     (nstart & HPAGE_PMD_MASK) >= next->vm_start &&
2383                     (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end)
2384                         split_huge_pmd_address(next, nstart, false, NULL);
2385         }
2386 }
2387 
2388 static void unmap_page(struct page *page)
2389 {
2390         enum ttu_flags ttu_flags = TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS |
2391                 TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD;
2392         bool unmap_success;
2393 
2394         VM_BUG_ON_PAGE(!PageHead(page), page);
2395 
2396         if (PageAnon(page))
2397                 ttu_flags |= TTU_SPLIT_FREEZE;
2398 
2399         unmap_success = try_to_unmap(page, ttu_flags);
2400         VM_BUG_ON_PAGE(!unmap_success, page);
2401 }
2402 
2403 static void remap_page(struct page *page)
2404 {
2405         int i;
2406         if (PageTransHuge(page)) {
2407                 remove_migration_ptes(page, page, true);
2408         } else {
2409                 for (i = 0; i < HPAGE_PMD_NR; i++)
2410                         remove_migration_ptes(page + i, page + i, true);
2411         }
2412 }
2413 
2414 static void __split_huge_page_tail(struct page *head, int tail,
2415                 struct lruvec *lruvec, struct list_head *list)
2416 {
2417         struct page *page_tail = head + tail;
2418 
2419         VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail);
2420 
2421         /*
2422          * Clone page flags before unfreezing refcount.
2423          *
2424          * After successful get_page_unless_zero() might follow flags change,
2425          * for exmaple lock_page() which set PG_waiters.
2426          */
2427         page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
2428         page_tail->flags |= (head->flags &
2429                         ((1L << PG_referenced) |
2430                          (1L << PG_swapbacked) |
2431                          (1L << PG_swapcache) |
2432                          (1L << PG_mlocked) |
2433                          (1L << PG_uptodate) |
2434                          (1L << PG_active) |
2435                          (1L << PG_workingset) |
2436                          (1L << PG_locked) |
2437                          (1L << PG_unevictable) |
2438                          (1L << PG_dirty)));
2439 
2440         /* ->mapping in first tail page is compound_mapcount */
2441         VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING,
2442                         page_tail);
2443         page_tail->mapping = head->mapping;
2444         page_tail->index = head->index + tail;
2445 
2446         /* Page flags must be visible before we make the page non-compound. */
2447         smp_wmb();
2448 
2449         /*
2450          * Clear PageTail before unfreezing page refcount.
2451          *
2452          * After successful get_page_unless_zero() might follow put_page()
2453          * which needs correct compound_head().
2454          */
2455         clear_compound_head(page_tail);
2456 
2457         /* Finally unfreeze refcount. Additional reference from page cache. */
2458         page_ref_unfreeze(page_tail, 1 + (!PageAnon(head) ||
2459                                           PageSwapCache(head)));
2460 
2461         if (page_is_young(head))
2462                 set_page_young(page_tail);
2463         if (page_is_idle(head))
2464                 set_page_idle(page_tail);
2465 
2466         page_cpupid_xchg_last(page_tail, page_cpupid_last(head));
2467 
2468         /*
2469          * always add to the tail because some iterators expect new
2470          * pages to show after the currently processed elements - e.g.
2471          * migrate_pages
2472          */
2473         lru_add_page_tail(head, page_tail, lruvec, list);
2474 }
2475 
2476 static void __split_huge_page(struct page *page, struct list_head *list,
2477                 pgoff_t end, unsigned long flags)
2478 {
2479         struct page *head = compound_head(page);
2480         struct zone *zone = page_zone(head);
2481         struct lruvec *lruvec;
2482         int i;
2483 
2484         lruvec = mem_cgroup_page_lruvec(head, zone->zone_pgdat);
2485 
2486         /* complete memcg works before add pages to LRU */
2487         mem_cgroup_split_huge_fixup(head);
2488 
2489         for (i = HPAGE_PMD_NR - 1; i >= 1; i--) {
2490                 __split_huge_page_tail(head, i, lruvec, list);
2491                 /* Some pages can be beyond i_size: drop them from page cache */
2492                 if (head[i].index >= end) {
2493                         ClearPageDirty(head + i);
2494                         __delete_from_page_cache(head + i, NULL);
2495                         if (IS_ENABLED(CONFIG_SHMEM) && PageSwapBacked(head))
2496                                 shmem_uncharge(head->mapping->host, 1);
2497                         put_page(head + i);
2498                 }
2499         }
2500 
2501         ClearPageCompound(head);
2502         /* See comment in __split_huge_page_tail() */
2503         if (PageAnon(head)) {
2504                 /* Additional pin to swap cache */
2505                 if (PageSwapCache(head))
2506                         page_ref_add(head, 2);
2507                 else
2508                         page_ref_inc(head);
2509         } else {
2510                 /* Additional pin to page cache */
2511                 page_ref_add(head, 2);
2512                 xa_unlock(&head->mapping->i_pages);
2513         }
2514 
2515         spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags);
2516 
2517         remap_page(head);
2518 
2519         for (i = 0; i < HPAGE_PMD_NR; i++) {
2520                 struct page *subpage = head + i;
2521                 if (subpage == page)
2522                         continue;
2523                 unlock_page(subpage);
2524 
2525                 /*
2526                  * Subpages may be freed if there wasn't any mapping
2527                  * like if add_to_swap() is running on a lru page that
2528                  * had its mapping zapped. And freeing these pages
2529                  * requires taking the lru_lock so we do the put_page
2530                  * of the tail pages after the split is complete.
2531                  */
2532                 put_page(subpage);
2533         }
2534 }
2535 
2536 int total_mapcount(struct page *page)
2537 {
2538         int i, compound, ret;
2539 
2540         VM_BUG_ON_PAGE(PageTail(page), page);
2541 
2542         if (likely(!PageCompound(page)))
2543                 return atomic_read(&page->_mapcount) + 1;
2544 
2545         compound = compound_mapcount(page);
2546         if (PageHuge(page))
2547                 return compound;
2548         ret = compound;
2549         for (i = 0; i < HPAGE_PMD_NR; i++)
2550                 ret += atomic_read(&page[i]._mapcount) + 1;
2551         /* File pages has compound_mapcount included in _mapcount */
2552         if (!PageAnon(page))
2553                 return ret - compound * HPAGE_PMD_NR;
2554         if (PageDoubleMap(page))
2555                 ret -= HPAGE_PMD_NR;
2556         return ret;
2557 }
2558 
2559 /*
2560  * This calculates accurately how many mappings a transparent hugepage
2561  * has (unlike page_mapcount() which isn't fully accurate). This full
2562  * accuracy is primarily needed to know if copy-on-write faults can
2563  * reuse the page and change the mapping to read-write instead of
2564  * copying them. At the same time this returns the total_mapcount too.
2565  *
2566  * The function returns the highest mapcount any one of the subpages
2567  * has. If the return value is one, even if different processes are
2568  * mapping different subpages of the transparent hugepage, they can
2569  * all reuse it, because each process is reusing a different subpage.
2570  *
2571  * The total_mapcount is instead counting all virtual mappings of the
2572  * subpages. If the total_mapcount is equal to "one", it tells the
2573  * caller all mappings belong to the same "mm" and in turn the
2574  * anon_vma of the transparent hugepage can become the vma->anon_vma
2575  * local one as no other process may be mapping any of the subpages.
2576  *
2577  * It would be more accurate to replace page_mapcount() with
2578  * page_trans_huge_mapcount(), however we only use
2579  * page_trans_huge_mapcount() in the copy-on-write faults where we
2580  * need full accuracy to avoid breaking page pinning, because
2581  * page_trans_huge_mapcount() is slower than page_mapcount().
2582  */
2583 int page_trans_huge_mapcount(struct page *page, int *total_mapcount)
2584 {
2585         int i, ret, _total_mapcount, mapcount;
2586 
2587         /* hugetlbfs shouldn't call it */
2588         VM_BUG_ON_PAGE(PageHuge(page), page);
2589 
2590         if (likely(!PageTransCompound(page))) {
2591                 mapcount = atomic_read(&page->_mapcount) + 1;
2592                 if (total_mapcount)
2593                         *total_mapcount = mapcount;
2594                 return mapcount;
2595         }
2596 
2597         page = compound_head(page);
2598 
2599         _total_mapcount = ret = 0;
2600         for (i = 0; i < HPAGE_PMD_NR; i++) {
2601                 mapcount = atomic_read(&page[i]._mapcount) + 1;
2602                 ret = max(ret, mapcount);
2603                 _total_mapcount += mapcount;
2604         }
2605         if (PageDoubleMap(page)) {
2606                 ret -= 1;
2607                 _total_mapcount -= HPAGE_PMD_NR;
2608         }
2609         mapcount = compound_mapcount(page);
2610         ret += mapcount;
2611         _total_mapcount += mapcount;
2612         if (total_mapcount)
2613                 *total_mapcount = _total_mapcount;
2614         return ret;
2615 }
2616 
2617 /* Racy check whether the huge page can be split */
2618 bool can_split_huge_page(struct page *page, int *pextra_pins)
2619 {
2620         int extra_pins;
2621 
2622         /* Additional pins from page cache */
2623         if (PageAnon(page))
2624                 extra_pins = PageSwapCache(page) ? HPAGE_PMD_NR : 0;
2625         else
2626                 extra_pins = HPAGE_PMD_NR;
2627         if (pextra_pins)
2628                 *pextra_pins = extra_pins;
2629         return total_mapcount(page) == page_count(page) - extra_pins - 1;
2630 }
2631 
2632 /*
2633  * This function splits huge page into normal pages. @page can point to any
2634  * subpage of huge page to split. Split doesn't change the position of @page.
2635  *
2636  * Only caller must hold pin on the @page, otherwise split fails with -EBUSY.
2637  * The huge page must be locked.
2638  *
2639  * If @list is null, tail pages will be added to LRU list, otherwise, to @list.
2640  *
2641  * Both head page and tail pages will inherit mapping, flags, and so on from
2642  * the hugepage.
2643  *
2644  * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if
2645  * they are not mapped.
2646  *
2647  * Returns 0 if the hugepage is split successfully.
2648  * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under
2649  * us.
2650  */
2651 int split_huge_page_to_list(struct page *page, struct list_head *list)
2652 {
2653         struct page *head = compound_head(page);
2654         struct pglist_data *pgdata = NODE_DATA(page_to_nid(head));
2655         struct anon_vma *anon_vma = NULL;
2656         struct address_space *mapping = NULL;
2657         int count, mapcount, extra_pins, ret;
2658         bool mlocked;
2659         unsigned long flags;
2660         pgoff_t end;
2661 
2662         VM_BUG_ON_PAGE(is_huge_zero_page(page), page);
2663         VM_BUG_ON_PAGE(!PageLocked(page), page);
2664         VM_BUG_ON_PAGE(!PageCompound(page), page);
2665 
2666         if (PageWriteback(page))
2667                 return -EBUSY;
2668 
2669         if (PageAnon(head)) {
2670                 /*
2671                  * The caller does not necessarily hold an mmap_sem that would
2672                  * prevent the anon_vma disappearing so we first we take a
2673                  * reference to it and then lock the anon_vma for write. This
2674                  * is similar to page_lock_anon_vma_read except the write lock
2675                  * is taken to serialise against parallel split or collapse
2676                  * operations.
2677                  */
2678                 anon_vma = page_get_anon_vma(head);
2679                 if (!anon_vma) {
2680                         ret = -EBUSY;
2681                         goto out;
2682                 }
2683                 end = -1;
2684                 mapping = NULL;
2685                 anon_vma_lock_write(anon_vma);
2686         } else {
2687                 mapping = head->mapping;
2688 
2689                 /* Truncated ? */
2690                 if (!mapping) {
2691                         ret = -EBUSY;
2692                         goto out;
2693                 }
2694 
2695                 anon_vma = NULL;
2696                 i_mmap_lock_read(mapping);
2697 
2698                 /*
2699                  *__split_huge_page() may need to trim off pages beyond EOF:
2700                  * but on 32-bit, i_size_read() takes an irq-unsafe seqlock,
2701                  * which cannot be nested inside the page tree lock. So note
2702                  * end now: i_size itself may be changed at any moment, but
2703                  * head page lock is good enough to serialize the trimming.
2704                  */
2705                 end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
2706         }
2707 
2708         /*
2709          * Racy check if we can split the page, before unmap_page() will
2710          * split PMDs
2711          */
2712         if (!can_split_huge_page(head, &extra_pins)) {
2713                 ret = -EBUSY;
2714                 goto out_unlock;
2715         }
2716 
2717         mlocked = PageMlocked(page);
2718         unmap_page(head);
2719         VM_BUG_ON_PAGE(compound_mapcount(head), head);
2720 
2721         /* Make sure the page is not on per-CPU pagevec as it takes pin */
2722         if (mlocked)
2723                 lru_add_drain();
2724 
2725         /* prevent PageLRU to go away from under us, and freeze lru stats */
2726         spin_lock_irqsave(zone_lru_lock(page_zone(head)), flags);
2727 
2728         if (mapping) {
2729                 XA_STATE(xas, &mapping->i_pages, page_index(head));
2730 
2731                 /*
2732                  * Check if the head page is present in page cache.
2733                  * We assume all tail are present too, if head is there.
2734                  */
2735                 xa_lock(&mapping->i_pages);
2736                 if (xas_load(&xas) != head)
2737                         goto fail;
2738         }
2739 
2740         /* Prevent deferred_split_scan() touching ->_refcount */
2741         spin_lock(&pgdata->split_queue_lock);
2742         count = page_count(head);
2743         mapcount = total_mapcount(head);
2744         if (!mapcount && page_ref_freeze(head, 1 + extra_pins)) {
2745                 if (!list_empty(page_deferred_list(head))) {
2746                         pgdata->split_queue_len--;
2747                         list_del(page_deferred_list(head));
2748                 }
2749                 if (mapping)
2750                         __dec_node_page_state(page, NR_SHMEM_THPS);
2751                 spin_unlock(&pgdata->split_queue_lock);
2752                 __split_huge_page(page, list, end, flags);
2753                 if (PageSwapCache(head)) {
2754                         swp_entry_t entry = { .val = page_private(head) };
2755 
2756                         ret = split_swap_cluster(entry);
2757                 } else
2758                         ret = 0;
2759         } else {
2760                 if (IS_ENABLED(CONFIG_DEBUG_VM) && mapcount) {
2761                         pr_alert("total_mapcount: %u, page_count(): %u\n",
2762                                         mapcount, count);
2763                         if (PageTail(page))
2764                                 dump_page(head, NULL);
2765                         dump_page(page, "total_mapcount(head) > 0");
2766                         BUG();
2767                 }
2768                 spin_unlock(&pgdata->split_queue_lock);
2769 fail:           if (mapping)
2770                         xa_unlock(&mapping->i_pages);
2771                 spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags);
2772                 remap_page(head);
2773                 ret = -EBUSY;
2774         }
2775 
2776 out_unlock:
2777         if (anon_vma) {
2778                 anon_vma_unlock_write(anon_vma);
2779                 put_anon_vma(anon_vma);
2780         }
2781         if (mapping)
2782                 i_mmap_unlock_read(mapping);
2783 out:
2784         count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED);
2785         return ret;
2786 }
2787 
2788 void free_transhuge_page(struct page *page)
2789 {
2790         struct pglist_data *pgdata = NODE_DATA(page_to_nid(page));
2791         unsigned long flags;
2792 
2793         spin_lock_irqsave(&pgdata->split_queue_lock, flags);
2794         if (!list_empty(page_deferred_list(page))) {
2795                 pgdata->split_queue_len--;
2796                 list_del(page_deferred_list(page));
2797         }
2798         spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2799         free_compound_page(page);
2800 }
2801 
2802 void deferred_split_huge_page(struct page *page)
2803 {
2804         struct pglist_data *pgdata = NODE_DATA(page_to_nid(page));
2805         unsigned long flags;
2806 
2807         VM_BUG_ON_PAGE(!PageTransHuge(page), page);
2808 
2809         spin_lock_irqsave(&pgdata->split_queue_lock, flags);
2810         if (list_empty(page_deferred_list(page))) {
2811                 count_vm_event(THP_DEFERRED_SPLIT_PAGE);
2812                 list_add_tail(page_deferred_list(page), &pgdata->split_queue);
2813                 pgdata->split_queue_len++;
2814         }
2815         spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2816 }
2817 
2818 static unsigned long deferred_split_count(struct shrinker *shrink,
2819                 struct shrink_control *sc)
2820 {
2821         struct pglist_data *pgdata = NODE_DATA(sc->nid);
2822         return READ_ONCE(pgdata->split_queue_len);
2823 }
2824 
2825 static unsigned long deferred_split_scan(struct shrinker *shrink,
2826                 struct shrink_control *sc)
2827 {
2828         struct pglist_data *pgdata = NODE_DATA(sc->nid);
2829         unsigned long flags;
2830         LIST_HEAD(list), *pos, *next;
2831         struct page *page;
2832         int split = 0;
2833 
2834         spin_lock_irqsave(&pgdata->split_queue_lock, flags);
2835         /* Take pin on all head pages to avoid freeing them under us */
2836         list_for_each_safe(pos, next, &pgdata->split_queue) {
2837                 page = list_entry((void *)pos, struct page, mapping);
2838                 page = compound_head(page);
2839                 if (get_page_unless_zero(page)) {
2840                         list_move(page_deferred_list(page), &list);
2841                 } else {
2842                         /* We lost race with put_compound_page() */
2843                         list_del_init(page_deferred_list(page));
2844                         pgdata->split_queue_len--;
2845                 }
2846                 if (!--sc->nr_to_scan)
2847                         break;
2848         }
2849         spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2850 
2851         list_for_each_safe(pos, next, &list) {
2852                 page = list_entry((void *)pos, struct page, mapping);
2853                 if (!trylock_page(page))
2854                         goto next;
2855                 /* split_huge_page() removes page from list on success */
2856                 if (!split_huge_page(page))
2857                         split++;
2858                 unlock_page(page);
2859 next:
2860                 put_page(page);
2861         }
2862 
2863         spin_lock_irqsave(&pgdata->split_queue_lock, flags);
2864         list_splice_tail(&list, &pgdata->split_queue);
2865         spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2866 
2867         /*
2868          * Stop shrinker if we didn't split any page, but the queue is empty.
2869          * This can happen if pages were freed under us.
2870          */
2871         if (!split && list_empty(&pgdata->split_queue))
2872                 return SHRINK_STOP;
2873         return split;
2874 }
2875 
2876 static struct shrinker deferred_split_shrinker = {
2877         .count_objects = deferred_split_count,
2878         .scan_objects = deferred_split_scan,
2879         .seeks = DEFAULT_SEEKS,
2880         .flags = SHRINKER_NUMA_AWARE,
2881 };
2882 
2883 #ifdef CONFIG_DEBUG_FS
2884 static int split_huge_pages_set(void *data, u64 val)
2885 {
2886         struct zone *zone;
2887         struct page *page;
2888         unsigned long pfn, max_zone_pfn;
2889         unsigned long total = 0, split = 0;
2890 
2891         if (val != 1)
2892                 return -EINVAL;
2893 
2894         for_each_populated_zone(zone) {
2895                 max_zone_pfn = zone_end_pfn(zone);
2896                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {
2897                         if (!pfn_valid(pfn))
2898                                 continue;
2899 
2900                         page = pfn_to_page(pfn);
2901                         if (!get_page_unless_zero(page))
2902                                 continue;
2903 
2904                         if (zone != page_zone(page))
2905                                 goto next;
2906 
2907                         if (!PageHead(page) || PageHuge(page) || !PageLRU(page))
2908                                 goto next;
2909 
2910                         total++;
2911                         lock_page(page);
2912                         if (!split_huge_page(page))
2913                                 split++;
2914                         unlock_page(page);
2915 next:
2916                         put_page(page);
2917                 }
2918         }
2919 
2920         pr_info("%lu of %lu THP split\n", split, total);
2921 
2922         return 0;
2923 }
2924 DEFINE_SIMPLE_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set,
2925                 "%llu\n");
2926 
2927 static int __init split_huge_pages_debugfs(void)
2928 {
2929         void *ret;
2930 
2931         ret = debugfs_create_file("split_huge_pages", 0200, NULL, NULL,
2932                         &split_huge_pages_fops);
2933         if (!ret)
2934                 pr_warn("Failed to create split_huge_pages in debugfs");
2935         return 0;
2936 }
2937 late_initcall(split_huge_pages_debugfs);
2938 #endif
2939 
2940 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
2941 void set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
2942                 struct page *page)
2943 {
2944         struct vm_area_struct *vma = pvmw->vma;
2945         struct mm_struct *mm = vma->vm_mm;
2946         unsigned long address = pvmw->address;
2947         pmd_t pmdval;
2948         swp_entry_t entry;
2949         pmd_t pmdswp;
2950 
2951         if (!(pvmw->pmd && !pvmw->pte))
2952                 return;
2953 
2954         flush_cache_range(vma, address, address + HPAGE_PMD_SIZE);
2955         pmdval = *pvmw->pmd;
2956         pmdp_invalidate(vma, address, pvmw->pmd);
2957         if (pmd_dirty(pmdval))
2958                 set_page_dirty(page);
2959         entry = make_migration_entry(page, pmd_write(pmdval));
2960         pmdswp = swp_entry_to_pmd(entry);
2961         if (pmd_soft_dirty(pmdval))
2962                 pmdswp = pmd_swp_mksoft_dirty(pmdswp);
2963         set_pmd_at(mm, address, pvmw->pmd, pmdswp);
2964         page_remove_rmap(page, true);
2965         put_page(page);
2966 }
2967 
2968 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)
2969 {
2970         struct vm_area_struct *vma = pvmw->vma;
2971         struct mm_struct *mm = vma->vm_mm;
2972         unsigned long address = pvmw->address;
2973         unsigned long mmun_start = address & HPAGE_PMD_MASK;
2974         pmd_t pmde;
2975         swp_entry_t entry;
2976 
2977         if (!(pvmw->pmd && !pvmw->pte))
2978                 return;
2979 
2980         entry = pmd_to_swp_entry(*pvmw->pmd);
2981         get_page(new);
2982         pmde = pmd_mkold(mk_huge_pmd(new, vma->vm_page_prot));
2983         if (pmd_swp_soft_dirty(*pvmw->pmd))
2984                 pmde = pmd_mksoft_dirty(pmde);
2985         if (is_write_migration_entry(entry))
2986                 pmde = maybe_pmd_mkwrite(pmde, vma);
2987 
2988         flush_cache_range(vma, mmun_start, mmun_start + HPAGE_PMD_SIZE);
2989         if (PageAnon(new))
2990                 page_add_anon_rmap(new, vma, mmun_start, true);
2991         else
2992                 page_add_file_rmap(new, true);
2993         set_pmd_at(mm, mmun_start, pvmw->pmd, pmde);
2994         if ((vma->vm_flags & VM_LOCKED) && !PageDoubleMap(new))
2995                 mlock_vma_page(new);
2996         update_mmu_cache_pmd(vma, address, pvmw->pmd);
2997 }
2998 #endif
2999 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp