~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/gup.c

Version: ~ [ linux-5.16-rc3 ] ~ [ linux-5.15.5 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.82 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.162 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.218 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.256 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.291 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.293 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-only
  2 #include <linux/kernel.h>
  3 #include <linux/errno.h>
  4 #include <linux/err.h>
  5 #include <linux/spinlock.h>
  6 
  7 #include <linux/mm.h>
  8 #include <linux/memremap.h>
  9 #include <linux/pagemap.h>
 10 #include <linux/rmap.h>
 11 #include <linux/swap.h>
 12 #include <linux/swapops.h>
 13 
 14 #include <linux/sched/signal.h>
 15 #include <linux/rwsem.h>
 16 #include <linux/hugetlb.h>
 17 #include <linux/migrate.h>
 18 #include <linux/mm_inline.h>
 19 #include <linux/sched/mm.h>
 20 
 21 #include <asm/mmu_context.h>
 22 #include <asm/tlbflush.h>
 23 
 24 #include "internal.h"
 25 
 26 struct follow_page_context {
 27         struct dev_pagemap *pgmap;
 28         unsigned int page_mask;
 29 };
 30 
 31 static void hpage_pincount_add(struct page *page, int refs)
 32 {
 33         VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
 34         VM_BUG_ON_PAGE(page != compound_head(page), page);
 35 
 36         atomic_add(refs, compound_pincount_ptr(page));
 37 }
 38 
 39 static void hpage_pincount_sub(struct page *page, int refs)
 40 {
 41         VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
 42         VM_BUG_ON_PAGE(page != compound_head(page), page);
 43 
 44         atomic_sub(refs, compound_pincount_ptr(page));
 45 }
 46 
 47 /*
 48  * Return the compound head page with ref appropriately incremented,
 49  * or NULL if that failed.
 50  */
 51 static inline struct page *try_get_compound_head(struct page *page, int refs)
 52 {
 53         struct page *head = compound_head(page);
 54 
 55         if (WARN_ON_ONCE(page_ref_count(head) < 0))
 56                 return NULL;
 57         if (unlikely(!page_cache_add_speculative(head, refs)))
 58                 return NULL;
 59         return head;
 60 }
 61 
 62 /*
 63  * try_grab_compound_head() - attempt to elevate a page's refcount, by a
 64  * flags-dependent amount.
 65  *
 66  * "grab" names in this file mean, "look at flags to decide whether to use
 67  * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.
 68  *
 69  * Either FOLL_PIN or FOLL_GET (or neither) must be set, but not both at the
 70  * same time. (That's true throughout the get_user_pages*() and
 71  * pin_user_pages*() APIs.) Cases:
 72  *
 73  *    FOLL_GET: page's refcount will be incremented by 1.
 74  *    FOLL_PIN: page's refcount will be incremented by GUP_PIN_COUNTING_BIAS.
 75  *
 76  * Return: head page (with refcount appropriately incremented) for success, or
 77  * NULL upon failure. If neither FOLL_GET nor FOLL_PIN was set, that's
 78  * considered failure, and furthermore, a likely bug in the caller, so a warning
 79  * is also emitted.
 80  */
 81 static __maybe_unused struct page *try_grab_compound_head(struct page *page,
 82                                                           int refs,
 83                                                           unsigned int flags)
 84 {
 85         if (flags & FOLL_GET)
 86                 return try_get_compound_head(page, refs);
 87         else if (flags & FOLL_PIN) {
 88                 int orig_refs = refs;
 89 
 90                 /*
 91                  * Can't do FOLL_LONGTERM + FOLL_PIN with CMA in the gup fast
 92                  * path, so fail and let the caller fall back to the slow path.
 93                  */
 94                 if (unlikely(flags & FOLL_LONGTERM) &&
 95                                 is_migrate_cma_page(page))
 96                         return NULL;
 97 
 98                 /*
 99                  * When pinning a compound page of order > 1 (which is what
100                  * hpage_pincount_available() checks for), use an exact count to
101                  * track it, via hpage_pincount_add/_sub().
102                  *
103                  * However, be sure to *also* increment the normal page refcount
104                  * field at least once, so that the page really is pinned.
105                  */
106                 if (!hpage_pincount_available(page))
107                         refs *= GUP_PIN_COUNTING_BIAS;
108 
109                 page = try_get_compound_head(page, refs);
110                 if (!page)
111                         return NULL;
112 
113                 if (hpage_pincount_available(page))
114                         hpage_pincount_add(page, refs);
115 
116                 mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED,
117                                     orig_refs);
118 
119                 return page;
120         }
121 
122         WARN_ON_ONCE(1);
123         return NULL;
124 }
125 
126 /**
127  * try_grab_page() - elevate a page's refcount by a flag-dependent amount
128  *
129  * This might not do anything at all, depending on the flags argument.
130  *
131  * "grab" names in this file mean, "look at flags to decide whether to use
132  * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.
133  *
134  * @page:    pointer to page to be grabbed
135  * @flags:   gup flags: these are the FOLL_* flag values.
136  *
137  * Either FOLL_PIN or FOLL_GET (or neither) may be set, but not both at the same
138  * time. Cases:
139  *
140  *    FOLL_GET: page's refcount will be incremented by 1.
141  *    FOLL_PIN: page's refcount will be incremented by GUP_PIN_COUNTING_BIAS.
142  *
143  * Return: true for success, or if no action was required (if neither FOLL_PIN
144  * nor FOLL_GET was set, nothing is done). False for failure: FOLL_GET or
145  * FOLL_PIN was set, but the page could not be grabbed.
146  */
147 bool __must_check try_grab_page(struct page *page, unsigned int flags)
148 {
149         WARN_ON_ONCE((flags & (FOLL_GET | FOLL_PIN)) == (FOLL_GET | FOLL_PIN));
150 
151         if (flags & FOLL_GET)
152                 return try_get_page(page);
153         else if (flags & FOLL_PIN) {
154                 int refs = 1;
155 
156                 page = compound_head(page);
157 
158                 if (WARN_ON_ONCE(page_ref_count(page) <= 0))
159                         return false;
160 
161                 if (hpage_pincount_available(page))
162                         hpage_pincount_add(page, 1);
163                 else
164                         refs = GUP_PIN_COUNTING_BIAS;
165 
166                 /*
167                  * Similar to try_grab_compound_head(): even if using the
168                  * hpage_pincount_add/_sub() routines, be sure to
169                  * *also* increment the normal page refcount field at least
170                  * once, so that the page really is pinned.
171                  */
172                 page_ref_add(page, refs);
173 
174                 mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED, 1);
175         }
176 
177         return true;
178 }
179 
180 #ifdef CONFIG_DEV_PAGEMAP_OPS
181 static bool __unpin_devmap_managed_user_page(struct page *page)
182 {
183         int count, refs = 1;
184 
185         if (!page_is_devmap_managed(page))
186                 return false;
187 
188         if (hpage_pincount_available(page))
189                 hpage_pincount_sub(page, 1);
190         else
191                 refs = GUP_PIN_COUNTING_BIAS;
192 
193         count = page_ref_sub_return(page, refs);
194 
195         mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_RELEASED, 1);
196         /*
197          * devmap page refcounts are 1-based, rather than 0-based: if
198          * refcount is 1, then the page is free and the refcount is
199          * stable because nobody holds a reference on the page.
200          */
201         if (count == 1)
202                 free_devmap_managed_page(page);
203         else if (!count)
204                 __put_page(page);
205 
206         return true;
207 }
208 #else
209 static bool __unpin_devmap_managed_user_page(struct page *page)
210 {
211         return false;
212 }
213 #endif /* CONFIG_DEV_PAGEMAP_OPS */
214 
215 /**
216  * unpin_user_page() - release a dma-pinned page
217  * @page:            pointer to page to be released
218  *
219  * Pages that were pinned via pin_user_pages*() must be released via either
220  * unpin_user_page(), or one of the unpin_user_pages*() routines. This is so
221  * that such pages can be separately tracked and uniquely handled. In
222  * particular, interactions with RDMA and filesystems need special handling.
223  */
224 void unpin_user_page(struct page *page)
225 {
226         int refs = 1;
227 
228         page = compound_head(page);
229 
230         /*
231          * For devmap managed pages we need to catch refcount transition from
232          * GUP_PIN_COUNTING_BIAS to 1, when refcount reach one it means the
233          * page is free and we need to inform the device driver through
234          * callback. See include/linux/memremap.h and HMM for details.
235          */
236         if (__unpin_devmap_managed_user_page(page))
237                 return;
238 
239         if (hpage_pincount_available(page))
240                 hpage_pincount_sub(page, 1);
241         else
242                 refs = GUP_PIN_COUNTING_BIAS;
243 
244         if (page_ref_sub_and_test(page, refs))
245                 __put_page(page);
246 
247         mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_RELEASED, 1);
248 }
249 EXPORT_SYMBOL(unpin_user_page);
250 
251 /**
252  * unpin_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages
253  * @pages:  array of pages to be maybe marked dirty, and definitely released.
254  * @npages: number of pages in the @pages array.
255  * @make_dirty: whether to mark the pages dirty
256  *
257  * "gup-pinned page" refers to a page that has had one of the get_user_pages()
258  * variants called on that page.
259  *
260  * For each page in the @pages array, make that page (or its head page, if a
261  * compound page) dirty, if @make_dirty is true, and if the page was previously
262  * listed as clean. In any case, releases all pages using unpin_user_page(),
263  * possibly via unpin_user_pages(), for the non-dirty case.
264  *
265  * Please see the unpin_user_page() documentation for details.
266  *
267  * set_page_dirty_lock() is used internally. If instead, set_page_dirty() is
268  * required, then the caller should a) verify that this is really correct,
269  * because _lock() is usually required, and b) hand code it:
270  * set_page_dirty_lock(), unpin_user_page().
271  *
272  */
273 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
274                                  bool make_dirty)
275 {
276         unsigned long index;
277 
278         /*
279          * TODO: this can be optimized for huge pages: if a series of pages is
280          * physically contiguous and part of the same compound page, then a
281          * single operation to the head page should suffice.
282          */
283 
284         if (!make_dirty) {
285                 unpin_user_pages(pages, npages);
286                 return;
287         }
288 
289         for (index = 0; index < npages; index++) {
290                 struct page *page = compound_head(pages[index]);
291                 /*
292                  * Checking PageDirty at this point may race with
293                  * clear_page_dirty_for_io(), but that's OK. Two key
294                  * cases:
295                  *
296                  * 1) This code sees the page as already dirty, so it
297                  * skips the call to set_page_dirty(). That could happen
298                  * because clear_page_dirty_for_io() called
299                  * page_mkclean(), followed by set_page_dirty().
300                  * However, now the page is going to get written back,
301                  * which meets the original intention of setting it
302                  * dirty, so all is well: clear_page_dirty_for_io() goes
303                  * on to call TestClearPageDirty(), and write the page
304                  * back.
305                  *
306                  * 2) This code sees the page as clean, so it calls
307                  * set_page_dirty(). The page stays dirty, despite being
308                  * written back, so it gets written back again in the
309                  * next writeback cycle. This is harmless.
310                  */
311                 if (!PageDirty(page))
312                         set_page_dirty_lock(page);
313                 unpin_user_page(page);
314         }
315 }
316 EXPORT_SYMBOL(unpin_user_pages_dirty_lock);
317 
318 /**
319  * unpin_user_pages() - release an array of gup-pinned pages.
320  * @pages:  array of pages to be marked dirty and released.
321  * @npages: number of pages in the @pages array.
322  *
323  * For each page in the @pages array, release the page using unpin_user_page().
324  *
325  * Please see the unpin_user_page() documentation for details.
326  */
327 void unpin_user_pages(struct page **pages, unsigned long npages)
328 {
329         unsigned long index;
330 
331         /*
332          * TODO: this can be optimized for huge pages: if a series of pages is
333          * physically contiguous and part of the same compound page, then a
334          * single operation to the head page should suffice.
335          */
336         for (index = 0; index < npages; index++)
337                 unpin_user_page(pages[index]);
338 }
339 EXPORT_SYMBOL(unpin_user_pages);
340 
341 #ifdef CONFIG_MMU
342 static struct page *no_page_table(struct vm_area_struct *vma,
343                 unsigned int flags)
344 {
345         /*
346          * When core dumping an enormous anonymous area that nobody
347          * has touched so far, we don't want to allocate unnecessary pages or
348          * page tables.  Return error instead of NULL to skip handle_mm_fault,
349          * then get_dump_page() will return NULL to leave a hole in the dump.
350          * But we can only make this optimization where a hole would surely
351          * be zero-filled if handle_mm_fault() actually did handle it.
352          */
353         if ((flags & FOLL_DUMP) &&
354                         (vma_is_anonymous(vma) || !vma->vm_ops->fault))
355                 return ERR_PTR(-EFAULT);
356         return NULL;
357 }
358 
359 static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address,
360                 pte_t *pte, unsigned int flags)
361 {
362         /* No page to get reference */
363         if (flags & FOLL_GET)
364                 return -EFAULT;
365 
366         if (flags & FOLL_TOUCH) {
367                 pte_t entry = *pte;
368 
369                 if (flags & FOLL_WRITE)
370                         entry = pte_mkdirty(entry);
371                 entry = pte_mkyoung(entry);
372 
373                 if (!pte_same(*pte, entry)) {
374                         set_pte_at(vma->vm_mm, address, pte, entry);
375                         update_mmu_cache(vma, address, pte);
376                 }
377         }
378 
379         /* Proper page table entry exists, but no corresponding struct page */
380         return -EEXIST;
381 }
382 
383 /*
384  * FOLL_FORCE can write to even unwritable pte's, but only
385  * after we've gone through a COW cycle and they are dirty.
386  */
387 static inline bool can_follow_write_pte(pte_t pte, unsigned int flags)
388 {
389         return pte_write(pte) ||
390                 ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pte_dirty(pte));
391 }
392 
393 static struct page *follow_page_pte(struct vm_area_struct *vma,
394                 unsigned long address, pmd_t *pmd, unsigned int flags,
395                 struct dev_pagemap **pgmap)
396 {
397         struct mm_struct *mm = vma->vm_mm;
398         struct page *page;
399         spinlock_t *ptl;
400         pte_t *ptep, pte;
401         int ret;
402 
403         /* FOLL_GET and FOLL_PIN are mutually exclusive. */
404         if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) ==
405                          (FOLL_PIN | FOLL_GET)))
406                 return ERR_PTR(-EINVAL);
407 retry:
408         if (unlikely(pmd_bad(*pmd)))
409                 return no_page_table(vma, flags);
410 
411         ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
412         pte = *ptep;
413         if (!pte_present(pte)) {
414                 swp_entry_t entry;
415                 /*
416                  * KSM's break_ksm() relies upon recognizing a ksm page
417                  * even while it is being migrated, so for that case we
418                  * need migration_entry_wait().
419                  */
420                 if (likely(!(flags & FOLL_MIGRATION)))
421                         goto no_page;
422                 if (pte_none(pte))
423                         goto no_page;
424                 entry = pte_to_swp_entry(pte);
425                 if (!is_migration_entry(entry))
426                         goto no_page;
427                 pte_unmap_unlock(ptep, ptl);
428                 migration_entry_wait(mm, pmd, address);
429                 goto retry;
430         }
431         if ((flags & FOLL_NUMA) && pte_protnone(pte))
432                 goto no_page;
433         if ((flags & FOLL_WRITE) && !can_follow_write_pte(pte, flags)) {
434                 pte_unmap_unlock(ptep, ptl);
435                 return NULL;
436         }
437 
438         page = vm_normal_page(vma, address, pte);
439         if (!page && pte_devmap(pte) && (flags & (FOLL_GET | FOLL_PIN))) {
440                 /*
441                  * Only return device mapping pages in the FOLL_GET or FOLL_PIN
442                  * case since they are only valid while holding the pgmap
443                  * reference.
444                  */
445                 *pgmap = get_dev_pagemap(pte_pfn(pte), *pgmap);
446                 if (*pgmap)
447                         page = pte_page(pte);
448                 else
449                         goto no_page;
450         } else if (unlikely(!page)) {
451                 if (flags & FOLL_DUMP) {
452                         /* Avoid special (like zero) pages in core dumps */
453                         page = ERR_PTR(-EFAULT);
454                         goto out;
455                 }
456 
457                 if (is_zero_pfn(pte_pfn(pte))) {
458                         page = pte_page(pte);
459                 } else {
460                         ret = follow_pfn_pte(vma, address, ptep, flags);
461                         page = ERR_PTR(ret);
462                         goto out;
463                 }
464         }
465 
466         if (flags & FOLL_SPLIT && PageTransCompound(page)) {
467                 get_page(page);
468                 pte_unmap_unlock(ptep, ptl);
469                 lock_page(page);
470                 ret = split_huge_page(page);
471                 unlock_page(page);
472                 put_page(page);
473                 if (ret)
474                         return ERR_PTR(ret);
475                 goto retry;
476         }
477 
478         /* try_grab_page() does nothing unless FOLL_GET or FOLL_PIN is set. */
479         if (unlikely(!try_grab_page(page, flags))) {
480                 page = ERR_PTR(-ENOMEM);
481                 goto out;
482         }
483         /*
484          * We need to make the page accessible if and only if we are going
485          * to access its content (the FOLL_PIN case).  Please see
486          * Documentation/core-api/pin_user_pages.rst for details.
487          */
488         if (flags & FOLL_PIN) {
489                 ret = arch_make_page_accessible(page);
490                 if (ret) {
491                         unpin_user_page(page);
492                         page = ERR_PTR(ret);
493                         goto out;
494                 }
495         }
496         if (flags & FOLL_TOUCH) {
497                 if ((flags & FOLL_WRITE) &&
498                     !pte_dirty(pte) && !PageDirty(page))
499                         set_page_dirty(page);
500                 /*
501                  * pte_mkyoung() would be more correct here, but atomic care
502                  * is needed to avoid losing the dirty bit: it is easier to use
503                  * mark_page_accessed().
504                  */
505                 mark_page_accessed(page);
506         }
507         if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
508                 /* Do not mlock pte-mapped THP */
509                 if (PageTransCompound(page))
510                         goto out;
511 
512                 /*
513                  * The preliminary mapping check is mainly to avoid the
514                  * pointless overhead of lock_page on the ZERO_PAGE
515                  * which might bounce very badly if there is contention.
516                  *
517                  * If the page is already locked, we don't need to
518                  * handle it now - vmscan will handle it later if and
519                  * when it attempts to reclaim the page.
520                  */
521                 if (page->mapping && trylock_page(page)) {
522                         lru_add_drain();  /* push cached pages to LRU */
523                         /*
524                          * Because we lock page here, and migration is
525                          * blocked by the pte's page reference, and we
526                          * know the page is still mapped, we don't even
527                          * need to check for file-cache page truncation.
528                          */
529                         mlock_vma_page(page);
530                         unlock_page(page);
531                 }
532         }
533 out:
534         pte_unmap_unlock(ptep, ptl);
535         return page;
536 no_page:
537         pte_unmap_unlock(ptep, ptl);
538         if (!pte_none(pte))
539                 return NULL;
540         return no_page_table(vma, flags);
541 }
542 
543 static struct page *follow_pmd_mask(struct vm_area_struct *vma,
544                                     unsigned long address, pud_t *pudp,
545                                     unsigned int flags,
546                                     struct follow_page_context *ctx)
547 {
548         pmd_t *pmd, pmdval;
549         spinlock_t *ptl;
550         struct page *page;
551         struct mm_struct *mm = vma->vm_mm;
552 
553         pmd = pmd_offset(pudp, address);
554         /*
555          * The READ_ONCE() will stabilize the pmdval in a register or
556          * on the stack so that it will stop changing under the code.
557          */
558         pmdval = READ_ONCE(*pmd);
559         if (pmd_none(pmdval))
560                 return no_page_table(vma, flags);
561         if (pmd_huge(pmdval) && is_vm_hugetlb_page(vma)) {
562                 page = follow_huge_pmd(mm, address, pmd, flags);
563                 if (page)
564                         return page;
565                 return no_page_table(vma, flags);
566         }
567         if (is_hugepd(__hugepd(pmd_val(pmdval)))) {
568                 page = follow_huge_pd(vma, address,
569                                       __hugepd(pmd_val(pmdval)), flags,
570                                       PMD_SHIFT);
571                 if (page)
572                         return page;
573                 return no_page_table(vma, flags);
574         }
575 retry:
576         if (!pmd_present(pmdval)) {
577                 if (likely(!(flags & FOLL_MIGRATION)))
578                         return no_page_table(vma, flags);
579                 VM_BUG_ON(thp_migration_supported() &&
580                                   !is_pmd_migration_entry(pmdval));
581                 if (is_pmd_migration_entry(pmdval))
582                         pmd_migration_entry_wait(mm, pmd);
583                 pmdval = READ_ONCE(*pmd);
584                 /*
585                  * MADV_DONTNEED may convert the pmd to null because
586                  * mmap_lock is held in read mode
587                  */
588                 if (pmd_none(pmdval))
589                         return no_page_table(vma, flags);
590                 goto retry;
591         }
592         if (pmd_devmap(pmdval)) {
593                 ptl = pmd_lock(mm, pmd);
594                 page = follow_devmap_pmd(vma, address, pmd, flags, &ctx->pgmap);
595                 spin_unlock(ptl);
596                 if (page)
597                         return page;
598         }
599         if (likely(!pmd_trans_huge(pmdval)))
600                 return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
601 
602         if ((flags & FOLL_NUMA) && pmd_protnone(pmdval))
603                 return no_page_table(vma, flags);
604 
605 retry_locked:
606         ptl = pmd_lock(mm, pmd);
607         if (unlikely(pmd_none(*pmd))) {
608                 spin_unlock(ptl);
609                 return no_page_table(vma, flags);
610         }
611         if (unlikely(!pmd_present(*pmd))) {
612                 spin_unlock(ptl);
613                 if (likely(!(flags & FOLL_MIGRATION)))
614                         return no_page_table(vma, flags);
615                 pmd_migration_entry_wait(mm, pmd);
616                 goto retry_locked;
617         }
618         if (unlikely(!pmd_trans_huge(*pmd))) {
619                 spin_unlock(ptl);
620                 return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
621         }
622         if (flags & (FOLL_SPLIT | FOLL_SPLIT_PMD)) {
623                 int ret;
624                 page = pmd_page(*pmd);
625                 if (is_huge_zero_page(page)) {
626                         spin_unlock(ptl);
627                         ret = 0;
628                         split_huge_pmd(vma, pmd, address);
629                         if (pmd_trans_unstable(pmd))
630                                 ret = -EBUSY;
631                 } else if (flags & FOLL_SPLIT) {
632                         if (unlikely(!try_get_page(page))) {
633                                 spin_unlock(ptl);
634                                 return ERR_PTR(-ENOMEM);
635                         }
636                         spin_unlock(ptl);
637                         lock_page(page);
638                         ret = split_huge_page(page);
639                         unlock_page(page);
640                         put_page(page);
641                         if (pmd_none(*pmd))
642                                 return no_page_table(vma, flags);
643                 } else {  /* flags & FOLL_SPLIT_PMD */
644                         spin_unlock(ptl);
645                         split_huge_pmd(vma, pmd, address);
646                         ret = pte_alloc(mm, pmd) ? -ENOMEM : 0;
647                 }
648 
649                 return ret ? ERR_PTR(ret) :
650                         follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
651         }
652         page = follow_trans_huge_pmd(vma, address, pmd, flags);
653         spin_unlock(ptl);
654         ctx->page_mask = HPAGE_PMD_NR - 1;
655         return page;
656 }
657 
658 static struct page *follow_pud_mask(struct vm_area_struct *vma,
659                                     unsigned long address, p4d_t *p4dp,
660                                     unsigned int flags,
661                                     struct follow_page_context *ctx)
662 {
663         pud_t *pud;
664         spinlock_t *ptl;
665         struct page *page;
666         struct mm_struct *mm = vma->vm_mm;
667 
668         pud = pud_offset(p4dp, address);
669         if (pud_none(*pud))
670                 return no_page_table(vma, flags);
671         if (pud_huge(*pud) && is_vm_hugetlb_page(vma)) {
672                 page = follow_huge_pud(mm, address, pud, flags);
673                 if (page)
674                         return page;
675                 return no_page_table(vma, flags);
676         }
677         if (is_hugepd(__hugepd(pud_val(*pud)))) {
678                 page = follow_huge_pd(vma, address,
679                                       __hugepd(pud_val(*pud)), flags,
680                                       PUD_SHIFT);
681                 if (page)
682                         return page;
683                 return no_page_table(vma, flags);
684         }
685         if (pud_devmap(*pud)) {
686                 ptl = pud_lock(mm, pud);
687                 page = follow_devmap_pud(vma, address, pud, flags, &ctx->pgmap);
688                 spin_unlock(ptl);
689                 if (page)
690                         return page;
691         }
692         if (unlikely(pud_bad(*pud)))
693                 return no_page_table(vma, flags);
694 
695         return follow_pmd_mask(vma, address, pud, flags, ctx);
696 }
697 
698 static struct page *follow_p4d_mask(struct vm_area_struct *vma,
699                                     unsigned long address, pgd_t *pgdp,
700                                     unsigned int flags,
701                                     struct follow_page_context *ctx)
702 {
703         p4d_t *p4d;
704         struct page *page;
705 
706         p4d = p4d_offset(pgdp, address);
707         if (p4d_none(*p4d))
708                 return no_page_table(vma, flags);
709         BUILD_BUG_ON(p4d_huge(*p4d));
710         if (unlikely(p4d_bad(*p4d)))
711                 return no_page_table(vma, flags);
712 
713         if (is_hugepd(__hugepd(p4d_val(*p4d)))) {
714                 page = follow_huge_pd(vma, address,
715                                       __hugepd(p4d_val(*p4d)), flags,
716                                       P4D_SHIFT);
717                 if (page)
718                         return page;
719                 return no_page_table(vma, flags);
720         }
721         return follow_pud_mask(vma, address, p4d, flags, ctx);
722 }
723 
724 /**
725  * follow_page_mask - look up a page descriptor from a user-virtual address
726  * @vma: vm_area_struct mapping @address
727  * @address: virtual address to look up
728  * @flags: flags modifying lookup behaviour
729  * @ctx: contains dev_pagemap for %ZONE_DEVICE memory pinning and a
730  *       pointer to output page_mask
731  *
732  * @flags can have FOLL_ flags set, defined in <linux/mm.h>
733  *
734  * When getting pages from ZONE_DEVICE memory, the @ctx->pgmap caches
735  * the device's dev_pagemap metadata to avoid repeating expensive lookups.
736  *
737  * On output, the @ctx->page_mask is set according to the size of the page.
738  *
739  * Return: the mapped (struct page *), %NULL if no mapping exists, or
740  * an error pointer if there is a mapping to something not represented
741  * by a page descriptor (see also vm_normal_page()).
742  */
743 static struct page *follow_page_mask(struct vm_area_struct *vma,
744                               unsigned long address, unsigned int flags,
745                               struct follow_page_context *ctx)
746 {
747         pgd_t *pgd;
748         struct page *page;
749         struct mm_struct *mm = vma->vm_mm;
750 
751         ctx->page_mask = 0;
752 
753         /* make this handle hugepd */
754         page = follow_huge_addr(mm, address, flags & FOLL_WRITE);
755         if (!IS_ERR(page)) {
756                 WARN_ON_ONCE(flags & (FOLL_GET | FOLL_PIN));
757                 return page;
758         }
759 
760         pgd = pgd_offset(mm, address);
761 
762         if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
763                 return no_page_table(vma, flags);
764 
765         if (pgd_huge(*pgd)) {
766                 page = follow_huge_pgd(mm, address, pgd, flags);
767                 if (page)
768                         return page;
769                 return no_page_table(vma, flags);
770         }
771         if (is_hugepd(__hugepd(pgd_val(*pgd)))) {
772                 page = follow_huge_pd(vma, address,
773                                       __hugepd(pgd_val(*pgd)), flags,
774                                       PGDIR_SHIFT);
775                 if (page)
776                         return page;
777                 return no_page_table(vma, flags);
778         }
779 
780         return follow_p4d_mask(vma, address, pgd, flags, ctx);
781 }
782 
783 struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
784                          unsigned int foll_flags)
785 {
786         struct follow_page_context ctx = { NULL };
787         struct page *page;
788 
789         page = follow_page_mask(vma, address, foll_flags, &ctx);
790         if (ctx.pgmap)
791                 put_dev_pagemap(ctx.pgmap);
792         return page;
793 }
794 
795 static int get_gate_page(struct mm_struct *mm, unsigned long address,
796                 unsigned int gup_flags, struct vm_area_struct **vma,
797                 struct page **page)
798 {
799         pgd_t *pgd;
800         p4d_t *p4d;
801         pud_t *pud;
802         pmd_t *pmd;
803         pte_t *pte;
804         int ret = -EFAULT;
805 
806         /* user gate pages are read-only */
807         if (gup_flags & FOLL_WRITE)
808                 return -EFAULT;
809         if (address > TASK_SIZE)
810                 pgd = pgd_offset_k(address);
811         else
812                 pgd = pgd_offset_gate(mm, address);
813         if (pgd_none(*pgd))
814                 return -EFAULT;
815         p4d = p4d_offset(pgd, address);
816         if (p4d_none(*p4d))
817                 return -EFAULT;
818         pud = pud_offset(p4d, address);
819         if (pud_none(*pud))
820                 return -EFAULT;
821         pmd = pmd_offset(pud, address);
822         if (!pmd_present(*pmd))
823                 return -EFAULT;
824         VM_BUG_ON(pmd_trans_huge(*pmd));
825         pte = pte_offset_map(pmd, address);
826         if (pte_none(*pte))
827                 goto unmap;
828         *vma = get_gate_vma(mm);
829         if (!page)
830                 goto out;
831         *page = vm_normal_page(*vma, address, *pte);
832         if (!*page) {
833                 if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(*pte)))
834                         goto unmap;
835                 *page = pte_page(*pte);
836         }
837         if (unlikely(!try_grab_page(*page, gup_flags))) {
838                 ret = -ENOMEM;
839                 goto unmap;
840         }
841 out:
842         ret = 0;
843 unmap:
844         pte_unmap(pte);
845         return ret;
846 }
847 
848 /*
849  * mmap_lock must be held on entry.  If @locked != NULL and *@flags
850  * does not include FOLL_NOWAIT, the mmap_lock may be released.  If it
851  * is, *@locked will be set to 0 and -EBUSY returned.
852  */
853 static int faultin_page(struct vm_area_struct *vma,
854                 unsigned long address, unsigned int *flags, int *locked)
855 {
856         unsigned int fault_flags = 0;
857         vm_fault_t ret;
858 
859         /* mlock all present pages, but do not fault in new pages */
860         if ((*flags & (FOLL_POPULATE | FOLL_MLOCK)) == FOLL_MLOCK)
861                 return -ENOENT;
862         if (*flags & FOLL_WRITE)
863                 fault_flags |= FAULT_FLAG_WRITE;
864         if (*flags & FOLL_REMOTE)
865                 fault_flags |= FAULT_FLAG_REMOTE;
866         if (locked)
867                 fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
868         if (*flags & FOLL_NOWAIT)
869                 fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT;
870         if (*flags & FOLL_TRIED) {
871                 /*
872                  * Note: FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_TRIED
873                  * can co-exist
874                  */
875                 fault_flags |= FAULT_FLAG_TRIED;
876         }
877 
878         ret = handle_mm_fault(vma, address, fault_flags, NULL);
879         if (ret & VM_FAULT_ERROR) {
880                 int err = vm_fault_to_errno(ret, *flags);
881 
882                 if (err)
883                         return err;
884                 BUG();
885         }
886 
887         if (ret & VM_FAULT_RETRY) {
888                 if (locked && !(fault_flags & FAULT_FLAG_RETRY_NOWAIT))
889                         *locked = 0;
890                 return -EBUSY;
891         }
892 
893         /*
894          * The VM_FAULT_WRITE bit tells us that do_wp_page has broken COW when
895          * necessary, even if maybe_mkwrite decided not to set pte_write. We
896          * can thus safely do subsequent page lookups as if they were reads.
897          * But only do so when looping for pte_write is futile: in some cases
898          * userspace may also be wanting to write to the gotten user page,
899          * which a read fault here might prevent (a readonly page might get
900          * reCOWed by userspace write).
901          */
902         if ((ret & VM_FAULT_WRITE) && !(vma->vm_flags & VM_WRITE))
903                 *flags |= FOLL_COW;
904         return 0;
905 }
906 
907 static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
908 {
909         vm_flags_t vm_flags = vma->vm_flags;
910         int write = (gup_flags & FOLL_WRITE);
911         int foreign = (gup_flags & FOLL_REMOTE);
912 
913         if (vm_flags & (VM_IO | VM_PFNMAP))
914                 return -EFAULT;
915 
916         if (gup_flags & FOLL_ANON && !vma_is_anonymous(vma))
917                 return -EFAULT;
918 
919         if (write) {
920                 if (!(vm_flags & VM_WRITE)) {
921                         if (!(gup_flags & FOLL_FORCE))
922                                 return -EFAULT;
923                         /*
924                          * We used to let the write,force case do COW in a
925                          * VM_MAYWRITE VM_SHARED !VM_WRITE vma, so ptrace could
926                          * set a breakpoint in a read-only mapping of an
927                          * executable, without corrupting the file (yet only
928                          * when that file had been opened for writing!).
929                          * Anon pages in shared mappings are surprising: now
930                          * just reject it.
931                          */
932                         if (!is_cow_mapping(vm_flags))
933                                 return -EFAULT;
934                 }
935         } else if (!(vm_flags & VM_READ)) {
936                 if (!(gup_flags & FOLL_FORCE))
937                         return -EFAULT;
938                 /*
939                  * Is there actually any vma we can reach here which does not
940                  * have VM_MAYREAD set?
941                  */
942                 if (!(vm_flags & VM_MAYREAD))
943                         return -EFAULT;
944         }
945         /*
946          * gups are always data accesses, not instruction
947          * fetches, so execute=false here
948          */
949         if (!arch_vma_access_permitted(vma, write, false, foreign))
950                 return -EFAULT;
951         return 0;
952 }
953 
954 /**
955  * __get_user_pages() - pin user pages in memory
956  * @mm:         mm_struct of target mm
957  * @start:      starting user address
958  * @nr_pages:   number of pages from start to pin
959  * @gup_flags:  flags modifying pin behaviour
960  * @pages:      array that receives pointers to the pages pinned.
961  *              Should be at least nr_pages long. Or NULL, if caller
962  *              only intends to ensure the pages are faulted in.
963  * @vmas:       array of pointers to vmas corresponding to each page.
964  *              Or NULL if the caller does not require them.
965  * @locked:     whether we're still with the mmap_lock held
966  *
967  * Returns either number of pages pinned (which may be less than the
968  * number requested), or an error. Details about the return value:
969  *
970  * -- If nr_pages is 0, returns 0.
971  * -- If nr_pages is >0, but no pages were pinned, returns -errno.
972  * -- If nr_pages is >0, and some pages were pinned, returns the number of
973  *    pages pinned. Again, this may be less than nr_pages.
974  * -- 0 return value is possible when the fault would need to be retried.
975  *
976  * The caller is responsible for releasing returned @pages, via put_page().
977  *
978  * @vmas are valid only as long as mmap_lock is held.
979  *
980  * Must be called with mmap_lock held.  It may be released.  See below.
981  *
982  * __get_user_pages walks a process's page tables and takes a reference to
983  * each struct page that each user address corresponds to at a given
984  * instant. That is, it takes the page that would be accessed if a user
985  * thread accesses the given user virtual address at that instant.
986  *
987  * This does not guarantee that the page exists in the user mappings when
988  * __get_user_pages returns, and there may even be a completely different
989  * page there in some cases (eg. if mmapped pagecache has been invalidated
990  * and subsequently re faulted). However it does guarantee that the page
991  * won't be freed completely. And mostly callers simply care that the page
992  * contains data that was valid *at some point in time*. Typically, an IO
993  * or similar operation cannot guarantee anything stronger anyway because
994  * locks can't be held over the syscall boundary.
995  *
996  * If @gup_flags & FOLL_WRITE == 0, the page must not be written to. If
997  * the page is written to, set_page_dirty (or set_page_dirty_lock, as
998  * appropriate) must be called after the page is finished with, and
999  * before put_page is called.
1000  *
1001  * If @locked != NULL, *@locked will be set to 0 when mmap_lock is
1002  * released by an up_read().  That can happen if @gup_flags does not
1003  * have FOLL_NOWAIT.
1004  *
1005  * A caller using such a combination of @locked and @gup_flags
1006  * must therefore hold the mmap_lock for reading only, and recognize
1007  * when it's been released.  Otherwise, it must be held for either
1008  * reading or writing and will not be released.
1009  *
1010  * In most cases, get_user_pages or get_user_pages_fast should be used
1011  * instead of __get_user_pages. __get_user_pages should be used only if
1012  * you need some special @gup_flags.
1013  */
1014 static long __get_user_pages(struct mm_struct *mm,
1015                 unsigned long start, unsigned long nr_pages,
1016                 unsigned int gup_flags, struct page **pages,
1017                 struct vm_area_struct **vmas, int *locked)
1018 {
1019         long ret = 0, i = 0;
1020         struct vm_area_struct *vma = NULL;
1021         struct follow_page_context ctx = { NULL };
1022 
1023         if (!nr_pages)
1024                 return 0;
1025 
1026         start = untagged_addr(start);
1027 
1028         VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN)));
1029 
1030         /*
1031          * If FOLL_FORCE is set then do not force a full fault as the hinting
1032          * fault information is unrelated to the reference behaviour of a task
1033          * using the address space
1034          */
1035         if (!(gup_flags & FOLL_FORCE))
1036                 gup_flags |= FOLL_NUMA;
1037 
1038         do {
1039                 struct page *page;
1040                 unsigned int foll_flags = gup_flags;
1041                 unsigned int page_increm;
1042 
1043                 /* first iteration or cross vma bound */
1044                 if (!vma || start >= vma->vm_end) {
1045                         vma = find_extend_vma(mm, start);
1046                         if (!vma && in_gate_area(mm, start)) {
1047                                 ret = get_gate_page(mm, start & PAGE_MASK,
1048                                                 gup_flags, &vma,
1049                                                 pages ? &pages[i] : NULL);
1050                                 if (ret)
1051                                         goto out;
1052                                 ctx.page_mask = 0;
1053                                 goto next_page;
1054                         }
1055 
1056                         if (!vma || check_vma_flags(vma, gup_flags)) {
1057                                 ret = -EFAULT;
1058                                 goto out;
1059                         }
1060                         if (is_vm_hugetlb_page(vma)) {
1061                                 i = follow_hugetlb_page(mm, vma, pages, vmas,
1062                                                 &start, &nr_pages, i,
1063                                                 gup_flags, locked);
1064                                 if (locked && *locked == 0) {
1065                                         /*
1066                                          * We've got a VM_FAULT_RETRY
1067                                          * and we've lost mmap_lock.
1068                                          * We must stop here.
1069                                          */
1070                                         BUG_ON(gup_flags & FOLL_NOWAIT);
1071                                         BUG_ON(ret != 0);
1072                                         goto out;
1073                                 }
1074                                 continue;
1075                         }
1076                 }
1077 retry:
1078                 /*
1079                  * If we have a pending SIGKILL, don't keep faulting pages and
1080                  * potentially allocating memory.
1081                  */
1082                 if (fatal_signal_pending(current)) {
1083                         ret = -EINTR;
1084                         goto out;
1085                 }
1086                 cond_resched();
1087 
1088                 page = follow_page_mask(vma, start, foll_flags, &ctx);
1089                 if (!page) {
1090                         ret = faultin_page(vma, start, &foll_flags, locked);
1091                         switch (ret) {
1092                         case 0:
1093                                 goto retry;
1094                         case -EBUSY:
1095                                 ret = 0;
1096                                 fallthrough;
1097                         case -EFAULT:
1098                         case -ENOMEM:
1099                         case -EHWPOISON:
1100                                 goto out;
1101                         case -ENOENT:
1102                                 goto next_page;
1103                         }
1104                         BUG();
1105                 } else if (PTR_ERR(page) == -EEXIST) {
1106                         /*
1107                          * Proper page table entry exists, but no corresponding
1108                          * struct page.
1109                          */
1110                         goto next_page;
1111                 } else if (IS_ERR(page)) {
1112                         ret = PTR_ERR(page);
1113                         goto out;
1114                 }
1115                 if (pages) {
1116                         pages[i] = page;
1117                         flush_anon_page(vma, page, start);
1118                         flush_dcache_page(page);
1119                         ctx.page_mask = 0;
1120                 }
1121 next_page:
1122                 if (vmas) {
1123                         vmas[i] = vma;
1124                         ctx.page_mask = 0;
1125                 }
1126                 page_increm = 1 + (~(start >> PAGE_SHIFT) & ctx.page_mask);
1127                 if (page_increm > nr_pages)
1128                         page_increm = nr_pages;
1129                 i += page_increm;
1130                 start += page_increm * PAGE_SIZE;
1131                 nr_pages -= page_increm;
1132         } while (nr_pages);
1133 out:
1134         if (ctx.pgmap)
1135                 put_dev_pagemap(ctx.pgmap);
1136         return i ? i : ret;
1137 }
1138 
1139 static bool vma_permits_fault(struct vm_area_struct *vma,
1140                               unsigned int fault_flags)
1141 {
1142         bool write   = !!(fault_flags & FAULT_FLAG_WRITE);
1143         bool foreign = !!(fault_flags & FAULT_FLAG_REMOTE);
1144         vm_flags_t vm_flags = write ? VM_WRITE : VM_READ;
1145 
1146         if (!(vm_flags & vma->vm_flags))
1147                 return false;
1148 
1149         /*
1150          * The architecture might have a hardware protection
1151          * mechanism other than read/write that can deny access.
1152          *
1153          * gup always represents data access, not instruction
1154          * fetches, so execute=false here:
1155          */
1156         if (!arch_vma_access_permitted(vma, write, false, foreign))
1157                 return false;
1158 
1159         return true;
1160 }
1161 
1162 /**
1163  * fixup_user_fault() - manually resolve a user page fault
1164  * @mm:         mm_struct of target mm
1165  * @address:    user address
1166  * @fault_flags:flags to pass down to handle_mm_fault()
1167  * @unlocked:   did we unlock the mmap_lock while retrying, maybe NULL if caller
1168  *              does not allow retry. If NULL, the caller must guarantee
1169  *              that fault_flags does not contain FAULT_FLAG_ALLOW_RETRY.
1170  *
1171  * This is meant to be called in the specific scenario where for locking reasons
1172  * we try to access user memory in atomic context (within a pagefault_disable()
1173  * section), this returns -EFAULT, and we want to resolve the user fault before
1174  * trying again.
1175  *
1176  * Typically this is meant to be used by the futex code.
1177  *
1178  * The main difference with get_user_pages() is that this function will
1179  * unconditionally call handle_mm_fault() which will in turn perform all the
1180  * necessary SW fixup of the dirty and young bits in the PTE, while
1181  * get_user_pages() only guarantees to update these in the struct page.
1182  *
1183  * This is important for some architectures where those bits also gate the
1184  * access permission to the page because they are maintained in software.  On
1185  * such architectures, gup() will not be enough to make a subsequent access
1186  * succeed.
1187  *
1188  * This function will not return with an unlocked mmap_lock. So it has not the
1189  * same semantics wrt the @mm->mmap_lock as does filemap_fault().
1190  */
1191 int fixup_user_fault(struct mm_struct *mm,
1192                      unsigned long address, unsigned int fault_flags,
1193                      bool *unlocked)
1194 {
1195         struct vm_area_struct *vma;
1196         vm_fault_t ret, major = 0;
1197 
1198         address = untagged_addr(address);
1199 
1200         if (unlocked)
1201                 fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
1202 
1203 retry:
1204         vma = find_extend_vma(mm, address);
1205         if (!vma || address < vma->vm_start)
1206                 return -EFAULT;
1207 
1208         if (!vma_permits_fault(vma, fault_flags))
1209                 return -EFAULT;
1210 
1211         if ((fault_flags & FAULT_FLAG_KILLABLE) &&
1212             fatal_signal_pending(current))
1213                 return -EINTR;
1214 
1215         ret = handle_mm_fault(vma, address, fault_flags, NULL);
1216         major |= ret & VM_FAULT_MAJOR;
1217         if (ret & VM_FAULT_ERROR) {
1218                 int err = vm_fault_to_errno(ret, 0);
1219 
1220                 if (err)
1221                         return err;
1222                 BUG();
1223         }
1224 
1225         if (ret & VM_FAULT_RETRY) {
1226                 mmap_read_lock(mm);
1227                 *unlocked = true;
1228                 fault_flags |= FAULT_FLAG_TRIED;
1229                 goto retry;
1230         }
1231 
1232         return 0;
1233 }
1234 EXPORT_SYMBOL_GPL(fixup_user_fault);
1235 
1236 /*
1237  * Please note that this function, unlike __get_user_pages will not
1238  * return 0 for nr_pages > 0 without FOLL_NOWAIT
1239  */
1240 static __always_inline long __get_user_pages_locked(struct mm_struct *mm,
1241                                                 unsigned long start,
1242                                                 unsigned long nr_pages,
1243                                                 struct page **pages,
1244                                                 struct vm_area_struct **vmas,
1245                                                 int *locked,
1246                                                 unsigned int flags)
1247 {
1248         long ret, pages_done;
1249         bool lock_dropped;
1250 
1251         if (locked) {
1252                 /* if VM_FAULT_RETRY can be returned, vmas become invalid */
1253                 BUG_ON(vmas);
1254                 /* check caller initialized locked */
1255                 BUG_ON(*locked != 1);
1256         }
1257 
1258         if (flags & FOLL_PIN)
1259                 atomic_set(&mm->has_pinned, 1);
1260 
1261         /*
1262          * FOLL_PIN and FOLL_GET are mutually exclusive. Traditional behavior
1263          * is to set FOLL_GET if the caller wants pages[] filled in (but has
1264          * carelessly failed to specify FOLL_GET), so keep doing that, but only
1265          * for FOLL_GET, not for the newer FOLL_PIN.
1266          *
1267          * FOLL_PIN always expects pages to be non-null, but no need to assert
1268          * that here, as any failures will be obvious enough.
1269          */
1270         if (pages && !(flags & FOLL_PIN))
1271                 flags |= FOLL_GET;
1272 
1273         pages_done = 0;
1274         lock_dropped = false;
1275         for (;;) {
1276                 ret = __get_user_pages(mm, start, nr_pages, flags, pages,
1277                                        vmas, locked);
1278                 if (!locked)
1279                         /* VM_FAULT_RETRY couldn't trigger, bypass */
1280                         return ret;
1281 
1282                 /* VM_FAULT_RETRY cannot return errors */
1283                 if (!*locked) {
1284                         BUG_ON(ret < 0);
1285                         BUG_ON(ret >= nr_pages);
1286                 }
1287 
1288                 if (ret > 0) {
1289                         nr_pages -= ret;
1290                         pages_done += ret;
1291                         if (!nr_pages)
1292                                 break;
1293                 }
1294                 if (*locked) {
1295                         /*
1296                          * VM_FAULT_RETRY didn't trigger or it was a
1297                          * FOLL_NOWAIT.
1298                          */
1299                         if (!pages_done)
1300                                 pages_done = ret;
1301                         break;
1302                 }
1303                 /*
1304                  * VM_FAULT_RETRY triggered, so seek to the faulting offset.
1305                  * For the prefault case (!pages) we only update counts.
1306                  */
1307                 if (likely(pages))
1308                         pages += ret;
1309                 start += ret << PAGE_SHIFT;
1310                 lock_dropped = true;
1311 
1312 retry:
1313                 /*
1314                  * Repeat on the address that fired VM_FAULT_RETRY
1315                  * with both FAULT_FLAG_ALLOW_RETRY and
1316                  * FAULT_FLAG_TRIED.  Note that GUP can be interrupted
1317                  * by fatal signals, so we need to check it before we
1318                  * start trying again otherwise it can loop forever.
1319                  */
1320 
1321                 if (fatal_signal_pending(current)) {
1322                         if (!pages_done)
1323                                 pages_done = -EINTR;
1324                         break;
1325                 }
1326 
1327                 ret = mmap_read_lock_killable(mm);
1328                 if (ret) {
1329                         BUG_ON(ret > 0);
1330                         if (!pages_done)
1331                                 pages_done = ret;
1332                         break;
1333                 }
1334 
1335                 *locked = 1;
1336                 ret = __get_user_pages(mm, start, 1, flags | FOLL_TRIED,
1337                                        pages, NULL, locked);
1338                 if (!*locked) {
1339                         /* Continue to retry until we succeeded */
1340                         BUG_ON(ret != 0);
1341                         goto retry;
1342                 }
1343                 if (ret != 1) {
1344                         BUG_ON(ret > 1);
1345                         if (!pages_done)
1346                                 pages_done = ret;
1347                         break;
1348                 }
1349                 nr_pages--;
1350                 pages_done++;
1351                 if (!nr_pages)
1352                         break;
1353                 if (likely(pages))
1354                         pages++;
1355                 start += PAGE_SIZE;
1356         }
1357         if (lock_dropped && *locked) {
1358                 /*
1359                  * We must let the caller know we temporarily dropped the lock
1360                  * and so the critical section protected by it was lost.
1361                  */
1362                 mmap_read_unlock(mm);
1363                 *locked = 0;
1364         }
1365         return pages_done;
1366 }
1367 
1368 /**
1369  * populate_vma_page_range() -  populate a range of pages in the vma.
1370  * @vma:   target vma
1371  * @start: start address
1372  * @end:   end address
1373  * @locked: whether the mmap_lock is still held
1374  *
1375  * This takes care of mlocking the pages too if VM_LOCKED is set.
1376  *
1377  * Return either number of pages pinned in the vma, or a negative error
1378  * code on error.
1379  *
1380  * vma->vm_mm->mmap_lock must be held.
1381  *
1382  * If @locked is NULL, it may be held for read or write and will
1383  * be unperturbed.
1384  *
1385  * If @locked is non-NULL, it must held for read only and may be
1386  * released.  If it's released, *@locked will be set to 0.
1387  */
1388 long populate_vma_page_range(struct vm_area_struct *vma,
1389                 unsigned long start, unsigned long end, int *locked)
1390 {
1391         struct mm_struct *mm = vma->vm_mm;
1392         unsigned long nr_pages = (end - start) / PAGE_SIZE;
1393         int gup_flags;
1394 
1395         VM_BUG_ON(start & ~PAGE_MASK);
1396         VM_BUG_ON(end   & ~PAGE_MASK);
1397         VM_BUG_ON_VMA(start < vma->vm_start, vma);
1398         VM_BUG_ON_VMA(end   > vma->vm_end, vma);
1399         mmap_assert_locked(mm);
1400 
1401         gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK;
1402         if (vma->vm_flags & VM_LOCKONFAULT)
1403                 gup_flags &= ~FOLL_POPULATE;
1404         /*
1405          * We want to touch writable mappings with a write fault in order
1406          * to break COW, except for shared mappings because these don't COW
1407          * and we would not want to dirty them for nothing.
1408          */
1409         if ((vma->vm_flags & (VM_WRITE | VM_SHARED)) == VM_WRITE)
1410                 gup_flags |= FOLL_WRITE;
1411 
1412         /*
1413          * We want mlock to succeed for regions that have any permissions
1414          * other than PROT_NONE.
1415          */
1416         if (vma_is_accessible(vma))
1417                 gup_flags |= FOLL_FORCE;
1418 
1419         /*
1420          * We made sure addr is within a VMA, so the following will
1421          * not result in a stack expansion that recurses back here.
1422          */
1423         return __get_user_pages(mm, start, nr_pages, gup_flags,
1424                                 NULL, NULL, locked);
1425 }
1426 
1427 /*
1428  * __mm_populate - populate and/or mlock pages within a range of address space.
1429  *
1430  * This is used to implement mlock() and the MAP_POPULATE / MAP_LOCKED mmap
1431  * flags. VMAs must be already marked with the desired vm_flags, and
1432  * mmap_lock must not be held.
1433  */
1434 int __mm_populate(unsigned long start, unsigned long len, int ignore_errors)
1435 {
1436         struct mm_struct *mm = current->mm;
1437         unsigned long end, nstart, nend;
1438         struct vm_area_struct *vma = NULL;
1439         int locked = 0;
1440         long ret = 0;
1441 
1442         end = start + len;
1443 
1444         for (nstart = start; nstart < end; nstart = nend) {
1445                 /*
1446                  * We want to fault in pages for [nstart; end) address range.
1447                  * Find first corresponding VMA.
1448                  */
1449                 if (!locked) {
1450                         locked = 1;
1451                         mmap_read_lock(mm);
1452                         vma = find_vma(mm, nstart);
1453                 } else if (nstart >= vma->vm_end)
1454                         vma = vma->vm_next;
1455                 if (!vma || vma->vm_start >= end)
1456                         break;
1457                 /*
1458                  * Set [nstart; nend) to intersection of desired address
1459                  * range with the first VMA. Also, skip undesirable VMA types.
1460                  */
1461                 nend = min(end, vma->vm_end);
1462                 if (vma->vm_flags & (VM_IO | VM_PFNMAP))
1463                         continue;
1464                 if (nstart < vma->vm_start)
1465                         nstart = vma->vm_start;
1466                 /*
1467                  * Now fault in a range of pages. populate_vma_page_range()
1468                  * double checks the vma flags, so that it won't mlock pages
1469                  * if the vma was already munlocked.
1470                  */
1471                 ret = populate_vma_page_range(vma, nstart, nend, &locked);
1472                 if (ret < 0) {
1473                         if (ignore_errors) {
1474                                 ret = 0;
1475                                 continue;       /* continue at next VMA */
1476                         }
1477                         break;
1478                 }
1479                 nend = nstart + ret * PAGE_SIZE;
1480                 ret = 0;
1481         }
1482         if (locked)
1483                 mmap_read_unlock(mm);
1484         return ret;     /* 0 or negative error code */
1485 }
1486 
1487 /**
1488  * get_dump_page() - pin user page in memory while writing it to core dump
1489  * @addr: user address
1490  *
1491  * Returns struct page pointer of user page pinned for dump,
1492  * to be freed afterwards by put_page().
1493  *
1494  * Returns NULL on any kind of failure - a hole must then be inserted into
1495  * the corefile, to preserve alignment with its headers; and also returns
1496  * NULL wherever the ZERO_PAGE, or an anonymous pte_none, has been found -
1497  * allowing a hole to be left in the corefile to save diskspace.
1498  *
1499  * Called without mmap_lock, but after all other threads have been killed.
1500  */
1501 #ifdef CONFIG_ELF_CORE
1502 struct page *get_dump_page(unsigned long addr)
1503 {
1504         struct vm_area_struct *vma;
1505         struct page *page;
1506 
1507         if (__get_user_pages(current->mm, addr, 1,
1508                              FOLL_FORCE | FOLL_DUMP | FOLL_GET, &page, &vma,
1509                              NULL) < 1)
1510                 return NULL;
1511         flush_cache_page(vma, addr, page_to_pfn(page));
1512         return page;
1513 }
1514 #endif /* CONFIG_ELF_CORE */
1515 #else /* CONFIG_MMU */
1516 static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start,
1517                 unsigned long nr_pages, struct page **pages,
1518                 struct vm_area_struct **vmas, int *locked,
1519                 unsigned int foll_flags)
1520 {
1521         struct vm_area_struct *vma;
1522         unsigned long vm_flags;
1523         int i;
1524 
1525         /* calculate required read or write permissions.
1526          * If FOLL_FORCE is set, we only require the "MAY" flags.
1527          */
1528         vm_flags  = (foll_flags & FOLL_WRITE) ?
1529                         (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
1530         vm_flags &= (foll_flags & FOLL_FORCE) ?
1531                         (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
1532 
1533         for (i = 0; i < nr_pages; i++) {
1534                 vma = find_vma(mm, start);
1535                 if (!vma)
1536                         goto finish_or_fault;
1537 
1538                 /* protect what we can, including chardevs */
1539                 if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
1540                     !(vm_flags & vma->vm_flags))
1541                         goto finish_or_fault;
1542 
1543                 if (pages) {
1544                         pages[i] = virt_to_page(start);
1545                         if (pages[i])
1546                                 get_page(pages[i]);
1547                 }
1548                 if (vmas)
1549                         vmas[i] = vma;
1550                 start = (start + PAGE_SIZE) & PAGE_MASK;
1551         }
1552 
1553         return i;
1554 
1555 finish_or_fault:
1556         return i ? : -EFAULT;
1557 }
1558 #endif /* !CONFIG_MMU */
1559 
1560 #if defined(CONFIG_FS_DAX) || defined (CONFIG_CMA)
1561 static bool check_dax_vmas(struct vm_area_struct **vmas, long nr_pages)
1562 {
1563         long i;
1564         struct vm_area_struct *vma_prev = NULL;
1565 
1566         for (i = 0; i < nr_pages; i++) {
1567                 struct vm_area_struct *vma = vmas[i];
1568 
1569                 if (vma == vma_prev)
1570                         continue;
1571 
1572                 vma_prev = vma;
1573 
1574                 if (vma_is_fsdax(vma))
1575                         return true;
1576         }
1577         return false;
1578 }
1579 
1580 #ifdef CONFIG_CMA
1581 static long check_and_migrate_cma_pages(struct mm_struct *mm,
1582                                         unsigned long start,
1583                                         unsigned long nr_pages,
1584                                         struct page **pages,
1585                                         struct vm_area_struct **vmas,
1586                                         unsigned int gup_flags)
1587 {
1588         unsigned long i;
1589         unsigned long step;
1590         bool drain_allow = true;
1591         bool migrate_allow = true;
1592         LIST_HEAD(cma_page_list);
1593         long ret = nr_pages;
1594         struct migration_target_control mtc = {
1595                 .nid = NUMA_NO_NODE,
1596                 .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_NOWARN,
1597         };
1598 
1599 check_again:
1600         for (i = 0; i < nr_pages;) {
1601 
1602                 struct page *head = compound_head(pages[i]);
1603 
1604                 /*
1605                  * gup may start from a tail page. Advance step by the left
1606                  * part.
1607                  */
1608                 step = compound_nr(head) - (pages[i] - head);
1609                 /*
1610                  * If we get a page from the CMA zone, since we are going to
1611                  * be pinning these entries, we might as well move them out
1612                  * of the CMA zone if possible.
1613                  */
1614                 if (is_migrate_cma_page(head)) {
1615                         if (PageHuge(head))
1616                                 isolate_huge_page(head, &cma_page_list);
1617                         else {
1618                                 if (!PageLRU(head) && drain_allow) {
1619                                         lru_add_drain_all();
1620                                         drain_allow = false;
1621                                 }
1622 
1623                                 if (!isolate_lru_page(head)) {
1624                                         list_add_tail(&head->lru, &cma_page_list);
1625                                         mod_node_page_state(page_pgdat(head),
1626                                                             NR_ISOLATED_ANON +
1627                                                             page_is_file_lru(head),
1628                                                             thp_nr_pages(head));
1629                                 }
1630                         }
1631                 }
1632 
1633                 i += step;
1634         }
1635 
1636         if (!list_empty(&cma_page_list)) {
1637                 /*
1638                  * drop the above get_user_pages reference.
1639                  */
1640                 if (gup_flags & FOLL_PIN)
1641                         unpin_user_pages(pages, nr_pages);
1642                 else
1643                         for (i = 0; i < nr_pages; i++)
1644                                 put_page(pages[i]);
1645 
1646                 if (migrate_pages(&cma_page_list, alloc_migration_target, NULL,
1647                         (unsigned long)&mtc, MIGRATE_SYNC, MR_CONTIG_RANGE)) {
1648                         /*
1649                          * some of the pages failed migration. Do get_user_pages
1650                          * without migration.
1651                          */
1652                         migrate_allow = false;
1653 
1654                         if (!list_empty(&cma_page_list))
1655                                 putback_movable_pages(&cma_page_list);
1656                 }
1657                 /*
1658                  * We did migrate all the pages, Try to get the page references
1659                  * again migrating any new CMA pages which we failed to isolate
1660                  * earlier.
1661                  */
1662                 ret = __get_user_pages_locked(mm, start, nr_pages,
1663                                                    pages, vmas, NULL,
1664                                                    gup_flags);
1665 
1666                 if ((ret > 0) && migrate_allow) {
1667                         nr_pages = ret;
1668                         drain_allow = true;
1669                         goto check_again;
1670                 }
1671         }
1672 
1673         return ret;
1674 }
1675 #else
1676 static long check_and_migrate_cma_pages(struct mm_struct *mm,
1677                                         unsigned long start,
1678                                         unsigned long nr_pages,
1679                                         struct page **pages,
1680                                         struct vm_area_struct **vmas,
1681                                         unsigned int gup_flags)
1682 {
1683         return nr_pages;
1684 }
1685 #endif /* CONFIG_CMA */
1686 
1687 /*
1688  * __gup_longterm_locked() is a wrapper for __get_user_pages_locked which
1689  * allows us to process the FOLL_LONGTERM flag.
1690  */
1691 static long __gup_longterm_locked(struct mm_struct *mm,
1692                                   unsigned long start,
1693                                   unsigned long nr_pages,
1694                                   struct page **pages,
1695                                   struct vm_area_struct **vmas,
1696                                   unsigned int gup_flags)
1697 {
1698         struct vm_area_struct **vmas_tmp = vmas;
1699         unsigned long flags = 0;
1700         long rc, i;
1701 
1702         if (gup_flags & FOLL_LONGTERM) {
1703                 if (!pages)
1704                         return -EINVAL;
1705 
1706                 if (!vmas_tmp) {
1707                         vmas_tmp = kcalloc(nr_pages,
1708                                            sizeof(struct vm_area_struct *),
1709                                            GFP_KERNEL);
1710                         if (!vmas_tmp)
1711                                 return -ENOMEM;
1712                 }
1713                 flags = memalloc_nocma_save();
1714         }
1715 
1716         rc = __get_user_pages_locked(mm, start, nr_pages, pages,
1717                                      vmas_tmp, NULL, gup_flags);
1718 
1719         if (gup_flags & FOLL_LONGTERM) {
1720                 if (rc < 0)
1721                         goto out;
1722 
1723                 if (check_dax_vmas(vmas_tmp, rc)) {
1724                         if (gup_flags & FOLL_PIN)
1725                                 unpin_user_pages(pages, rc);
1726                         else
1727                                 for (i = 0; i < rc; i++)
1728                                         put_page(pages[i]);
1729                         rc = -EOPNOTSUPP;
1730                         goto out;
1731                 }
1732 
1733                 rc = check_and_migrate_cma_pages(mm, start, rc, pages,
1734                                                  vmas_tmp, gup_flags);
1735 out:
1736                 memalloc_nocma_restore(flags);
1737         }
1738 
1739         if (vmas_tmp != vmas)
1740                 kfree(vmas_tmp);
1741         return rc;
1742 }
1743 #else /* !CONFIG_FS_DAX && !CONFIG_CMA */
1744 static __always_inline long __gup_longterm_locked(struct mm_struct *mm,
1745                                                   unsigned long start,
1746                                                   unsigned long nr_pages,
1747                                                   struct page **pages,
1748                                                   struct vm_area_struct **vmas,
1749                                                   unsigned int flags)
1750 {
1751         return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
1752                                        NULL, flags);
1753 }
1754 #endif /* CONFIG_FS_DAX || CONFIG_CMA */
1755 
1756 #ifdef CONFIG_MMU
1757 static long __get_user_pages_remote(struct mm_struct *mm,
1758                                     unsigned long start, unsigned long nr_pages,
1759                                     unsigned int gup_flags, struct page **pages,
1760                                     struct vm_area_struct **vmas, int *locked)
1761 {
1762         /*
1763          * Parts of FOLL_LONGTERM behavior are incompatible with
1764          * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
1765          * vmas. However, this only comes up if locked is set, and there are
1766          * callers that do request FOLL_LONGTERM, but do not set locked. So,
1767          * allow what we can.
1768          */
1769         if (gup_flags & FOLL_LONGTERM) {
1770                 if (WARN_ON_ONCE(locked))
1771                         return -EINVAL;
1772                 /*
1773                  * This will check the vmas (even if our vmas arg is NULL)
1774                  * and return -ENOTSUPP if DAX isn't allowed in this case:
1775                  */
1776                 return __gup_longterm_locked(mm, start, nr_pages, pages,
1777                                              vmas, gup_flags | FOLL_TOUCH |
1778                                              FOLL_REMOTE);
1779         }
1780 
1781         return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
1782                                        locked,
1783                                        gup_flags | FOLL_TOUCH | FOLL_REMOTE);
1784 }
1785 
1786 /**
1787  * get_user_pages_remote() - pin user pages in memory
1788  * @mm:         mm_struct of target mm
1789  * @start:      starting user address
1790  * @nr_pages:   number of pages from start to pin
1791  * @gup_flags:  flags modifying lookup behaviour
1792  * @pages:      array that receives pointers to the pages pinned.
1793  *              Should be at least nr_pages long. Or NULL, if caller
1794  *              only intends to ensure the pages are faulted in.
1795  * @vmas:       array of pointers to vmas corresponding to each page.
1796  *              Or NULL if the caller does not require them.
1797  * @locked:     pointer to lock flag indicating whether lock is held and
1798  *              subsequently whether VM_FAULT_RETRY functionality can be
1799  *              utilised. Lock must initially be held.
1800  *
1801  * Returns either number of pages pinned (which may be less than the
1802  * number requested), or an error. Details about the return value:
1803  *
1804  * -- If nr_pages is 0, returns 0.
1805  * -- If nr_pages is >0, but no pages were pinned, returns -errno.
1806  * -- If nr_pages is >0, and some pages were pinned, returns the number of
1807  *    pages pinned. Again, this may be less than nr_pages.
1808  *
1809  * The caller is responsible for releasing returned @pages, via put_page().
1810  *
1811  * @vmas are valid only as long as mmap_lock is held.
1812  *
1813  * Must be called with mmap_lock held for read or write.
1814  *
1815  * get_user_pages_remote walks a process's page tables and takes a reference
1816  * to each struct page that each user address corresponds to at a given
1817  * instant. That is, it takes the page that would be accessed if a user
1818  * thread accesses the given user virtual address at that instant.
1819  *
1820  * This does not guarantee that the page exists in the user mappings when
1821  * get_user_pages_remote returns, and there may even be a completely different
1822  * page there in some cases (eg. if mmapped pagecache has been invalidated
1823  * and subsequently re faulted). However it does guarantee that the page
1824  * won't be freed completely. And mostly callers simply care that the page
1825  * contains data that was valid *at some point in time*. Typically, an IO
1826  * or similar operation cannot guarantee anything stronger anyway because
1827  * locks can't be held over the syscall boundary.
1828  *
1829  * If gup_flags & FOLL_WRITE == 0, the page must not be written to. If the page
1830  * is written to, set_page_dirty (or set_page_dirty_lock, as appropriate) must
1831  * be called after the page is finished with, and before put_page is called.
1832  *
1833  * get_user_pages_remote is typically used for fewer-copy IO operations,
1834  * to get a handle on the memory by some means other than accesses
1835  * via the user virtual addresses. The pages may be submitted for
1836  * DMA to devices or accessed via their kernel linear mapping (via the
1837  * kmap APIs). Care should be taken to use the correct cache flushing APIs.
1838  *
1839  * See also get_user_pages_fast, for performance critical applications.
1840  *
1841  * get_user_pages_remote should be phased out in favor of
1842  * get_user_pages_locked|unlocked or get_user_pages_fast. Nothing
1843  * should use get_user_pages_remote because it cannot pass
1844  * FAULT_FLAG_ALLOW_RETRY to handle_mm_fault.
1845  */
1846 long get_user_pages_remote(struct mm_struct *mm,
1847                 unsigned long start, unsigned long nr_pages,
1848                 unsigned int gup_flags, struct page **pages,
1849                 struct vm_area_struct **vmas, int *locked)
1850 {
1851         /*
1852          * FOLL_PIN must only be set internally by the pin_user_pages*() APIs,
1853          * never directly by the caller, so enforce that with an assertion:
1854          */
1855         if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
1856                 return -EINVAL;
1857 
1858         return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
1859                                        pages, vmas, locked);
1860 }
1861 EXPORT_SYMBOL(get_user_pages_remote);
1862 
1863 #else /* CONFIG_MMU */
1864 long get_user_pages_remote(struct mm_struct *mm,
1865                            unsigned long start, unsigned long nr_pages,
1866                            unsigned int gup_flags, struct page **pages,
1867                            struct vm_area_struct **vmas, int *locked)
1868 {
1869         return 0;
1870 }
1871 
1872 static long __get_user_pages_remote(struct mm_struct *mm,
1873                                     unsigned long start, unsigned long nr_pages,
1874                                     unsigned int gup_flags, struct page **pages,
1875                                     struct vm_area_struct **vmas, int *locked)
1876 {
1877         return 0;
1878 }
1879 #endif /* !CONFIG_MMU */
1880 
1881 /**
1882  * get_user_pages() - pin user pages in memory
1883  * @start:      starting user address
1884  * @nr_pages:   number of pages from start to pin
1885  * @gup_flags:  flags modifying lookup behaviour
1886  * @pages:      array that receives pointers to the pages pinned.
1887  *              Should be at least nr_pages long. Or NULL, if caller
1888  *              only intends to ensure the pages are faulted in.
1889  * @vmas:       array of pointers to vmas corresponding to each page.
1890  *              Or NULL if the caller does not require them.
1891  *
1892  * This is the same as get_user_pages_remote(), just with a less-flexible
1893  * calling convention where we assume that the mm being operated on belongs to
1894  * the current task, and doesn't allow passing of a locked parameter.  We also
1895  * obviously don't pass FOLL_REMOTE in here.
1896  */
1897 long get_user_pages(unsigned long start, unsigned long nr_pages,
1898                 unsigned int gup_flags, struct page **pages,
1899                 struct vm_area_struct **vmas)
1900 {
1901         /*
1902          * FOLL_PIN must only be set internally by the pin_user_pages*() APIs,
1903          * never directly by the caller, so enforce that with an assertion:
1904          */
1905         if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
1906                 return -EINVAL;
1907 
1908         return __gup_longterm_locked(current->mm, start, nr_pages,
1909                                      pages, vmas, gup_flags | FOLL_TOUCH);
1910 }
1911 EXPORT_SYMBOL(get_user_pages);
1912 
1913 /**
1914  * get_user_pages_locked() is suitable to replace the form:
1915  *
1916  *      mmap_read_lock(mm);
1917  *      do_something()
1918  *      get_user_pages(mm, ..., pages, NULL);
1919  *      mmap_read_unlock(mm);
1920  *
1921  *  to:
1922  *
1923  *      int locked = 1;
1924  *      mmap_read_lock(mm);
1925  *      do_something()
1926  *      get_user_pages_locked(mm, ..., pages, &locked);
1927  *      if (locked)
1928  *          mmap_read_unlock(mm);
1929  *
1930  * @start:      starting user address
1931  * @nr_pages:   number of pages from start to pin
1932  * @gup_flags:  flags modifying lookup behaviour
1933  * @pages:      array that receives pointers to the pages pinned.
1934  *              Should be at least nr_pages long. Or NULL, if caller
1935  *              only intends to ensure the pages are faulted in.
1936  * @locked:     pointer to lock flag indicating whether lock is held and
1937  *              subsequently whether VM_FAULT_RETRY functionality can be
1938  *              utilised. Lock must initially be held.
1939  *
1940  * We can leverage the VM_FAULT_RETRY functionality in the page fault
1941  * paths better by using either get_user_pages_locked() or
1942  * get_user_pages_unlocked().
1943  *
1944  */
1945 long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
1946                            unsigned int gup_flags, struct page **pages,
1947                            int *locked)
1948 {
1949         /*
1950          * FIXME: Current FOLL_LONGTERM behavior is incompatible with
1951          * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
1952          * vmas.  As there are no users of this flag in this call we simply
1953          * disallow this option for now.
1954          */
1955         if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
1956                 return -EINVAL;
1957         /*
1958          * FOLL_PIN must only be set internally by the pin_user_pages*() APIs,
1959          * never directly by the caller, so enforce that:
1960          */
1961         if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
1962                 return -EINVAL;
1963 
1964         return __get_user_pages_locked(current->mm, start, nr_pages,
1965                                        pages, NULL, locked,
1966                                        gup_flags | FOLL_TOUCH);
1967 }
1968 EXPORT_SYMBOL(get_user_pages_locked);
1969 
1970 /*
1971  * get_user_pages_unlocked() is suitable to replace the form:
1972  *
1973  *      mmap_read_lock(mm);
1974  *      get_user_pages(mm, ..., pages, NULL);
1975  *      mmap_read_unlock(mm);
1976  *
1977  *  with:
1978  *
1979  *      get_user_pages_unlocked(mm, ..., pages);
1980  *
1981  * It is functionally equivalent to get_user_pages_fast so
1982  * get_user_pages_fast should be used instead if specific gup_flags
1983  * (e.g. FOLL_FORCE) are not required.
1984  */
1985 long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
1986                              struct page **pages, unsigned int gup_flags)
1987 {
1988         struct mm_struct *mm = current->mm;
1989         int locked = 1;
1990         long ret;
1991 
1992         /*
1993          * FIXME: Current FOLL_LONGTERM behavior is incompatible with
1994          * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
1995          * vmas.  As there are no users of this flag in this call we simply
1996          * disallow this option for now.
1997          */
1998         if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
1999                 return -EINVAL;
2000 
2001         mmap_read_lock(mm);
2002         ret = __get_user_pages_locked(mm, start, nr_pages, pages, NULL,
2003                                       &locked, gup_flags | FOLL_TOUCH);
2004         if (locked)
2005                 mmap_read_unlock(mm);
2006         return ret;
2007 }
2008 EXPORT_SYMBOL(get_user_pages_unlocked);
2009 
2010 /*
2011  * Fast GUP
2012  *
2013  * get_user_pages_fast attempts to pin user pages by walking the page
2014  * tables directly and avoids taking locks. Thus the walker needs to be
2015  * protected from page table pages being freed from under it, and should
2016  * block any THP splits.
2017  *
2018  * One way to achieve this is to have the walker disable interrupts, and
2019  * rely on IPIs from the TLB flushing code blocking before the page table
2020  * pages are freed. This is unsuitable for architectures that do not need
2021  * to broadcast an IPI when invalidating TLBs.
2022  *
2023  * Another way to achieve this is to batch up page table containing pages
2024  * belonging to more than one mm_user, then rcu_sched a callback to free those
2025  * pages. Disabling interrupts will allow the fast_gup walker to both block
2026  * the rcu_sched callback, and an IPI that we broadcast for splitting THPs
2027  * (which is a relatively rare event). The code below adopts this strategy.
2028  *
2029  * Before activating this code, please be aware that the following assumptions
2030  * are currently made:
2031  *
2032  *  *) Either MMU_GATHER_RCU_TABLE_FREE is enabled, and tlb_remove_table() is used to
2033  *  free pages containing page tables or TLB flushing requires IPI broadcast.
2034  *
2035  *  *) ptes can be read atomically by the architecture.
2036  *
2037  *  *) access_ok is sufficient to validate userspace address ranges.
2038  *
2039  * The last two assumptions can be relaxed by the addition of helper functions.
2040  *
2041  * This code is based heavily on the PowerPC implementation by Nick Piggin.
2042  */
2043 #ifdef CONFIG_HAVE_FAST_GUP
2044 
2045 static void put_compound_head(struct page *page, int refs, unsigned int flags)
2046 {
2047         if (flags & FOLL_PIN) {
2048                 mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_RELEASED,
2049                                     refs);
2050 
2051                 if (hpage_pincount_available(page))
2052                         hpage_pincount_sub(page, refs);
2053                 else
2054                         refs *= GUP_PIN_COUNTING_BIAS;
2055         }
2056 
2057         VM_BUG_ON_PAGE(page_ref_count(page) < refs, page);
2058         /*
2059          * Calling put_page() for each ref is unnecessarily slow. Only the last
2060          * ref needs a put_page().
2061          */
2062         if (refs > 1)
2063                 page_ref_sub(page, refs - 1);
2064         put_page(page);
2065 }
2066 
2067 #ifdef CONFIG_GUP_GET_PTE_LOW_HIGH
2068 
2069 /*
2070  * WARNING: only to be used in the get_user_pages_fast() implementation.
2071  *
2072  * With get_user_pages_fast(), we walk down the pagetables without taking any
2073  * locks.  For this we would like to load the pointers atomically, but sometimes
2074  * that is not possible (e.g. without expensive cmpxchg8b on x86_32 PAE).  What
2075  * we do have is the guarantee that a PTE will only either go from not present
2076  * to present, or present to not present or both -- it will not switch to a
2077  * completely different present page without a TLB flush in between; something
2078  * that we are blocking by holding interrupts off.
2079  *
2080  * Setting ptes from not present to present goes:
2081  *
2082  *   ptep->pte_high = h;
2083  *   smp_wmb();
2084  *   ptep->pte_low = l;
2085  *
2086  * And present to not present goes:
2087  *
2088  *   ptep->pte_low = 0;
2089  *   smp_wmb();
2090  *   ptep->pte_high = 0;
2091  *
2092  * We must ensure here that the load of pte_low sees 'l' IFF pte_high sees 'h'.
2093  * We load pte_high *after* loading pte_low, which ensures we don't see an older
2094  * value of pte_high.  *Then* we recheck pte_low, which ensures that we haven't
2095  * picked up a changed pte high. We might have gotten rubbish values from
2096  * pte_low and pte_high, but we are guaranteed that pte_low will not have the
2097  * present bit set *unless* it is 'l'. Because get_user_pages_fast() only
2098  * operates on present ptes we're safe.
2099  */
2100 static inline pte_t gup_get_pte(pte_t *ptep)
2101 {
2102         pte_t pte;
2103 
2104         do {
2105                 pte.pte_low = ptep->pte_low;
2106                 smp_rmb();
2107                 pte.pte_high = ptep->pte_high;
2108                 smp_rmb();
2109         } while (unlikely(pte.pte_low != ptep->pte_low));
2110 
2111         return pte;
2112 }
2113 #else /* CONFIG_GUP_GET_PTE_LOW_HIGH */
2114 /*
2115  * We require that the PTE can be read atomically.
2116  */
2117 static inline pte_t gup_get_pte(pte_t *ptep)
2118 {
2119         return ptep_get(ptep);
2120 }
2121 #endif /* CONFIG_GUP_GET_PTE_LOW_HIGH */
2122 
2123 static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start,
2124                                             unsigned int flags,
2125                                             struct page **pages)
2126 {
2127         while ((*nr) - nr_start) {
2128                 struct page *page = pages[--(*nr)];
2129 
2130                 ClearPageReferenced(page);
2131                 if (flags & FOLL_PIN)
2132                         unpin_user_page(page);
2133                 else
2134                         put_page(page);
2135         }
2136 }
2137 
2138 #ifdef CONFIG_ARCH_HAS_PTE_SPECIAL
2139 static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
2140                          unsigned int flags, struct page **pages, int *nr)
2141 {
2142         struct dev_pagemap *pgmap = NULL;
2143         int nr_start = *nr, ret = 0;
2144         pte_t *ptep, *ptem;
2145 
2146         ptem = ptep = pte_offset_map(&pmd, addr);
2147         do {
2148                 pte_t pte = gup_get_pte(ptep);
2149                 struct page *head, *page;
2150 
2151                 /*
2152                  * Similar to the PMD case below, NUMA hinting must take slow
2153                  * path using the pte_protnone check.
2154                  */
2155                 if (pte_protnone(pte))
2156                         goto pte_unmap;
2157 
2158                 if (!pte_access_permitted(pte, flags & FOLL_WRITE))
2159                         goto pte_unmap;
2160 
2161                 if (pte_devmap(pte)) {
2162                         if (unlikely(flags & FOLL_LONGTERM))
2163                                 goto pte_unmap;
2164 
2165                         pgmap = get_dev_pagemap(pte_pfn(pte), pgmap);
2166                         if (unlikely(!pgmap)) {
2167                                 undo_dev_pagemap(nr, nr_start, flags, pages);
2168                                 goto pte_unmap;
2169                         }
2170                 } else if (pte_special(pte))
2171                         goto pte_unmap;
2172 
2173                 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
2174                 page = pte_page(pte);
2175 
2176                 head = try_grab_compound_head(page, 1, flags);
2177                 if (!head)
2178                         goto pte_unmap;
2179 
2180                 if (unlikely(pte_val(pte) != pte_val(*ptep))) {
2181                         put_compound_head(head, 1, flags);
2182                         goto pte_unmap;
2183                 }
2184 
2185                 VM_BUG_ON_PAGE(compound_head(page) != head, page);
2186 
2187                 /*
2188                  * We need to make the page accessible if and only if we are
2189                  * going to access its content (the FOLL_PIN case).  Please
2190                  * see Documentation/core-api/pin_user_pages.rst for
2191                  * details.
2192                  */
2193                 if (flags & FOLL_PIN) {
2194                         ret = arch_make_page_accessible(page);
2195                         if (ret) {
2196                                 unpin_user_page(page);
2197                                 goto pte_unmap;
2198                         }
2199                 }
2200                 SetPageReferenced(page);
2201                 pages[*nr] = page;
2202                 (*nr)++;
2203 
2204         } while (ptep++, addr += PAGE_SIZE, addr != end);
2205 
2206         ret = 1;
2207 
2208 pte_unmap:
2209         if (pgmap)
2210                 put_dev_pagemap(pgmap);
2211         pte_unmap(ptem);
2212         return ret;
2213 }
2214 #else
2215 
2216 /*
2217  * If we can't determine whether or not a pte is special, then fail immediately
2218  * for ptes. Note, we can still pin HugeTLB and THP as these are guaranteed not
2219  * to be special.
2220  *
2221  * For a futex to be placed on a THP tail page, get_futex_key requires a
2222  * get_user_pages_fast_only implementation that can pin pages. Thus it's still
2223  * useful to have gup_huge_pmd even if we can't operate on ptes.
2224  */
2225 static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
2226                          unsigned int flags, struct page **pages, int *nr)
2227 {
2228         return 0;
2229 }
2230 #endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */
2231 
2232 #if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
2233 static int __gup_device_huge(unsigned long pfn, unsigned long addr,
2234                              unsigned long end, unsigned int flags,
2235                              struct page **pages, int *nr)
2236 {
2237         int nr_start = *nr;
2238         struct dev_pagemap *pgmap = NULL;
2239 
2240         do {
2241                 struct page *page = pfn_to_page(pfn);
2242 
2243                 pgmap = get_dev_pagemap(pfn, pgmap);
2244                 if (unlikely(!pgmap)) {
2245                         undo_dev_pagemap(nr, nr_start, flags, pages);
2246                         return 0;
2247                 }
2248                 SetPageReferenced(page);
2249                 pages[*nr] = page;
2250                 if (unlikely(!try_grab_page(page, flags))) {
2251                         undo_dev_pagemap(nr, nr_start, flags, pages);
2252                         return 0;
2253                 }
2254                 (*nr)++;
2255                 pfn++;
2256         } while (addr += PAGE_SIZE, addr != end);
2257 
2258         if (pgmap)
2259                 put_dev_pagemap(pgmap);
2260         return 1;
2261 }
2262 
2263 static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
2264                                  unsigned long end, unsigned int flags,
2265                                  struct page **pages, int *nr)
2266 {
2267         unsigned long fault_pfn;
2268         int nr_start = *nr;
2269 
2270         fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
2271         if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
2272                 return 0;
2273 
2274         if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
2275                 undo_dev_pagemap(nr, nr_start, flags, pages);
2276                 return 0;
2277         }
2278         return 1;
2279 }
2280 
2281 static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
2282                                  unsigned long end, unsigned int flags,
2283                                  struct page **pages, int *nr)
2284 {
2285         unsigned long fault_pfn;
2286         int nr_start = *nr;
2287 
2288         fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
2289         if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
2290                 return 0;
2291 
2292         if (unlikely(pud_val(orig) != pud_val(*pudp))) {
2293                 undo_dev_pagemap(nr, nr_start, flags, pages);
2294                 return 0;
2295         }
2296         return 1;
2297 }
2298 #else
2299 static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
2300                                  unsigned long end, unsigned int flags,
2301                                  struct page **pages, int *nr)
2302 {
2303         BUILD_BUG();
2304         return 0;
2305 }
2306 
2307 static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,
2308                                  unsigned long end, unsigned int flags,
2309                                  struct page **pages, int *nr)
2310 {
2311         BUILD_BUG();
2312         return 0;
2313 }
2314 #endif
2315 
2316 static int record_subpages(struct page *page, unsigned long addr,
2317                            unsigned long end, struct page **pages)
2318 {
2319         int nr;
2320 
2321         for (nr = 0; addr != end; addr += PAGE_SIZE)
2322                 pages[nr++] = page++;
2323 
2324         return nr;
2325 }
2326 
2327 #ifdef CONFIG_ARCH_HAS_HUGEPD
2328 static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
2329                                       unsigned long sz)
2330 {
2331         unsigned long __boundary = (addr + sz) & ~(sz-1);
2332         return (__boundary - 1 < end - 1) ? __boundary : end;
2333 }
2334 
2335 static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
2336                        unsigned long end, unsigned int flags,
2337                        struct page **pages, int *nr)
2338 {
2339         unsigned long pte_end;
2340         struct page *head, *page;
2341         pte_t pte;
2342         int refs;
2343 
2344         pte_end = (addr + sz) & ~(sz-1);
2345         if (pte_end < end)
2346                 end = pte_end;
2347 
2348         pte = huge_ptep_get(ptep);
2349 
2350         if (!pte_access_permitted(pte, flags & FOLL_WRITE))
2351                 return 0;
2352 
2353         /* hugepages are never "special" */
2354         VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
2355 
2356         head = pte_page(pte);
2357         page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
2358         refs = record_subpages(page, addr, end, pages + *nr);
2359 
2360         head = try_grab_compound_head(head, refs, flags);
2361         if (!head)
2362                 return 0;
2363 
2364         if (unlikely(pte_val(pte) != pte_val(*ptep))) {
2365                 put_compound_head(head, refs, flags);
2366                 return 0;
2367         }
2368 
2369         *nr += refs;
2370         SetPageReferenced(head);
2371         return 1;
2372 }
2373 
2374 static int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
2375                 unsigned int pdshift, unsigned long end, unsigned int flags,
2376                 struct page **pages, int *nr)
2377 {
2378         pte_t *ptep;
2379         unsigned long sz = 1UL << hugepd_shift(hugepd);
2380         unsigned long next;
2381 
2382         ptep = hugepte_offset(hugepd, addr, pdshift);
2383         do {
2384                 next = hugepte_addr_end(addr, end, sz);
2385                 if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr))
2386                         return 0;
2387         } while (ptep++, addr = next, addr != end);
2388 
2389         return 1;
2390 }
2391 #else
2392 static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
2393                 unsigned int pdshift, unsigned long end, unsigned int flags,
2394                 struct page **pages, int *nr)
2395 {
2396         return 0;
2397 }
2398 #endif /* CONFIG_ARCH_HAS_HUGEPD */
2399 
2400 static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
2401                         unsigned long end, unsigned int flags,
2402                         struct page **pages, int *nr)
2403 {
2404         struct page *head, *page;
2405         int refs;
2406 
2407         if (!pmd_access_permitted(orig, flags & FOLL_WRITE))
2408                 return 0;
2409 
2410         if (pmd_devmap(orig)) {
2411                 if (unlikely(flags & FOLL_LONGTERM))
2412                         return 0;
2413                 return __gup_device_huge_pmd(orig, pmdp, addr, end, flags,
2414                                              pages, nr);
2415         }
2416 
2417         page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
2418         refs = record_subpages(page, addr, end, pages + *nr);
2419 
2420         head = try_grab_compound_head(pmd_page(orig), refs, flags);
2421         if (!head)
2422                 return 0;
2423 
2424         if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
2425                 put_compound_head(head, refs, flags);
2426                 return 0;
2427         }
2428 
2429         *nr += refs;
2430         SetPageReferenced(head);
2431         return 1;
2432 }
2433 
2434 static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
2435                         unsigned long end, unsigned int flags,
2436                         struct page **pages, int *nr)
2437 {
2438         struct page *head, *page;
2439         int refs;
2440 
2441         if (!pud_access_permitted(orig, flags & FOLL_WRITE))
2442                 return 0;
2443 
2444         if (pud_devmap(orig)) {
2445                 if (unlikely(flags & FOLL_LONGTERM))
2446                         return 0;
2447                 return __gup_device_huge_pud(orig, pudp, addr, end, flags,
2448                                              pages, nr);
2449         }
2450 
2451         page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
2452         refs = record_subpages(page, addr, end, pages + *nr);
2453 
2454         head = try_grab_compound_head(pud_page(orig), refs, flags);
2455         if (!head)
2456                 return 0;
2457 
2458         if (unlikely(pud_val(orig) != pud_val(*pudp))) {
2459                 put_compound_head(head, refs, flags);
2460                 return 0;
2461         }
2462 
2463         *nr += refs;
2464         SetPageReferenced(head);
2465         return 1;
2466 }
2467 
2468 static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
2469                         unsigned long end, unsigned int flags,
2470                         struct page **pages, int *nr)
2471 {
2472         int refs;
2473         struct page *head, *page;
2474 
2475         if (!pgd_access_permitted(orig, flags & FOLL_WRITE))
2476                 return 0;
2477 
2478         BUILD_BUG_ON(pgd_devmap(orig));
2479 
2480         page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);
2481         refs = record_subpages(page, addr, end, pages + *nr);
2482 
2483         head = try_grab_compound_head(pgd_page(orig), refs, flags);
2484         if (!head)
2485                 return 0;
2486 
2487         if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) {
2488                 put_compound_head(head, refs, flags);
2489                 return 0;
2490         }
2491 
2492         *nr += refs;
2493         SetPageReferenced(head);
2494         return 1;
2495 }
2496 
2497 static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned long end,
2498                 unsigned int flags, struct page **pages, int *nr)
2499 {
2500         unsigned long next;
2501         pmd_t *pmdp;
2502 
2503         pmdp = pmd_offset_lockless(pudp, pud, addr);
2504         do {
2505                 pmd_t pmd = READ_ONCE(*pmdp);
2506 
2507                 next = pmd_addr_end(addr, end);
2508                 if (!pmd_present(pmd))
2509                         return 0;
2510 
2511                 if (unlikely(pmd_trans_huge(pmd) || pmd_huge(pmd) ||
2512                              pmd_devmap(pmd))) {
2513                         /*
2514                          * NUMA hinting faults need to be handled in the GUP
2515                          * slowpath for accounting purposes and so that they
2516                          * can be serialised against THP migration.
2517                          */
2518                         if (pmd_protnone(pmd))
2519                                 return 0;
2520 
2521                         if (!gup_huge_pmd(pmd, pmdp, addr, next, flags,
2522                                 pages, nr))
2523                                 return 0;
2524 
2525                 } else if (unlikely(is_hugepd(__hugepd(pmd_val(pmd))))) {
2526                         /*
2527                          * architecture have different format for hugetlbfs
2528                          * pmd format and THP pmd format
2529                          */
2530                         if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
2531                                          PMD_SHIFT, next, flags, pages, nr))
2532                                 return 0;
2533                 } else if (!gup_pte_range(pmd, addr, next, flags, pages, nr))
2534                         return 0;
2535         } while (pmdp++, addr = next, addr != end);
2536 
2537         return 1;
2538 }
2539 
2540 static int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, unsigned long end,
2541                          unsigned int flags, struct page **pages, int *nr)
2542 {
2543         unsigned long next;
2544         pud_t *pudp;
2545 
2546         pudp = pud_offset_lockless(p4dp, p4d, addr);
2547         do {
2548                 pud_t pud = READ_ONCE(*pudp);
2549 
2550                 next = pud_addr_end(addr, end);
2551                 if (unlikely(!pud_present(pud)))
2552                         return 0;
2553                 if (unlikely(pud_huge(pud))) {
2554                         if (!gup_huge_pud(pud, pudp, addr, next, flags,
2555                                           pages, nr))
2556                                 return 0;
2557                 } else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) {
2558                         if (!gup_huge_pd(__hugepd(pud_val(pud)), addr,
2559                                          PUD_SHIFT, next, flags, pages, nr))
2560                                 return 0;
2561                 } else if (!gup_pmd_range(pudp, pud, addr, next, flags, pages, nr))
2562                         return 0;
2563         } while (pudp++, addr = next, addr != end);
2564 
2565         return 1;
2566 }
2567 
2568 static int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned long end,
2569                          unsigned int flags, struct page **pages, int *nr)
2570 {
2571         unsigned long next;
2572         p4d_t *p4dp;
2573 
2574         p4dp = p4d_offset_lockless(pgdp, pgd, addr);
2575         do {
2576                 p4d_t p4d = READ_ONCE(*p4dp);
2577 
2578                 next = p4d_addr_end(addr, end);
2579                 if (p4d_none(p4d))
2580                         return 0;
2581                 BUILD_BUG_ON(p4d_huge(p4d));
2582                 if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) {
2583                         if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,
2584                                          P4D_SHIFT, next, flags, pages, nr))
2585                                 return 0;
2586                 } else if (!gup_pud_range(p4dp, p4d, addr, next, flags, pages, nr))
2587                         return 0;
2588         } while (p4dp++, addr = next, addr != end);
2589 
2590         return 1;
2591 }
2592 
2593 static void gup_pgd_range(unsigned long addr, unsigned long end,
2594                 unsigned int flags, struct page **pages, int *nr)
2595 {
2596         unsigned long next;
2597         pgd_t *pgdp;
2598 
2599         pgdp = pgd_offset(current->mm, addr);
2600         do {
2601                 pgd_t pgd = READ_ONCE(*pgdp);
2602 
2603                 next = pgd_addr_end(addr, end);
2604                 if (pgd_none(pgd))
2605                         return;
2606                 if (unlikely(pgd_huge(pgd))) {
2607                         if (!gup_huge_pgd(pgd, pgdp, addr, next, flags,
2608                                           pages, nr))
2609                                 return;
2610                 } else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) {
2611                         if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
2612                                          PGDIR_SHIFT, next, flags, pages, nr))
2613                                 return;
2614                 } else if (!gup_p4d_range(pgdp, pgd, addr, next, flags, pages, nr))
2615                         return;
2616         } while (pgdp++, addr = next, addr != end);
2617 }
2618 #else
2619 static inline void gup_pgd_range(unsigned long addr, unsigned long end,
2620                 unsigned int flags, struct page **pages, int *nr)
2621 {
2622 }
2623 #endif /* CONFIG_HAVE_FAST_GUP */
2624 
2625 #ifndef gup_fast_permitted
2626 /*
2627  * Check if it's allowed to use get_user_pages_fast_only() for the range, or
2628  * we need to fall back to the slow version:
2629  */
2630 static bool gup_fast_permitted(unsigned long start, unsigned long end)
2631 {
2632         return true;
2633 }
2634 #endif
2635 
2636 static int __gup_longterm_unlocked(unsigned long start, int nr_pages,
2637                                    unsigned int gup_flags, struct page **pages)
2638 {
2639         int ret;
2640 
2641         /*
2642          * FIXME: FOLL_LONGTERM does not work with
2643          * get_user_pages_unlocked() (see comments in that function)
2644          */
2645         if (gup_flags & FOLL_LONGTERM) {
2646                 mmap_read_lock(current->mm);
2647                 ret = __gup_longterm_locked(current->mm,
2648                                             start, nr_pages,
2649                                             pages, NULL, gup_flags);
2650                 mmap_read_unlock(current->mm);
2651         } else {
2652                 ret = get_user_pages_unlocked(start, nr_pages,
2653                                               pages, gup_flags);
2654         }
2655 
2656         return ret;
2657 }
2658 
2659 static int internal_get_user_pages_fast(unsigned long start, int nr_pages,
2660                                         unsigned int gup_flags,
2661                                         struct page **pages)
2662 {
2663         unsigned long addr, len, end;
2664         unsigned long flags;
2665         int nr_pinned = 0, ret = 0;
2666 
2667         if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM |
2668                                        FOLL_FORCE | FOLL_PIN | FOLL_GET |
2669                                        FOLL_FAST_ONLY)))
2670                 return -EINVAL;
2671 
2672         if (gup_flags & FOLL_PIN)
2673                 atomic_set(&current->mm->has_pinned, 1);
2674 
2675         if (!(gup_flags & FOLL_FAST_ONLY))
2676                 might_lock_read(&current->mm->mmap_lock);
2677 
2678         start = untagged_addr(start) & PAGE_MASK;
2679         addr = start;
2680         len = (unsigned long) nr_pages << PAGE_SHIFT;
2681         end = start + len;
2682 
2683         if (end <= start)
2684                 return 0;
2685         if (unlikely(!access_ok((void __user *)start, len)))
2686                 return -EFAULT;
2687 
2688         /*
2689          * Disable interrupts. The nested form is used, in order to allow
2690          * full, general purpose use of this routine.
2691          *
2692          * With interrupts disabled, we block page table pages from being
2693          * freed from under us. See struct mmu_table_batch comments in
2694          * include/asm-generic/tlb.h for more details.
2695          *
2696          * We do not adopt an rcu_read_lock(.) here as we also want to
2697          * block IPIs that come from THPs splitting.
2698          */
2699         if (IS_ENABLED(CONFIG_HAVE_FAST_GUP) && gup_fast_permitted(start, end)) {
2700                 unsigned long fast_flags = gup_flags;
2701 
2702                 local_irq_save(flags);
2703                 gup_pgd_range(addr, end, fast_flags, pages, &nr_pinned);
2704                 local_irq_restore(flags);
2705                 ret = nr_pinned;
2706         }
2707 
2708         if (nr_pinned < nr_pages && !(gup_flags & FOLL_FAST_ONLY)) {
2709                 /* Try to get the remaining pages with get_user_pages */
2710                 start += nr_pinned << PAGE_SHIFT;
2711                 pages += nr_pinned;
2712 
2713                 ret = __gup_longterm_unlocked(start, nr_pages - nr_pinned,
2714                                               gup_flags, pages);
2715 
2716                 /* Have to be a bit careful with return values */
2717                 if (nr_pinned > 0) {
2718                         if (ret < 0)
2719                                 ret = nr_pinned;
2720                         else
2721                                 ret += nr_pinned;
2722                 }
2723         }
2724 
2725         return ret;
2726 }
2727 /**
2728  * get_user_pages_fast_only() - pin user pages in memory
2729  * @start:      starting user address
2730  * @nr_pages:   number of pages from start to pin
2731  * @gup_flags:  flags modifying pin behaviour
2732  * @pages:      array that receives pointers to the pages pinned.
2733  *              Should be at least nr_pages long.
2734  *
2735  * Like get_user_pages_fast() except it's IRQ-safe in that it won't fall back to
2736  * the regular GUP.
2737  * Note a difference with get_user_pages_fast: this always returns the
2738  * number of pages pinned, 0 if no pages were pinned.
2739  *
2740  * If the architecture does not support this function, simply return with no
2741  * pages pinned.
2742  *
2743  * Careful, careful! COW breaking can go either way, so a non-write
2744  * access can get ambiguous page results. If you call this function without
2745  * 'write' set, you'd better be sure that you're ok with that ambiguity.
2746  */
2747 int get_user_pages_fast_only(unsigned long start, int nr_pages,
2748                              unsigned int gup_flags, struct page **pages)
2749 {
2750         int nr_pinned;
2751         /*
2752          * Internally (within mm/gup.c), gup fast variants must set FOLL_GET,
2753          * because gup fast is always a "pin with a +1 page refcount" request.
2754          *
2755          * FOLL_FAST_ONLY is required in order to match the API description of
2756          * this routine: no fall back to regular ("slow") GUP.
2757          */
2758         gup_flags |= FOLL_GET | FOLL_FAST_ONLY;
2759 
2760         nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags,
2761                                                  pages);
2762 
2763         /*
2764          * As specified in the API description above, this routine is not
2765          * allowed to return negative values. However, the common core
2766          * routine internal_get_user_pages_fast() *can* return -errno.
2767          * Therefore, correct for that here:
2768          */
2769         if (nr_pinned < 0)
2770                 nr_pinned = 0;
2771 
2772         return nr_pinned;
2773 }
2774 EXPORT_SYMBOL_GPL(get_user_pages_fast_only);
2775 
2776 /**
2777  * get_user_pages_fast() - pin user pages in memory
2778  * @start:      starting user address
2779  * @nr_pages:   number of pages from start to pin
2780  * @gup_flags:  flags modifying pin behaviour
2781  * @pages:      array that receives pointers to the pages pinned.
2782  *              Should be at least nr_pages long.
2783  *
2784  * Attempt to pin user pages in memory without taking mm->mmap_lock.
2785  * If not successful, it will fall back to taking the lock and
2786  * calling get_user_pages().
2787  *
2788  * Returns number of pages pinned. This may be fewer than the number requested.
2789  * If nr_pages is 0 or negative, returns 0. If no pages were pinned, returns
2790  * -errno.
2791  */
2792 int get_user_pages_fast(unsigned long start, int nr_pages,
2793                         unsigned int gup_flags, struct page **pages)
2794 {
2795         /*
2796          * FOLL_PIN must only be set internally by the pin_user_pages*() APIs,
2797          * never directly by the caller, so enforce that:
2798          */
2799         if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
2800                 return -EINVAL;
2801 
2802         /*
2803          * The caller may or may not have explicitly set FOLL_GET; either way is
2804          * OK. However, internally (within mm/gup.c), gup fast variants must set
2805          * FOLL_GET, because gup fast is always a "pin with a +1 page refcount"
2806          * request.
2807          */
2808         gup_flags |= FOLL_GET;
2809         return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
2810 }
2811 EXPORT_SYMBOL_GPL(get_user_pages_fast);
2812 
2813 /**
2814  * pin_user_pages_fast() - pin user pages in memory without taking locks
2815  *
2816  * @start:      starting user address
2817  * @nr_pages:   number of pages from start to pin
2818  * @gup_flags:  flags modifying pin behaviour
2819  * @pages:      array that receives pointers to the pages pinned.
2820  *              Should be at least nr_pages long.
2821  *
2822  * Nearly the same as get_user_pages_fast(), except that FOLL_PIN is set. See
2823  * get_user_pages_fast() for documentation on the function arguments, because
2824  * the arguments here are identical.
2825  *
2826  * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
2827  * see Documentation/core-api/pin_user_pages.rst for further details.
2828  */
2829 int pin_user_pages_fast(unsigned long start, int nr_pages,
2830                         unsigned int gup_flags, struct page **pages)
2831 {
2832         /* FOLL_GET and FOLL_PIN are mutually exclusive. */
2833         if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2834                 return -EINVAL;
2835 
2836         gup_flags |= FOLL_PIN;
2837         return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
2838 }
2839 EXPORT_SYMBOL_GPL(pin_user_pages_fast);
2840 
2841 /*
2842  * This is the FOLL_PIN equivalent of get_user_pages_fast_only(). Behavior
2843  * is the same, except that this one sets FOLL_PIN instead of FOLL_GET.
2844  *
2845  * The API rules are the same, too: no negative values may be returned.
2846  */
2847 int pin_user_pages_fast_only(unsigned long start, int nr_pages,
2848                              unsigned int gup_flags, struct page **pages)
2849 {
2850         int nr_pinned;
2851 
2852         /*
2853          * FOLL_GET and FOLL_PIN are mutually exclusive. Note that the API
2854          * rules require returning 0, rather than -errno:
2855          */
2856         if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2857                 return 0;
2858         /*
2859          * FOLL_FAST_ONLY is required in order to match the API description of
2860          * this routine: no fall back to regular ("slow") GUP.
2861          */
2862         gup_flags |= (FOLL_PIN | FOLL_FAST_ONLY);
2863         nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags,
2864                                                  pages);
2865         /*
2866          * This routine is not allowed to return negative values. However,
2867          * internal_get_user_pages_fast() *can* return -errno. Therefore,
2868          * correct for that here:
2869          */
2870         if (nr_pinned < 0)
2871                 nr_pinned = 0;
2872 
2873         return nr_pinned;
2874 }
2875 EXPORT_SYMBOL_GPL(pin_user_pages_fast_only);
2876 
2877 /**
2878  * pin_user_pages_remote() - pin pages of a remote process
2879  *
2880  * @mm:         mm_struct of target mm
2881  * @start:      starting user address
2882  * @nr_pages:   number of pages from start to pin
2883  * @gup_flags:  flags modifying lookup behaviour
2884  * @pages:      array that receives pointers to the pages pinned.
2885  *              Should be at least nr_pages long. Or NULL, if caller
2886  *              only intends to ensure the pages are faulted in.
2887  * @vmas:       array of pointers to vmas corresponding to each page.
2888  *              Or NULL if the caller does not require them.
2889  * @locked:     pointer to lock flag indicating whether lock is held and
2890  *              subsequently whether VM_FAULT_RETRY functionality can be
2891  *              utilised. Lock must initially be held.
2892  *
2893  * Nearly the same as get_user_pages_remote(), except that FOLL_PIN is set. See
2894  * get_user_pages_remote() for documentation on the function arguments, because
2895  * the arguments here are identical.
2896  *
2897  * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
2898  * see Documentation/core-api/pin_user_pages.rst for details.
2899  */
2900 long pin_user_pages_remote(struct mm_struct *mm,
2901                            unsigned long start, unsigned long nr_pages,
2902                            unsigned int gup_flags, struct page **pages,
2903                            struct vm_area_struct **vmas, int *locked)
2904 {
2905         /* FOLL_GET and FOLL_PIN are mutually exclusive. */
2906         if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2907                 return -EINVAL;
2908 
2909         gup_flags |= FOLL_PIN;
2910         return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
2911                                        pages, vmas, locked);
2912 }
2913 EXPORT_SYMBOL(pin_user_pages_remote);
2914 
2915 /**
2916  * pin_user_pages() - pin user pages in memory for use by other devices
2917  *
2918  * @start:      starting user address
2919  * @nr_pages:   number of pages from start to pin
2920  * @gup_flags:  flags modifying lookup behaviour
2921  * @pages:      array that receives pointers to the pages pinned.
2922  *              Should be at least nr_pages long. Or NULL, if caller
2923  *              only intends to ensure the pages are faulted in.
2924  * @vmas:       array of pointers to vmas corresponding to each page.
2925  *              Or NULL if the caller does not require them.
2926  *
2927  * Nearly the same as get_user_pages(), except that FOLL_TOUCH is not set, and
2928  * FOLL_PIN is set.
2929  *
2930  * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
2931  * see Documentation/core-api/pin_user_pages.rst for details.
2932  */
2933 long pin_user_pages(unsigned long start, unsigned long nr_pages,
2934                     unsigned int gup_flags, struct page **pages,
2935                     struct vm_area_struct **vmas)
2936 {
2937         /* FOLL_GET and FOLL_PIN are mutually exclusive. */
2938         if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2939                 return -EINVAL;
2940 
2941         gup_flags |= FOLL_PIN;
2942         return __gup_longterm_locked(current->mm, start, nr_pages,
2943                                      pages, vmas, gup_flags);
2944 }
2945 EXPORT_SYMBOL(pin_user_pages);
2946 
2947 /*
2948  * pin_user_pages_unlocked() is the FOLL_PIN variant of
2949  * get_user_pages_unlocked(). Behavior is the same, except that this one sets
2950  * FOLL_PIN and rejects FOLL_GET.
2951  */
2952 long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
2953                              struct page **pages, unsigned int gup_flags)
2954 {
2955         /* FOLL_GET and FOLL_PIN are mutually exclusive. */
2956         if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2957                 return -EINVAL;
2958 
2959         gup_flags |= FOLL_PIN;
2960         return get_user_pages_unlocked(start, nr_pages, pages, gup_flags);
2961 }
2962 EXPORT_SYMBOL(pin_user_pages_unlocked);
2963 
2964 /*
2965  * pin_user_pages_locked() is the FOLL_PIN variant of get_user_pages_locked().
2966  * Behavior is the same, except that this one sets FOLL_PIN and rejects
2967  * FOLL_GET.
2968  */
2969 long pin_user_pages_locked(unsigned long start, unsigned long nr_pages,
2970                            unsigned int gup_flags, struct page **pages,
2971                            int *locked)
2972 {
2973         /*
2974          * FIXME: Current FOLL_LONGTERM behavior is incompatible with
2975          * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
2976          * vmas.  As there are no users of this flag in this call we simply
2977          * disallow this option for now.
2978          */
2979         if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
2980                 return -EINVAL;
2981 
2982         /* FOLL_GET and FOLL_PIN are mutually exclusive. */
2983         if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2984                 return -EINVAL;
2985 
2986         gup_flags |= FOLL_PIN;
2987         return __get_user_pages_locked(current->mm, start, nr_pages,
2988                                        pages, NULL, locked,
2989                                        gup_flags | FOLL_TOUCH);
2990 }
2991 EXPORT_SYMBOL(pin_user_pages_locked);
2992 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp