~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/hmm.c

Version: ~ [ linux-5.12-rc7 ] ~ [ linux-5.11.13 ] ~ [ linux-5.10.29 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.111 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.186 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.230 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.266 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.266 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-or-later
  2 /*
  3  * Copyright 2013 Red Hat Inc.
  4  *
  5  * Authors: Jérôme Glisse <jglisse@redhat.com>
  6  */
  7 /*
  8  * Refer to include/linux/hmm.h for information about heterogeneous memory
  9  * management or HMM for short.
 10  */
 11 #include <linux/pagewalk.h>
 12 #include <linux/hmm.h>
 13 #include <linux/init.h>
 14 #include <linux/rmap.h>
 15 #include <linux/swap.h>
 16 #include <linux/slab.h>
 17 #include <linux/sched.h>
 18 #include <linux/mmzone.h>
 19 #include <linux/pagemap.h>
 20 #include <linux/swapops.h>
 21 #include <linux/hugetlb.h>
 22 #include <linux/memremap.h>
 23 #include <linux/sched/mm.h>
 24 #include <linux/jump_label.h>
 25 #include <linux/dma-mapping.h>
 26 #include <linux/mmu_notifier.h>
 27 #include <linux/memory_hotplug.h>
 28 
 29 struct hmm_vma_walk {
 30         struct hmm_range        *range;
 31         struct dev_pagemap      *pgmap;
 32         unsigned long           last;
 33         unsigned int            flags;
 34 };
 35 
 36 static int hmm_vma_do_fault(struct mm_walk *walk, unsigned long addr,
 37                             bool write_fault, uint64_t *pfn)
 38 {
 39         unsigned int flags = FAULT_FLAG_REMOTE;
 40         struct hmm_vma_walk *hmm_vma_walk = walk->private;
 41         struct hmm_range *range = hmm_vma_walk->range;
 42         struct vm_area_struct *vma = walk->vma;
 43         vm_fault_t ret;
 44 
 45         if (!vma)
 46                 goto err;
 47 
 48         if (hmm_vma_walk->flags & HMM_FAULT_ALLOW_RETRY)
 49                 flags |= FAULT_FLAG_ALLOW_RETRY;
 50         if (write_fault)
 51                 flags |= FAULT_FLAG_WRITE;
 52 
 53         ret = handle_mm_fault(vma, addr, flags);
 54         if (ret & VM_FAULT_RETRY) {
 55                 /* Note, handle_mm_fault did up_read(&mm->mmap_sem)) */
 56                 return -EAGAIN;
 57         }
 58         if (ret & VM_FAULT_ERROR)
 59                 goto err;
 60 
 61         return -EBUSY;
 62 
 63 err:
 64         *pfn = range->values[HMM_PFN_ERROR];
 65         return -EFAULT;
 66 }
 67 
 68 static int hmm_pfns_fill(unsigned long addr, unsigned long end,
 69                 struct hmm_range *range, enum hmm_pfn_value_e value)
 70 {
 71         uint64_t *pfns = range->pfns;
 72         unsigned long i;
 73 
 74         i = (addr - range->start) >> PAGE_SHIFT;
 75         for (; addr < end; addr += PAGE_SIZE, i++)
 76                 pfns[i] = range->values[value];
 77 
 78         return 0;
 79 }
 80 
 81 /*
 82  * hmm_vma_walk_hole_() - handle a range lacking valid pmd or pte(s)
 83  * @addr: range virtual start address (inclusive)
 84  * @end: range virtual end address (exclusive)
 85  * @fault: should we fault or not ?
 86  * @write_fault: write fault ?
 87  * @walk: mm_walk structure
 88  * Return: 0 on success, -EBUSY after page fault, or page fault error
 89  *
 90  * This function will be called whenever pmd_none() or pte_none() returns true,
 91  * or whenever there is no page directory covering the virtual address range.
 92  */
 93 static int hmm_vma_walk_hole_(unsigned long addr, unsigned long end,
 94                               bool fault, bool write_fault,
 95                               struct mm_walk *walk)
 96 {
 97         struct hmm_vma_walk *hmm_vma_walk = walk->private;
 98         struct hmm_range *range = hmm_vma_walk->range;
 99         uint64_t *pfns = range->pfns;
100         unsigned long i;
101 
102         hmm_vma_walk->last = addr;
103         i = (addr - range->start) >> PAGE_SHIFT;
104 
105         if (write_fault && walk->vma && !(walk->vma->vm_flags & VM_WRITE))
106                 return -EPERM;
107 
108         for (; addr < end; addr += PAGE_SIZE, i++) {
109                 pfns[i] = range->values[HMM_PFN_NONE];
110                 if (fault || write_fault) {
111                         int ret;
112 
113                         ret = hmm_vma_do_fault(walk, addr, write_fault,
114                                                &pfns[i]);
115                         if (ret != -EBUSY)
116                                 return ret;
117                 }
118         }
119 
120         return (fault || write_fault) ? -EBUSY : 0;
121 }
122 
123 static inline void hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
124                                       uint64_t pfns, uint64_t cpu_flags,
125                                       bool *fault, bool *write_fault)
126 {
127         struct hmm_range *range = hmm_vma_walk->range;
128 
129         if (hmm_vma_walk->flags & HMM_FAULT_SNAPSHOT)
130                 return;
131 
132         /*
133          * So we not only consider the individual per page request we also
134          * consider the default flags requested for the range. The API can
135          * be used 2 ways. The first one where the HMM user coalesces
136          * multiple page faults into one request and sets flags per pfn for
137          * those faults. The second one where the HMM user wants to pre-
138          * fault a range with specific flags. For the latter one it is a
139          * waste to have the user pre-fill the pfn arrays with a default
140          * flags value.
141          */
142         pfns = (pfns & range->pfn_flags_mask) | range->default_flags;
143 
144         /* We aren't ask to do anything ... */
145         if (!(pfns & range->flags[HMM_PFN_VALID]))
146                 return;
147         /* If this is device memory then only fault if explicitly requested */
148         if ((cpu_flags & range->flags[HMM_PFN_DEVICE_PRIVATE])) {
149                 /* Do we fault on device memory ? */
150                 if (pfns & range->flags[HMM_PFN_DEVICE_PRIVATE]) {
151                         *write_fault = pfns & range->flags[HMM_PFN_WRITE];
152                         *fault = true;
153                 }
154                 return;
155         }
156 
157         /* If CPU page table is not valid then we need to fault */
158         *fault = !(cpu_flags & range->flags[HMM_PFN_VALID]);
159         /* Need to write fault ? */
160         if ((pfns & range->flags[HMM_PFN_WRITE]) &&
161             !(cpu_flags & range->flags[HMM_PFN_WRITE])) {
162                 *write_fault = true;
163                 *fault = true;
164         }
165 }
166 
167 static void hmm_range_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
168                                  const uint64_t *pfns, unsigned long npages,
169                                  uint64_t cpu_flags, bool *fault,
170                                  bool *write_fault)
171 {
172         unsigned long i;
173 
174         if (hmm_vma_walk->flags & HMM_FAULT_SNAPSHOT) {
175                 *fault = *write_fault = false;
176                 return;
177         }
178 
179         *fault = *write_fault = false;
180         for (i = 0; i < npages; ++i) {
181                 hmm_pte_need_fault(hmm_vma_walk, pfns[i], cpu_flags,
182                                    fault, write_fault);
183                 if ((*write_fault))
184                         return;
185         }
186 }
187 
188 static int hmm_vma_walk_hole(unsigned long addr, unsigned long end,
189                              __always_unused int depth, struct mm_walk *walk)
190 {
191         struct hmm_vma_walk *hmm_vma_walk = walk->private;
192         struct hmm_range *range = hmm_vma_walk->range;
193         bool fault, write_fault;
194         unsigned long i, npages;
195         uint64_t *pfns;
196 
197         i = (addr - range->start) >> PAGE_SHIFT;
198         npages = (end - addr) >> PAGE_SHIFT;
199         pfns = &range->pfns[i];
200         hmm_range_need_fault(hmm_vma_walk, pfns, npages,
201                              0, &fault, &write_fault);
202         return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
203 }
204 
205 static inline uint64_t pmd_to_hmm_pfn_flags(struct hmm_range *range, pmd_t pmd)
206 {
207         if (pmd_protnone(pmd))
208                 return 0;
209         return pmd_write(pmd) ? range->flags[HMM_PFN_VALID] |
210                                 range->flags[HMM_PFN_WRITE] :
211                                 range->flags[HMM_PFN_VALID];
212 }
213 
214 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
215 static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,
216                 unsigned long end, uint64_t *pfns, pmd_t pmd)
217 {
218         struct hmm_vma_walk *hmm_vma_walk = walk->private;
219         struct hmm_range *range = hmm_vma_walk->range;
220         unsigned long pfn, npages, i;
221         bool fault, write_fault;
222         uint64_t cpu_flags;
223 
224         npages = (end - addr) >> PAGE_SHIFT;
225         cpu_flags = pmd_to_hmm_pfn_flags(range, pmd);
226         hmm_range_need_fault(hmm_vma_walk, pfns, npages, cpu_flags,
227                              &fault, &write_fault);
228 
229         if (pmd_protnone(pmd) || fault || write_fault)
230                 return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
231 
232         pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
233         for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) {
234                 if (pmd_devmap(pmd)) {
235                         hmm_vma_walk->pgmap = get_dev_pagemap(pfn,
236                                               hmm_vma_walk->pgmap);
237                         if (unlikely(!hmm_vma_walk->pgmap))
238                                 return -EBUSY;
239                 }
240                 pfns[i] = hmm_device_entry_from_pfn(range, pfn) | cpu_flags;
241         }
242         if (hmm_vma_walk->pgmap) {
243                 put_dev_pagemap(hmm_vma_walk->pgmap);
244                 hmm_vma_walk->pgmap = NULL;
245         }
246         hmm_vma_walk->last = end;
247         return 0;
248 }
249 #else /* CONFIG_TRANSPARENT_HUGEPAGE */
250 /* stub to allow the code below to compile */
251 int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,
252                 unsigned long end, uint64_t *pfns, pmd_t pmd);
253 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
254 
255 static inline uint64_t pte_to_hmm_pfn_flags(struct hmm_range *range, pte_t pte)
256 {
257         if (pte_none(pte) || !pte_present(pte) || pte_protnone(pte))
258                 return 0;
259         return pte_write(pte) ? range->flags[HMM_PFN_VALID] |
260                                 range->flags[HMM_PFN_WRITE] :
261                                 range->flags[HMM_PFN_VALID];
262 }
263 
264 static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
265                               unsigned long end, pmd_t *pmdp, pte_t *ptep,
266                               uint64_t *pfn)
267 {
268         struct hmm_vma_walk *hmm_vma_walk = walk->private;
269         struct hmm_range *range = hmm_vma_walk->range;
270         bool fault, write_fault;
271         uint64_t cpu_flags;
272         pte_t pte = *ptep;
273         uint64_t orig_pfn = *pfn;
274 
275         *pfn = range->values[HMM_PFN_NONE];
276         fault = write_fault = false;
277 
278         if (pte_none(pte)) {
279                 hmm_pte_need_fault(hmm_vma_walk, orig_pfn, 0,
280                                    &fault, &write_fault);
281                 if (fault || write_fault)
282                         goto fault;
283                 return 0;
284         }
285 
286         if (!pte_present(pte)) {
287                 swp_entry_t entry = pte_to_swp_entry(pte);
288 
289                 if (!non_swap_entry(entry)) {
290                         cpu_flags = pte_to_hmm_pfn_flags(range, pte);
291                         hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
292                                            &fault, &write_fault);
293                         if (fault || write_fault)
294                                 goto fault;
295                         return 0;
296                 }
297 
298                 /*
299                  * This is a special swap entry, ignore migration, use
300                  * device and report anything else as error.
301                  */
302                 if (is_device_private_entry(entry)) {
303                         cpu_flags = range->flags[HMM_PFN_VALID] |
304                                 range->flags[HMM_PFN_DEVICE_PRIVATE];
305                         cpu_flags |= is_write_device_private_entry(entry) ?
306                                 range->flags[HMM_PFN_WRITE] : 0;
307                         hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
308                                            &fault, &write_fault);
309                         if (fault || write_fault)
310                                 goto fault;
311                         *pfn = hmm_device_entry_from_pfn(range,
312                                             swp_offset(entry));
313                         *pfn |= cpu_flags;
314                         return 0;
315                 }
316 
317                 if (is_migration_entry(entry)) {
318                         if (fault || write_fault) {
319                                 pte_unmap(ptep);
320                                 hmm_vma_walk->last = addr;
321                                 migration_entry_wait(walk->mm, pmdp, addr);
322                                 return -EBUSY;
323                         }
324                         return 0;
325                 }
326 
327                 /* Report error for everything else */
328                 *pfn = range->values[HMM_PFN_ERROR];
329                 return -EFAULT;
330         } else {
331                 cpu_flags = pte_to_hmm_pfn_flags(range, pte);
332                 hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
333                                    &fault, &write_fault);
334         }
335 
336         if (fault || write_fault)
337                 goto fault;
338 
339         if (pte_devmap(pte)) {
340                 hmm_vma_walk->pgmap = get_dev_pagemap(pte_pfn(pte),
341                                               hmm_vma_walk->pgmap);
342                 if (unlikely(!hmm_vma_walk->pgmap))
343                         return -EBUSY;
344         } else if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) && pte_special(pte)) {
345                 if (!is_zero_pfn(pte_pfn(pte))) {
346                         *pfn = range->values[HMM_PFN_SPECIAL];
347                         return -EFAULT;
348                 }
349                 /*
350                  * Since each architecture defines a struct page for the zero
351                  * page, just fall through and treat it like a normal page.
352                  */
353         }
354 
355         *pfn = hmm_device_entry_from_pfn(range, pte_pfn(pte)) | cpu_flags;
356         return 0;
357 
358 fault:
359         if (hmm_vma_walk->pgmap) {
360                 put_dev_pagemap(hmm_vma_walk->pgmap);
361                 hmm_vma_walk->pgmap = NULL;
362         }
363         pte_unmap(ptep);
364         /* Fault any virtual address we were asked to fault */
365         return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
366 }
367 
368 static int hmm_vma_walk_pmd(pmd_t *pmdp,
369                             unsigned long start,
370                             unsigned long end,
371                             struct mm_walk *walk)
372 {
373         struct hmm_vma_walk *hmm_vma_walk = walk->private;
374         struct hmm_range *range = hmm_vma_walk->range;
375         uint64_t *pfns = range->pfns;
376         unsigned long addr = start, i;
377         pte_t *ptep;
378         pmd_t pmd;
379 
380 again:
381         pmd = READ_ONCE(*pmdp);
382         if (pmd_none(pmd))
383                 return hmm_vma_walk_hole(start, end, -1, walk);
384 
385         if (thp_migration_supported() && is_pmd_migration_entry(pmd)) {
386                 bool fault, write_fault;
387                 unsigned long npages;
388                 uint64_t *pfns;
389 
390                 i = (addr - range->start) >> PAGE_SHIFT;
391                 npages = (end - addr) >> PAGE_SHIFT;
392                 pfns = &range->pfns[i];
393 
394                 hmm_range_need_fault(hmm_vma_walk, pfns, npages,
395                                      0, &fault, &write_fault);
396                 if (fault || write_fault) {
397                         hmm_vma_walk->last = addr;
398                         pmd_migration_entry_wait(walk->mm, pmdp);
399                         return -EBUSY;
400                 }
401                 return 0;
402         } else if (!pmd_present(pmd))
403                 return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);
404 
405         if (pmd_devmap(pmd) || pmd_trans_huge(pmd)) {
406                 /*
407                  * No need to take pmd_lock here, even if some other thread
408                  * is splitting the huge pmd we will get that event through
409                  * mmu_notifier callback.
410                  *
411                  * So just read pmd value and check again it's a transparent
412                  * huge or device mapping one and compute corresponding pfn
413                  * values.
414                  */
415                 pmd = pmd_read_atomic(pmdp);
416                 barrier();
417                 if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))
418                         goto again;
419 
420                 i = (addr - range->start) >> PAGE_SHIFT;
421                 return hmm_vma_handle_pmd(walk, addr, end, &pfns[i], pmd);
422         }
423 
424         /*
425          * We have handled all the valid cases above ie either none, migration,
426          * huge or transparent huge. At this point either it is a valid pmd
427          * entry pointing to pte directory or it is a bad pmd that will not
428          * recover.
429          */
430         if (pmd_bad(pmd))
431                 return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);
432 
433         ptep = pte_offset_map(pmdp, addr);
434         i = (addr - range->start) >> PAGE_SHIFT;
435         for (; addr < end; addr += PAGE_SIZE, ptep++, i++) {
436                 int r;
437 
438                 r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, &pfns[i]);
439                 if (r) {
440                         /* hmm_vma_handle_pte() did unmap pte directory */
441                         hmm_vma_walk->last = addr;
442                         return r;
443                 }
444         }
445         if (hmm_vma_walk->pgmap) {
446                 /*
447                  * We do put_dev_pagemap() here and not in hmm_vma_handle_pte()
448                  * so that we can leverage get_dev_pagemap() optimization which
449                  * will not re-take a reference on a pgmap if we already have
450                  * one.
451                  */
452                 put_dev_pagemap(hmm_vma_walk->pgmap);
453                 hmm_vma_walk->pgmap = NULL;
454         }
455         pte_unmap(ptep - 1);
456 
457         hmm_vma_walk->last = addr;
458         return 0;
459 }
460 
461 #if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && \
462     defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
463 static inline uint64_t pud_to_hmm_pfn_flags(struct hmm_range *range, pud_t pud)
464 {
465         if (!pud_present(pud))
466                 return 0;
467         return pud_write(pud) ? range->flags[HMM_PFN_VALID] |
468                                 range->flags[HMM_PFN_WRITE] :
469                                 range->flags[HMM_PFN_VALID];
470 }
471 
472 static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end,
473                 struct mm_walk *walk)
474 {
475         struct hmm_vma_walk *hmm_vma_walk = walk->private;
476         struct hmm_range *range = hmm_vma_walk->range;
477         unsigned long addr = start;
478         pud_t pud;
479         int ret = 0;
480         spinlock_t *ptl = pud_trans_huge_lock(pudp, walk->vma);
481 
482         if (!ptl)
483                 return 0;
484 
485         /* Normally we don't want to split the huge page */
486         walk->action = ACTION_CONTINUE;
487 
488         pud = READ_ONCE(*pudp);
489         if (pud_none(pud)) {
490                 ret = hmm_vma_walk_hole(start, end, -1, walk);
491                 goto out_unlock;
492         }
493 
494         if (pud_huge(pud) && pud_devmap(pud)) {
495                 unsigned long i, npages, pfn;
496                 uint64_t *pfns, cpu_flags;
497                 bool fault, write_fault;
498 
499                 if (!pud_present(pud)) {
500                         ret = hmm_vma_walk_hole(start, end, -1, walk);
501                         goto out_unlock;
502                 }
503 
504                 i = (addr - range->start) >> PAGE_SHIFT;
505                 npages = (end - addr) >> PAGE_SHIFT;
506                 pfns = &range->pfns[i];
507 
508                 cpu_flags = pud_to_hmm_pfn_flags(range, pud);
509                 hmm_range_need_fault(hmm_vma_walk, pfns, npages,
510                                      cpu_flags, &fault, &write_fault);
511                 if (fault || write_fault) {
512                         ret = hmm_vma_walk_hole_(addr, end, fault,
513                                                  write_fault, walk);
514                         goto out_unlock;
515                 }
516 
517                 pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
518                 for (i = 0; i < npages; ++i, ++pfn) {
519                         hmm_vma_walk->pgmap = get_dev_pagemap(pfn,
520                                               hmm_vma_walk->pgmap);
521                         if (unlikely(!hmm_vma_walk->pgmap)) {
522                                 ret = -EBUSY;
523                                 goto out_unlock;
524                         }
525                         pfns[i] = hmm_device_entry_from_pfn(range, pfn) |
526                                   cpu_flags;
527                 }
528                 if (hmm_vma_walk->pgmap) {
529                         put_dev_pagemap(hmm_vma_walk->pgmap);
530                         hmm_vma_walk->pgmap = NULL;
531                 }
532                 hmm_vma_walk->last = end;
533                 goto out_unlock;
534         }
535 
536         /* Ask for the PUD to be split */
537         walk->action = ACTION_SUBTREE;
538 
539 out_unlock:
540         spin_unlock(ptl);
541         return ret;
542 }
543 #else
544 #define hmm_vma_walk_pud        NULL
545 #endif
546 
547 #ifdef CONFIG_HUGETLB_PAGE
548 static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask,
549                                       unsigned long start, unsigned long end,
550                                       struct mm_walk *walk)
551 {
552         unsigned long addr = start, i, pfn;
553         struct hmm_vma_walk *hmm_vma_walk = walk->private;
554         struct hmm_range *range = hmm_vma_walk->range;
555         struct vm_area_struct *vma = walk->vma;
556         uint64_t orig_pfn, cpu_flags;
557         bool fault, write_fault;
558         spinlock_t *ptl;
559         pte_t entry;
560         int ret = 0;
561 
562         ptl = huge_pte_lock(hstate_vma(vma), walk->mm, pte);
563         entry = huge_ptep_get(pte);
564 
565         i = (start - range->start) >> PAGE_SHIFT;
566         orig_pfn = range->pfns[i];
567         range->pfns[i] = range->values[HMM_PFN_NONE];
568         cpu_flags = pte_to_hmm_pfn_flags(range, entry);
569         fault = write_fault = false;
570         hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
571                            &fault, &write_fault);
572         if (fault || write_fault) {
573                 ret = -ENOENT;
574                 goto unlock;
575         }
576 
577         pfn = pte_pfn(entry) + ((start & ~hmask) >> PAGE_SHIFT);
578         for (; addr < end; addr += PAGE_SIZE, i++, pfn++)
579                 range->pfns[i] = hmm_device_entry_from_pfn(range, pfn) |
580                                  cpu_flags;
581         hmm_vma_walk->last = end;
582 
583 unlock:
584         spin_unlock(ptl);
585 
586         if (ret == -ENOENT)
587                 return hmm_vma_walk_hole_(addr, end, fault, write_fault, walk);
588 
589         return ret;
590 }
591 #else
592 #define hmm_vma_walk_hugetlb_entry NULL
593 #endif /* CONFIG_HUGETLB_PAGE */
594 
595 static int hmm_vma_walk_test(unsigned long start, unsigned long end,
596                              struct mm_walk *walk)
597 {
598         struct hmm_vma_walk *hmm_vma_walk = walk->private;
599         struct hmm_range *range = hmm_vma_walk->range;
600         struct vm_area_struct *vma = walk->vma;
601 
602         /*
603          * Skip vma ranges that don't have struct page backing them or
604          * map I/O devices directly.
605          */
606         if (vma->vm_flags & (VM_IO | VM_PFNMAP | VM_MIXEDMAP))
607                 return -EFAULT;
608 
609         /*
610          * If the vma does not allow read access, then assume that it does not
611          * allow write access either. HMM does not support architectures
612          * that allow write without read.
613          */
614         if (!(vma->vm_flags & VM_READ)) {
615                 bool fault, write_fault;
616 
617                 /*
618                  * Check to see if a fault is requested for any page in the
619                  * range.
620                  */
621                 hmm_range_need_fault(hmm_vma_walk, range->pfns +
622                                         ((start - range->start) >> PAGE_SHIFT),
623                                         (end - start) >> PAGE_SHIFT,
624                                         0, &fault, &write_fault);
625                 if (fault || write_fault)
626                         return -EFAULT;
627 
628                 hmm_pfns_fill(start, end, range, HMM_PFN_NONE);
629                 hmm_vma_walk->last = end;
630 
631                 /* Skip this vma and continue processing the next vma. */
632                 return 1;
633         }
634 
635         return 0;
636 }
637 
638 static const struct mm_walk_ops hmm_walk_ops = {
639         .pud_entry      = hmm_vma_walk_pud,
640         .pmd_entry      = hmm_vma_walk_pmd,
641         .pte_hole       = hmm_vma_walk_hole,
642         .hugetlb_entry  = hmm_vma_walk_hugetlb_entry,
643         .test_walk      = hmm_vma_walk_test,
644 };
645 
646 /**
647  * hmm_range_fault - try to fault some address in a virtual address range
648  * @range:      range being faulted
649  * @flags:      HMM_FAULT_* flags
650  *
651  * Return: the number of valid pages in range->pfns[] (from range start
652  * address), which may be zero.  On error one of the following status codes
653  * can be returned:
654  *
655  * -EINVAL:     Invalid arguments or mm or virtual address is in an invalid vma
656  *              (e.g., device file vma).
657  * -ENOMEM:     Out of memory.
658  * -EPERM:      Invalid permission (e.g., asking for write and range is read
659  *              only).
660  * -EAGAIN:     A page fault needs to be retried and mmap_sem was dropped.
661  * -EBUSY:      The range has been invalidated and the caller needs to wait for
662  *              the invalidation to finish.
663  * -EFAULT:     Invalid (i.e., either no valid vma or it is illegal to access
664  *              that range) number of valid pages in range->pfns[] (from
665  *              range start address).
666  *
667  * This is similar to a regular CPU page fault except that it will not trigger
668  * any memory migration if the memory being faulted is not accessible by CPUs
669  * and caller does not ask for migration.
670  *
671  * On error, for one virtual address in the range, the function will mark the
672  * corresponding HMM pfn entry with an error flag.
673  */
674 long hmm_range_fault(struct hmm_range *range, unsigned int flags)
675 {
676         struct hmm_vma_walk hmm_vma_walk = {
677                 .range = range,
678                 .last = range->start,
679                 .flags = flags,
680         };
681         struct mm_struct *mm = range->notifier->mm;
682         int ret;
683 
684         lockdep_assert_held(&mm->mmap_sem);
685 
686         do {
687                 /* If range is no longer valid force retry. */
688                 if (mmu_interval_check_retry(range->notifier,
689                                              range->notifier_seq))
690                         return -EBUSY;
691                 ret = walk_page_range(mm, hmm_vma_walk.last, range->end,
692                                       &hmm_walk_ops, &hmm_vma_walk);
693         } while (ret == -EBUSY);
694 
695         if (ret)
696                 return ret;
697         return (hmm_vma_walk.last - range->start) >> PAGE_SHIFT;
698 }
699 EXPORT_SYMBOL(hmm_range_fault);
700 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp