~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/fs/dax.c

Version: ~ [ linux-4.15-rc8 ] ~ [ linux-4.14.13 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.76 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.111 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.48 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.91 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.53 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.98 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * fs/dax.c - Direct Access filesystem code
  3  * Copyright (c) 2013-2014 Intel Corporation
  4  * Author: Matthew Wilcox <matthew.r.wilcox@intel.com>
  5  * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
  6  *
  7  * This program is free software; you can redistribute it and/or modify it
  8  * under the terms and conditions of the GNU General Public License,
  9  * version 2, as published by the Free Software Foundation.
 10  *
 11  * This program is distributed in the hope it will be useful, but WITHOUT
 12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 14  * more details.
 15  */
 16 
 17 #include <linux/atomic.h>
 18 #include <linux/blkdev.h>
 19 #include <linux/buffer_head.h>
 20 #include <linux/dax.h>
 21 #include <linux/fs.h>
 22 #include <linux/genhd.h>
 23 #include <linux/highmem.h>
 24 #include <linux/memcontrol.h>
 25 #include <linux/mm.h>
 26 #include <linux/mutex.h>
 27 #include <linux/pagevec.h>
 28 #include <linux/sched.h>
 29 #include <linux/sched/signal.h>
 30 #include <linux/uio.h>
 31 #include <linux/vmstat.h>
 32 #include <linux/pfn_t.h>
 33 #include <linux/sizes.h>
 34 #include <linux/mmu_notifier.h>
 35 #include <linux/iomap.h>
 36 #include "internal.h"
 37 
 38 #define CREATE_TRACE_POINTS
 39 #include <trace/events/fs_dax.h>
 40 
 41 /* We choose 4096 entries - same as per-zone page wait tables */
 42 #define DAX_WAIT_TABLE_BITS 12
 43 #define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)
 44 
 45 /* The 'colour' (ie low bits) within a PMD of a page offset.  */
 46 #define PG_PMD_COLOUR   ((PMD_SIZE >> PAGE_SHIFT) - 1)
 47 
 48 static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
 49 
 50 static int __init init_dax_wait_table(void)
 51 {
 52         int i;
 53 
 54         for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++)
 55                 init_waitqueue_head(wait_table + i);
 56         return 0;
 57 }
 58 fs_initcall(init_dax_wait_table);
 59 
 60 /*
 61  * We use lowest available bit in exceptional entry for locking, one bit for
 62  * the entry size (PMD) and two more to tell us if the entry is a zero page or
 63  * an empty entry that is just used for locking.  In total four special bits.
 64  *
 65  * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE
 66  * and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem
 67  * block allocation.
 68  */
 69 #define RADIX_DAX_SHIFT         (RADIX_TREE_EXCEPTIONAL_SHIFT + 4)
 70 #define RADIX_DAX_ENTRY_LOCK    (1 << RADIX_TREE_EXCEPTIONAL_SHIFT)
 71 #define RADIX_DAX_PMD           (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1))
 72 #define RADIX_DAX_ZERO_PAGE     (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2))
 73 #define RADIX_DAX_EMPTY         (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 3))
 74 
 75 static unsigned long dax_radix_sector(void *entry)
 76 {
 77         return (unsigned long)entry >> RADIX_DAX_SHIFT;
 78 }
 79 
 80 static void *dax_radix_locked_entry(sector_t sector, unsigned long flags)
 81 {
 82         return (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY | flags |
 83                         ((unsigned long)sector << RADIX_DAX_SHIFT) |
 84                         RADIX_DAX_ENTRY_LOCK);
 85 }
 86 
 87 static unsigned int dax_radix_order(void *entry)
 88 {
 89         if ((unsigned long)entry & RADIX_DAX_PMD)
 90                 return PMD_SHIFT - PAGE_SHIFT;
 91         return 0;
 92 }
 93 
 94 static int dax_is_pmd_entry(void *entry)
 95 {
 96         return (unsigned long)entry & RADIX_DAX_PMD;
 97 }
 98 
 99 static int dax_is_pte_entry(void *entry)
100 {
101         return !((unsigned long)entry & RADIX_DAX_PMD);
102 }
103 
104 static int dax_is_zero_entry(void *entry)
105 {
106         return (unsigned long)entry & RADIX_DAX_ZERO_PAGE;
107 }
108 
109 static int dax_is_empty_entry(void *entry)
110 {
111         return (unsigned long)entry & RADIX_DAX_EMPTY;
112 }
113 
114 /*
115  * DAX radix tree locking
116  */
117 struct exceptional_entry_key {
118         struct address_space *mapping;
119         pgoff_t entry_start;
120 };
121 
122 struct wait_exceptional_entry_queue {
123         wait_queue_entry_t wait;
124         struct exceptional_entry_key key;
125 };
126 
127 static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping,
128                 pgoff_t index, void *entry, struct exceptional_entry_key *key)
129 {
130         unsigned long hash;
131 
132         /*
133          * If 'entry' is a PMD, align the 'index' that we use for the wait
134          * queue to the start of that PMD.  This ensures that all offsets in
135          * the range covered by the PMD map to the same bit lock.
136          */
137         if (dax_is_pmd_entry(entry))
138                 index &= ~PG_PMD_COLOUR;
139 
140         key->mapping = mapping;
141         key->entry_start = index;
142 
143         hash = hash_long((unsigned long)mapping ^ index, DAX_WAIT_TABLE_BITS);
144         return wait_table + hash;
145 }
146 
147 static int wake_exceptional_entry_func(wait_queue_entry_t *wait, unsigned int mode,
148                                        int sync, void *keyp)
149 {
150         struct exceptional_entry_key *key = keyp;
151         struct wait_exceptional_entry_queue *ewait =
152                 container_of(wait, struct wait_exceptional_entry_queue, wait);
153 
154         if (key->mapping != ewait->key.mapping ||
155             key->entry_start != ewait->key.entry_start)
156                 return 0;
157         return autoremove_wake_function(wait, mode, sync, NULL);
158 }
159 
160 /*
161  * We do not necessarily hold the mapping->tree_lock when we call this
162  * function so it is possible that 'entry' is no longer a valid item in the
163  * radix tree.  This is okay because all we really need to do is to find the
164  * correct waitqueue where tasks might be waiting for that old 'entry' and
165  * wake them.
166  */
167 static void dax_wake_mapping_entry_waiter(struct address_space *mapping,
168                 pgoff_t index, void *entry, bool wake_all)
169 {
170         struct exceptional_entry_key key;
171         wait_queue_head_t *wq;
172 
173         wq = dax_entry_waitqueue(mapping, index, entry, &key);
174 
175         /*
176          * Checking for locked entry and prepare_to_wait_exclusive() happens
177          * under mapping->tree_lock, ditto for entry handling in our callers.
178          * So at this point all tasks that could have seen our entry locked
179          * must be in the waitqueue and the following check will see them.
180          */
181         if (waitqueue_active(wq))
182                 __wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key);
183 }
184 
185 /*
186  * Check whether the given slot is locked. The function must be called with
187  * mapping->tree_lock held
188  */
189 static inline int slot_locked(struct address_space *mapping, void **slot)
190 {
191         unsigned long entry = (unsigned long)
192                 radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
193         return entry & RADIX_DAX_ENTRY_LOCK;
194 }
195 
196 /*
197  * Mark the given slot is locked. The function must be called with
198  * mapping->tree_lock held
199  */
200 static inline void *lock_slot(struct address_space *mapping, void **slot)
201 {
202         unsigned long entry = (unsigned long)
203                 radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
204 
205         entry |= RADIX_DAX_ENTRY_LOCK;
206         radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry);
207         return (void *)entry;
208 }
209 
210 /*
211  * Mark the given slot is unlocked. The function must be called with
212  * mapping->tree_lock held
213  */
214 static inline void *unlock_slot(struct address_space *mapping, void **slot)
215 {
216         unsigned long entry = (unsigned long)
217                 radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
218 
219         entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK;
220         radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry);
221         return (void *)entry;
222 }
223 
224 /*
225  * Lookup entry in radix tree, wait for it to become unlocked if it is
226  * exceptional entry and return it. The caller must call
227  * put_unlocked_mapping_entry() when he decided not to lock the entry or
228  * put_locked_mapping_entry() when he locked the entry and now wants to
229  * unlock it.
230  *
231  * The function must be called with mapping->tree_lock held.
232  */
233 static void *get_unlocked_mapping_entry(struct address_space *mapping,
234                                         pgoff_t index, void ***slotp)
235 {
236         void *entry, **slot;
237         struct wait_exceptional_entry_queue ewait;
238         wait_queue_head_t *wq;
239 
240         init_wait(&ewait.wait);
241         ewait.wait.func = wake_exceptional_entry_func;
242 
243         for (;;) {
244                 entry = __radix_tree_lookup(&mapping->page_tree, index, NULL,
245                                           &slot);
246                 if (!entry ||
247                     WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)) ||
248                     !slot_locked(mapping, slot)) {
249                         if (slotp)
250                                 *slotp = slot;
251                         return entry;
252                 }
253 
254                 wq = dax_entry_waitqueue(mapping, index, entry, &ewait.key);
255                 prepare_to_wait_exclusive(wq, &ewait.wait,
256                                           TASK_UNINTERRUPTIBLE);
257                 spin_unlock_irq(&mapping->tree_lock);
258                 schedule();
259                 finish_wait(wq, &ewait.wait);
260                 spin_lock_irq(&mapping->tree_lock);
261         }
262 }
263 
264 static void dax_unlock_mapping_entry(struct address_space *mapping,
265                                      pgoff_t index)
266 {
267         void *entry, **slot;
268 
269         spin_lock_irq(&mapping->tree_lock);
270         entry = __radix_tree_lookup(&mapping->page_tree, index, NULL, &slot);
271         if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) ||
272                          !slot_locked(mapping, slot))) {
273                 spin_unlock_irq(&mapping->tree_lock);
274                 return;
275         }
276         unlock_slot(mapping, slot);
277         spin_unlock_irq(&mapping->tree_lock);
278         dax_wake_mapping_entry_waiter(mapping, index, entry, false);
279 }
280 
281 static void put_locked_mapping_entry(struct address_space *mapping,
282                 pgoff_t index)
283 {
284         dax_unlock_mapping_entry(mapping, index);
285 }
286 
287 /*
288  * Called when we are done with radix tree entry we looked up via
289  * get_unlocked_mapping_entry() and which we didn't lock in the end.
290  */
291 static void put_unlocked_mapping_entry(struct address_space *mapping,
292                                        pgoff_t index, void *entry)
293 {
294         if (!entry)
295                 return;
296 
297         /* We have to wake up next waiter for the radix tree entry lock */
298         dax_wake_mapping_entry_waiter(mapping, index, entry, false);
299 }
300 
301 /*
302  * Find radix tree entry at given index. If it points to an exceptional entry,
303  * return it with the radix tree entry locked. If the radix tree doesn't
304  * contain given index, create an empty exceptional entry for the index and
305  * return with it locked.
306  *
307  * When requesting an entry with size RADIX_DAX_PMD, grab_mapping_entry() will
308  * either return that locked entry or will return an error.  This error will
309  * happen if there are any 4k entries within the 2MiB range that we are
310  * requesting.
311  *
312  * We always favor 4k entries over 2MiB entries. There isn't a flow where we
313  * evict 4k entries in order to 'upgrade' them to a 2MiB entry.  A 2MiB
314  * insertion will fail if it finds any 4k entries already in the tree, and a
315  * 4k insertion will cause an existing 2MiB entry to be unmapped and
316  * downgraded to 4k entries.  This happens for both 2MiB huge zero pages as
317  * well as 2MiB empty entries.
318  *
319  * The exception to this downgrade path is for 2MiB DAX PMD entries that have
320  * real storage backing them.  We will leave these real 2MiB DAX entries in
321  * the tree, and PTE writes will simply dirty the entire 2MiB DAX entry.
322  *
323  * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For
324  * persistent memory the benefit is doubtful. We can add that later if we can
325  * show it helps.
326  */
327 static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index,
328                 unsigned long size_flag)
329 {
330         bool pmd_downgrade = false; /* splitting 2MiB entry into 4k entries? */
331         void *entry, **slot;
332 
333 restart:
334         spin_lock_irq(&mapping->tree_lock);
335         entry = get_unlocked_mapping_entry(mapping, index, &slot);
336 
337         if (WARN_ON_ONCE(entry && !radix_tree_exceptional_entry(entry))) {
338                 entry = ERR_PTR(-EIO);
339                 goto out_unlock;
340         }
341 
342         if (entry) {
343                 if (size_flag & RADIX_DAX_PMD) {
344                         if (dax_is_pte_entry(entry)) {
345                                 put_unlocked_mapping_entry(mapping, index,
346                                                 entry);
347                                 entry = ERR_PTR(-EEXIST);
348                                 goto out_unlock;
349                         }
350                 } else { /* trying to grab a PTE entry */
351                         if (dax_is_pmd_entry(entry) &&
352                             (dax_is_zero_entry(entry) ||
353                              dax_is_empty_entry(entry))) {
354                                 pmd_downgrade = true;
355                         }
356                 }
357         }
358 
359         /* No entry for given index? Make sure radix tree is big enough. */
360         if (!entry || pmd_downgrade) {
361                 int err;
362 
363                 if (pmd_downgrade) {
364                         /*
365                          * Make sure 'entry' remains valid while we drop
366                          * mapping->tree_lock.
367                          */
368                         entry = lock_slot(mapping, slot);
369                 }
370 
371                 spin_unlock_irq(&mapping->tree_lock);
372                 /*
373                  * Besides huge zero pages the only other thing that gets
374                  * downgraded are empty entries which don't need to be
375                  * unmapped.
376                  */
377                 if (pmd_downgrade && dax_is_zero_entry(entry))
378                         unmap_mapping_range(mapping,
379                                 (index << PAGE_SHIFT) & PMD_MASK, PMD_SIZE, 0);
380 
381                 err = radix_tree_preload(
382                                 mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM);
383                 if (err) {
384                         if (pmd_downgrade)
385                                 put_locked_mapping_entry(mapping, index);
386                         return ERR_PTR(err);
387                 }
388                 spin_lock_irq(&mapping->tree_lock);
389 
390                 if (!entry) {
391                         /*
392                          * We needed to drop the page_tree lock while calling
393                          * radix_tree_preload() and we didn't have an entry to
394                          * lock.  See if another thread inserted an entry at
395                          * our index during this time.
396                          */
397                         entry = __radix_tree_lookup(&mapping->page_tree, index,
398                                         NULL, &slot);
399                         if (entry) {
400                                 radix_tree_preload_end();
401                                 spin_unlock_irq(&mapping->tree_lock);
402                                 goto restart;
403                         }
404                 }
405 
406                 if (pmd_downgrade) {
407                         radix_tree_delete(&mapping->page_tree, index);
408                         mapping->nrexceptional--;
409                         dax_wake_mapping_entry_waiter(mapping, index, entry,
410                                         true);
411                 }
412 
413                 entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY);
414 
415                 err = __radix_tree_insert(&mapping->page_tree, index,
416                                 dax_radix_order(entry), entry);
417                 radix_tree_preload_end();
418                 if (err) {
419                         spin_unlock_irq(&mapping->tree_lock);
420                         /*
421                          * Our insertion of a DAX entry failed, most likely
422                          * because we were inserting a PMD entry and it
423                          * collided with a PTE sized entry at a different
424                          * index in the PMD range.  We haven't inserted
425                          * anything into the radix tree and have no waiters to
426                          * wake.
427                          */
428                         return ERR_PTR(err);
429                 }
430                 /* Good, we have inserted empty locked entry into the tree. */
431                 mapping->nrexceptional++;
432                 spin_unlock_irq(&mapping->tree_lock);
433                 return entry;
434         }
435         entry = lock_slot(mapping, slot);
436  out_unlock:
437         spin_unlock_irq(&mapping->tree_lock);
438         return entry;
439 }
440 
441 static int __dax_invalidate_mapping_entry(struct address_space *mapping,
442                                           pgoff_t index, bool trunc)
443 {
444         int ret = 0;
445         void *entry;
446         struct radix_tree_root *page_tree = &mapping->page_tree;
447 
448         spin_lock_irq(&mapping->tree_lock);
449         entry = get_unlocked_mapping_entry(mapping, index, NULL);
450         if (!entry || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)))
451                 goto out;
452         if (!trunc &&
453             (radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_DIRTY) ||
454              radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE)))
455                 goto out;
456         radix_tree_delete(page_tree, index);
457         mapping->nrexceptional--;
458         ret = 1;
459 out:
460         put_unlocked_mapping_entry(mapping, index, entry);
461         spin_unlock_irq(&mapping->tree_lock);
462         return ret;
463 }
464 /*
465  * Delete exceptional DAX entry at @index from @mapping. Wait for radix tree
466  * entry to get unlocked before deleting it.
467  */
468 int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)
469 {
470         int ret = __dax_invalidate_mapping_entry(mapping, index, true);
471 
472         /*
473          * This gets called from truncate / punch_hole path. As such, the caller
474          * must hold locks protecting against concurrent modifications of the
475          * radix tree (usually fs-private i_mmap_sem for writing). Since the
476          * caller has seen exceptional entry for this index, we better find it
477          * at that index as well...
478          */
479         WARN_ON_ONCE(!ret);
480         return ret;
481 }
482 
483 /*
484  * Invalidate exceptional DAX entry if it is clean.
485  */
486 int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
487                                       pgoff_t index)
488 {
489         return __dax_invalidate_mapping_entry(mapping, index, false);
490 }
491 
492 static int copy_user_dax(struct block_device *bdev, struct dax_device *dax_dev,
493                 sector_t sector, size_t size, struct page *to,
494                 unsigned long vaddr)
495 {
496         void *vto, *kaddr;
497         pgoff_t pgoff;
498         pfn_t pfn;
499         long rc;
500         int id;
501 
502         rc = bdev_dax_pgoff(bdev, sector, size, &pgoff);
503         if (rc)
504                 return rc;
505 
506         id = dax_read_lock();
507         rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, &pfn);
508         if (rc < 0) {
509                 dax_read_unlock(id);
510                 return rc;
511         }
512         vto = kmap_atomic(to);
513         copy_user_page(vto, (void __force *)kaddr, vaddr, to);
514         kunmap_atomic(vto);
515         dax_read_unlock(id);
516         return 0;
517 }
518 
519 /*
520  * By this point grab_mapping_entry() has ensured that we have a locked entry
521  * of the appropriate size so we don't have to worry about downgrading PMDs to
522  * PTEs.  If we happen to be trying to insert a PTE and there is a PMD
523  * already in the tree, we will skip the insertion and just dirty the PMD as
524  * appropriate.
525  */
526 static void *dax_insert_mapping_entry(struct address_space *mapping,
527                                       struct vm_fault *vmf,
528                                       void *entry, sector_t sector,
529                                       unsigned long flags, bool dirty)
530 {
531         struct radix_tree_root *page_tree = &mapping->page_tree;
532         void *new_entry;
533         pgoff_t index = vmf->pgoff;
534 
535         if (dirty)
536                 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
537 
538         if (dax_is_zero_entry(entry) && !(flags & RADIX_DAX_ZERO_PAGE)) {
539                 /* we are replacing a zero page with block mapping */
540                 if (dax_is_pmd_entry(entry))
541                         unmap_mapping_range(mapping,
542                                         (vmf->pgoff << PAGE_SHIFT) & PMD_MASK,
543                                         PMD_SIZE, 0);
544                 else /* pte entry */
545                         unmap_mapping_range(mapping, vmf->pgoff << PAGE_SHIFT,
546                                         PAGE_SIZE, 0);
547         }
548 
549         spin_lock_irq(&mapping->tree_lock);
550         new_entry = dax_radix_locked_entry(sector, flags);
551 
552         if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) {
553                 /*
554                  * Only swap our new entry into the radix tree if the current
555                  * entry is a zero page or an empty entry.  If a normal PTE or
556                  * PMD entry is already in the tree, we leave it alone.  This
557                  * means that if we are trying to insert a PTE and the
558                  * existing entry is a PMD, we will just leave the PMD in the
559                  * tree and dirty it if necessary.
560                  */
561                 struct radix_tree_node *node;
562                 void **slot;
563                 void *ret;
564 
565                 ret = __radix_tree_lookup(page_tree, index, &node, &slot);
566                 WARN_ON_ONCE(ret != entry);
567                 __radix_tree_replace(page_tree, node, slot,
568                                      new_entry, NULL);
569                 entry = new_entry;
570         }
571 
572         if (dirty)
573                 radix_tree_tag_set(page_tree, index, PAGECACHE_TAG_DIRTY);
574 
575         spin_unlock_irq(&mapping->tree_lock);
576         return entry;
577 }
578 
579 static inline unsigned long
580 pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma)
581 {
582         unsigned long address;
583 
584         address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
585         VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
586         return address;
587 }
588 
589 /* Walk all mappings of a given index of a file and writeprotect them */
590 static void dax_mapping_entry_mkclean(struct address_space *mapping,
591                                       pgoff_t index, unsigned long pfn)
592 {
593         struct vm_area_struct *vma;
594         pte_t pte, *ptep = NULL;
595         pmd_t *pmdp = NULL;
596         spinlock_t *ptl;
597 
598         i_mmap_lock_read(mapping);
599         vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) {
600                 unsigned long address, start, end;
601 
602                 cond_resched();
603 
604                 if (!(vma->vm_flags & VM_SHARED))
605                         continue;
606 
607                 address = pgoff_address(index, vma);
608 
609                 /*
610                  * Note because we provide start/end to follow_pte_pmd it will
611                  * call mmu_notifier_invalidate_range_start() on our behalf
612                  * before taking any lock.
613                  */
614                 if (follow_pte_pmd(vma->vm_mm, address, &start, &end, &ptep, &pmdp, &ptl))
615                         continue;
616 
617                 /*
618                  * No need to call mmu_notifier_invalidate_range() as we are
619                  * downgrading page table protection not changing it to point
620                  * to a new page.
621                  *
622                  * See Documentation/vm/mmu_notifier.txt
623                  */
624                 if (pmdp) {
625 #ifdef CONFIG_FS_DAX_PMD
626                         pmd_t pmd;
627 
628                         if (pfn != pmd_pfn(*pmdp))
629                                 goto unlock_pmd;
630                         if (!pmd_dirty(*pmdp) && !pmd_write(*pmdp))
631                                 goto unlock_pmd;
632 
633                         flush_cache_page(vma, address, pfn);
634                         pmd = pmdp_huge_clear_flush(vma, address, pmdp);
635                         pmd = pmd_wrprotect(pmd);
636                         pmd = pmd_mkclean(pmd);
637                         set_pmd_at(vma->vm_mm, address, pmdp, pmd);
638 unlock_pmd:
639                         spin_unlock(ptl);
640 #endif
641                 } else {
642                         if (pfn != pte_pfn(*ptep))
643                                 goto unlock_pte;
644                         if (!pte_dirty(*ptep) && !pte_write(*ptep))
645                                 goto unlock_pte;
646 
647                         flush_cache_page(vma, address, pfn);
648                         pte = ptep_clear_flush(vma, address, ptep);
649                         pte = pte_wrprotect(pte);
650                         pte = pte_mkclean(pte);
651                         set_pte_at(vma->vm_mm, address, ptep, pte);
652 unlock_pte:
653                         pte_unmap_unlock(ptep, ptl);
654                 }
655 
656                 mmu_notifier_invalidate_range_end(vma->vm_mm, start, end);
657         }
658         i_mmap_unlock_read(mapping);
659 }
660 
661 static int dax_writeback_one(struct block_device *bdev,
662                 struct dax_device *dax_dev, struct address_space *mapping,
663                 pgoff_t index, void *entry)
664 {
665         struct radix_tree_root *page_tree = &mapping->page_tree;
666         void *entry2, **slot, *kaddr;
667         long ret = 0, id;
668         sector_t sector;
669         pgoff_t pgoff;
670         size_t size;
671         pfn_t pfn;
672 
673         /*
674          * A page got tagged dirty in DAX mapping? Something is seriously
675          * wrong.
676          */
677         if (WARN_ON(!radix_tree_exceptional_entry(entry)))
678                 return -EIO;
679 
680         spin_lock_irq(&mapping->tree_lock);
681         entry2 = get_unlocked_mapping_entry(mapping, index, &slot);
682         /* Entry got punched out / reallocated? */
683         if (!entry2 || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry2)))
684                 goto put_unlocked;
685         /*
686          * Entry got reallocated elsewhere? No need to writeback. We have to
687          * compare sectors as we must not bail out due to difference in lockbit
688          * or entry type.
689          */
690         if (dax_radix_sector(entry2) != dax_radix_sector(entry))
691                 goto put_unlocked;
692         if (WARN_ON_ONCE(dax_is_empty_entry(entry) ||
693                                 dax_is_zero_entry(entry))) {
694                 ret = -EIO;
695                 goto put_unlocked;
696         }
697 
698         /* Another fsync thread may have already written back this entry */
699         if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))
700                 goto put_unlocked;
701         /* Lock the entry to serialize with page faults */
702         entry = lock_slot(mapping, slot);
703         /*
704          * We can clear the tag now but we have to be careful so that concurrent
705          * dax_writeback_one() calls for the same index cannot finish before we
706          * actually flush the caches. This is achieved as the calls will look
707          * at the entry only under tree_lock and once they do that they will
708          * see the entry locked and wait for it to unlock.
709          */
710         radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_TOWRITE);
711         spin_unlock_irq(&mapping->tree_lock);
712 
713         /*
714          * Even if dax_writeback_mapping_range() was given a wbc->range_start
715          * in the middle of a PMD, the 'index' we are given will be aligned to
716          * the start index of the PMD, as will the sector we pull from
717          * 'entry'.  This allows us to flush for PMD_SIZE and not have to
718          * worry about partial PMD writebacks.
719          */
720         sector = dax_radix_sector(entry);
721         size = PAGE_SIZE << dax_radix_order(entry);
722 
723         id = dax_read_lock();
724         ret = bdev_dax_pgoff(bdev, sector, size, &pgoff);
725         if (ret)
726                 goto dax_unlock;
727 
728         /*
729          * dax_direct_access() may sleep, so cannot hold tree_lock over
730          * its invocation.
731          */
732         ret = dax_direct_access(dax_dev, pgoff, size / PAGE_SIZE, &kaddr, &pfn);
733         if (ret < 0)
734                 goto dax_unlock;
735 
736         if (WARN_ON_ONCE(ret < size / PAGE_SIZE)) {
737                 ret = -EIO;
738                 goto dax_unlock;
739         }
740 
741         dax_mapping_entry_mkclean(mapping, index, pfn_t_to_pfn(pfn));
742         dax_flush(dax_dev, kaddr, size);
743         /*
744          * After we have flushed the cache, we can clear the dirty tag. There
745          * cannot be new dirty data in the pfn after the flush has completed as
746          * the pfn mappings are writeprotected and fault waits for mapping
747          * entry lock.
748          */
749         spin_lock_irq(&mapping->tree_lock);
750         radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_DIRTY);
751         spin_unlock_irq(&mapping->tree_lock);
752         trace_dax_writeback_one(mapping->host, index, size >> PAGE_SHIFT);
753  dax_unlock:
754         dax_read_unlock(id);
755         put_locked_mapping_entry(mapping, index);
756         return ret;
757 
758  put_unlocked:
759         put_unlocked_mapping_entry(mapping, index, entry2);
760         spin_unlock_irq(&mapping->tree_lock);
761         return ret;
762 }
763 
764 /*
765  * Flush the mapping to the persistent domain within the byte range of [start,
766  * end]. This is required by data integrity operations to ensure file data is
767  * on persistent storage prior to completion of the operation.
768  */
769 int dax_writeback_mapping_range(struct address_space *mapping,
770                 struct block_device *bdev, struct writeback_control *wbc)
771 {
772         struct inode *inode = mapping->host;
773         pgoff_t start_index, end_index;
774         pgoff_t indices[PAGEVEC_SIZE];
775         struct dax_device *dax_dev;
776         struct pagevec pvec;
777         bool done = false;
778         int i, ret = 0;
779 
780         if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
781                 return -EIO;
782 
783         if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL)
784                 return 0;
785 
786         dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
787         if (!dax_dev)
788                 return -EIO;
789 
790         start_index = wbc->range_start >> PAGE_SHIFT;
791         end_index = wbc->range_end >> PAGE_SHIFT;
792 
793         trace_dax_writeback_range(inode, start_index, end_index);
794 
795         tag_pages_for_writeback(mapping, start_index, end_index);
796 
797         pagevec_init(&pvec);
798         while (!done) {
799                 pvec.nr = find_get_entries_tag(mapping, start_index,
800                                 PAGECACHE_TAG_TOWRITE, PAGEVEC_SIZE,
801                                 pvec.pages, indices);
802 
803                 if (pvec.nr == 0)
804                         break;
805 
806                 for (i = 0; i < pvec.nr; i++) {
807                         if (indices[i] > end_index) {
808                                 done = true;
809                                 break;
810                         }
811 
812                         ret = dax_writeback_one(bdev, dax_dev, mapping,
813                                         indices[i], pvec.pages[i]);
814                         if (ret < 0) {
815                                 mapping_set_error(mapping, ret);
816                                 goto out;
817                         }
818                 }
819                 start_index = indices[pvec.nr - 1] + 1;
820         }
821 out:
822         put_dax(dax_dev);
823         trace_dax_writeback_range_done(inode, start_index, end_index);
824         return (ret < 0 ? ret : 0);
825 }
826 EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);
827 
828 static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos)
829 {
830         return (iomap->addr + (pos & PAGE_MASK) - iomap->offset) >> 9;
831 }
832 
833 static int dax_iomap_pfn(struct iomap *iomap, loff_t pos, size_t size,
834                          pfn_t *pfnp)
835 {
836         const sector_t sector = dax_iomap_sector(iomap, pos);
837         pgoff_t pgoff;
838         void *kaddr;
839         int id, rc;
840         long length;
841 
842         rc = bdev_dax_pgoff(iomap->bdev, sector, size, &pgoff);
843         if (rc)
844                 return rc;
845         id = dax_read_lock();
846         length = dax_direct_access(iomap->dax_dev, pgoff, PHYS_PFN(size),
847                                    &kaddr, pfnp);
848         if (length < 0) {
849                 rc = length;
850                 goto out;
851         }
852         rc = -EINVAL;
853         if (PFN_PHYS(length) < size)
854                 goto out;
855         if (pfn_t_to_pfn(*pfnp) & (PHYS_PFN(size)-1))
856                 goto out;
857         /* For larger pages we need devmap */
858         if (length > 1 && !pfn_t_devmap(*pfnp))
859                 goto out;
860         rc = 0;
861 out:
862         dax_read_unlock(id);
863         return rc;
864 }
865 
866 /*
867  * The user has performed a load from a hole in the file.  Allocating a new
868  * page in the file would cause excessive storage usage for workloads with
869  * sparse files.  Instead we insert a read-only mapping of the 4k zero page.
870  * If this page is ever written to we will re-fault and change the mapping to
871  * point to real DAX storage instead.
872  */
873 static int dax_load_hole(struct address_space *mapping, void *entry,
874                          struct vm_fault *vmf)
875 {
876         struct inode *inode = mapping->host;
877         unsigned long vaddr = vmf->address;
878         int ret = VM_FAULT_NOPAGE;
879         struct page *zero_page;
880         void *entry2;
881 
882         zero_page = ZERO_PAGE(0);
883         if (unlikely(!zero_page)) {
884                 ret = VM_FAULT_OOM;
885                 goto out;
886         }
887 
888         entry2 = dax_insert_mapping_entry(mapping, vmf, entry, 0,
889                         RADIX_DAX_ZERO_PAGE, false);
890         if (IS_ERR(entry2)) {
891                 ret = VM_FAULT_SIGBUS;
892                 goto out;
893         }
894 
895         vm_insert_mixed(vmf->vma, vaddr, page_to_pfn_t(zero_page));
896 out:
897         trace_dax_load_hole(inode, vmf, ret);
898         return ret;
899 }
900 
901 static bool dax_range_is_aligned(struct block_device *bdev,
902                                  unsigned int offset, unsigned int length)
903 {
904         unsigned short sector_size = bdev_logical_block_size(bdev);
905 
906         if (!IS_ALIGNED(offset, sector_size))
907                 return false;
908         if (!IS_ALIGNED(length, sector_size))
909                 return false;
910 
911         return true;
912 }
913 
914 int __dax_zero_page_range(struct block_device *bdev,
915                 struct dax_device *dax_dev, sector_t sector,
916                 unsigned int offset, unsigned int size)
917 {
918         if (dax_range_is_aligned(bdev, offset, size)) {
919                 sector_t start_sector = sector + (offset >> 9);
920 
921                 return blkdev_issue_zeroout(bdev, start_sector,
922                                 size >> 9, GFP_NOFS, 0);
923         } else {
924                 pgoff_t pgoff;
925                 long rc, id;
926                 void *kaddr;
927                 pfn_t pfn;
928 
929                 rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff);
930                 if (rc)
931                         return rc;
932 
933                 id = dax_read_lock();
934                 rc = dax_direct_access(dax_dev, pgoff, 1, &kaddr,
935                                 &pfn);
936                 if (rc < 0) {
937                         dax_read_unlock(id);
938                         return rc;
939                 }
940                 memset(kaddr + offset, 0, size);
941                 dax_flush(dax_dev, kaddr + offset, size);
942                 dax_read_unlock(id);
943         }
944         return 0;
945 }
946 EXPORT_SYMBOL_GPL(__dax_zero_page_range);
947 
948 static loff_t
949 dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
950                 struct iomap *iomap)
951 {
952         struct block_device *bdev = iomap->bdev;
953         struct dax_device *dax_dev = iomap->dax_dev;
954         struct iov_iter *iter = data;
955         loff_t end = pos + length, done = 0;
956         ssize_t ret = 0;
957         int id;
958 
959         if (iov_iter_rw(iter) == READ) {
960                 end = min(end, i_size_read(inode));
961                 if (pos >= end)
962                         return 0;
963 
964                 if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN)
965                         return iov_iter_zero(min(length, end - pos), iter);
966         }
967 
968         if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED))
969                 return -EIO;
970 
971         /*
972          * Write can allocate block for an area which has a hole page mapped
973          * into page tables. We have to tear down these mappings so that data
974          * written by write(2) is visible in mmap.
975          */
976         if (iomap->flags & IOMAP_F_NEW) {
977                 invalidate_inode_pages2_range(inode->i_mapping,
978                                               pos >> PAGE_SHIFT,
979                                               (end - 1) >> PAGE_SHIFT);
980         }
981 
982         id = dax_read_lock();
983         while (pos < end) {
984                 unsigned offset = pos & (PAGE_SIZE - 1);
985                 const size_t size = ALIGN(length + offset, PAGE_SIZE);
986                 const sector_t sector = dax_iomap_sector(iomap, pos);
987                 ssize_t map_len;
988                 pgoff_t pgoff;
989                 void *kaddr;
990                 pfn_t pfn;
991 
992                 if (fatal_signal_pending(current)) {
993                         ret = -EINTR;
994                         break;
995                 }
996 
997                 ret = bdev_dax_pgoff(bdev, sector, size, &pgoff);
998                 if (ret)
999                         break;
1000 
1001                 map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size),
1002                                 &kaddr, &pfn);
1003                 if (map_len < 0) {
1004                         ret = map_len;
1005                         break;
1006                 }
1007 
1008                 map_len = PFN_PHYS(map_len);
1009                 kaddr += offset;
1010                 map_len -= offset;
1011                 if (map_len > end - pos)
1012                         map_len = end - pos;
1013 
1014                 /*
1015                  * The userspace address for the memory copy has already been
1016                  * validated via access_ok() in either vfs_read() or
1017                  * vfs_write(), depending on which operation we are doing.
1018                  */
1019                 if (iov_iter_rw(iter) == WRITE)
1020                         map_len = dax_copy_from_iter(dax_dev, pgoff, kaddr,
1021                                         map_len, iter);
1022                 else
1023                         map_len = copy_to_iter(kaddr, map_len, iter);
1024                 if (map_len <= 0) {
1025                         ret = map_len ? map_len : -EFAULT;
1026                         break;
1027                 }
1028 
1029                 pos += map_len;
1030                 length -= map_len;
1031                 done += map_len;
1032         }
1033         dax_read_unlock(id);
1034 
1035         return done ? done : ret;
1036 }
1037 
1038 /**
1039  * dax_iomap_rw - Perform I/O to a DAX file
1040  * @iocb:       The control block for this I/O
1041  * @iter:       The addresses to do I/O from or to
1042  * @ops:        iomap ops passed from the file system
1043  *
1044  * This function performs read and write operations to directly mapped
1045  * persistent memory.  The callers needs to take care of read/write exclusion
1046  * and evicting any page cache pages in the region under I/O.
1047  */
1048 ssize_t
1049 dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
1050                 const struct iomap_ops *ops)
1051 {
1052         struct address_space *mapping = iocb->ki_filp->f_mapping;
1053         struct inode *inode = mapping->host;
1054         loff_t pos = iocb->ki_pos, ret = 0, done = 0;
1055         unsigned flags = 0;
1056 
1057         if (iov_iter_rw(iter) == WRITE) {
1058                 lockdep_assert_held_exclusive(&inode->i_rwsem);
1059                 flags |= IOMAP_WRITE;
1060         } else {
1061                 lockdep_assert_held(&inode->i_rwsem);
1062         }
1063 
1064         while (iov_iter_count(iter)) {
1065                 ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops,
1066                                 iter, dax_iomap_actor);
1067                 if (ret <= 0)
1068                         break;
1069                 pos += ret;
1070                 done += ret;
1071         }
1072 
1073         iocb->ki_pos += done;
1074         return done ? done : ret;
1075 }
1076 EXPORT_SYMBOL_GPL(dax_iomap_rw);
1077 
1078 static int dax_fault_return(int error)
1079 {
1080         if (error == 0)
1081                 return VM_FAULT_NOPAGE;
1082         if (error == -ENOMEM)
1083                 return VM_FAULT_OOM;
1084         return VM_FAULT_SIGBUS;
1085 }
1086 
1087 /*
1088  * MAP_SYNC on a dax mapping guarantees dirty metadata is
1089  * flushed on write-faults (non-cow), but not read-faults.
1090  */
1091 static bool dax_fault_is_synchronous(unsigned long flags,
1092                 struct vm_area_struct *vma, struct iomap *iomap)
1093 {
1094         return (flags & IOMAP_WRITE) && (vma->vm_flags & VM_SYNC)
1095                 && (iomap->flags & IOMAP_F_DIRTY);
1096 }
1097 
1098 static int dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp,
1099                                const struct iomap_ops *ops)
1100 {
1101         struct vm_area_struct *vma = vmf->vma;
1102         struct address_space *mapping = vma->vm_file->f_mapping;
1103         struct inode *inode = mapping->host;
1104         unsigned long vaddr = vmf->address;
1105         loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT;
1106         struct iomap iomap = { 0 };
1107         unsigned flags = IOMAP_FAULT;
1108         int error, major = 0;
1109         bool write = vmf->flags & FAULT_FLAG_WRITE;
1110         bool sync;
1111         int vmf_ret = 0;
1112         void *entry;
1113         pfn_t pfn;
1114 
1115         trace_dax_pte_fault(inode, vmf, vmf_ret);
1116         /*
1117          * Check whether offset isn't beyond end of file now. Caller is supposed
1118          * to hold locks serializing us with truncate / punch hole so this is
1119          * a reliable test.
1120          */
1121         if (pos >= i_size_read(inode)) {
1122                 vmf_ret = VM_FAULT_SIGBUS;
1123                 goto out;
1124         }
1125 
1126         if (write && !vmf->cow_page)
1127                 flags |= IOMAP_WRITE;
1128 
1129         entry = grab_mapping_entry(mapping, vmf->pgoff, 0);
1130         if (IS_ERR(entry)) {
1131                 vmf_ret = dax_fault_return(PTR_ERR(entry));
1132                 goto out;
1133         }
1134 
1135         /*
1136          * It is possible, particularly with mixed reads & writes to private
1137          * mappings, that we have raced with a PMD fault that overlaps with
1138          * the PTE we need to set up.  If so just return and the fault will be
1139          * retried.
1140          */
1141         if (pmd_trans_huge(*vmf->pmd) || pmd_devmap(*vmf->pmd)) {
1142                 vmf_ret = VM_FAULT_NOPAGE;
1143                 goto unlock_entry;
1144         }
1145 
1146         /*
1147          * Note that we don't bother to use iomap_apply here: DAX required
1148          * the file system block size to be equal the page size, which means
1149          * that we never have to deal with more than a single extent here.
1150          */
1151         error = ops->iomap_begin(inode, pos, PAGE_SIZE, flags, &iomap);
1152         if (error) {
1153                 vmf_ret = dax_fault_return(error);
1154                 goto unlock_entry;
1155         }
1156         if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) {
1157                 error = -EIO;   /* fs corruption? */
1158                 goto error_finish_iomap;
1159         }
1160 
1161         if (vmf->cow_page) {
1162                 sector_t sector = dax_iomap_sector(&iomap, pos);
1163 
1164                 switch (iomap.type) {
1165                 case IOMAP_HOLE:
1166                 case IOMAP_UNWRITTEN:
1167                         clear_user_highpage(vmf->cow_page, vaddr);
1168                         break;
1169                 case IOMAP_MAPPED:
1170                         error = copy_user_dax(iomap.bdev, iomap.dax_dev,
1171                                         sector, PAGE_SIZE, vmf->cow_page, vaddr);
1172                         break;
1173                 default:
1174                         WARN_ON_ONCE(1);
1175                         error = -EIO;
1176                         break;
1177                 }
1178 
1179                 if (error)
1180                         goto error_finish_iomap;
1181 
1182                 __SetPageUptodate(vmf->cow_page);
1183                 vmf_ret = finish_fault(vmf);
1184                 if (!vmf_ret)
1185                         vmf_ret = VM_FAULT_DONE_COW;
1186                 goto finish_iomap;
1187         }
1188 
1189         sync = dax_fault_is_synchronous(flags, vma, &iomap);
1190 
1191         switch (iomap.type) {
1192         case IOMAP_MAPPED:
1193                 if (iomap.flags & IOMAP_F_NEW) {
1194                         count_vm_event(PGMAJFAULT);
1195                         count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
1196                         major = VM_FAULT_MAJOR;
1197                 }
1198                 error = dax_iomap_pfn(&iomap, pos, PAGE_SIZE, &pfn);
1199                 if (error < 0)
1200                         goto error_finish_iomap;
1201 
1202                 entry = dax_insert_mapping_entry(mapping, vmf, entry,
1203                                                  dax_iomap_sector(&iomap, pos),
1204                                                  0, write && !sync);
1205                 if (IS_ERR(entry)) {
1206                         error = PTR_ERR(entry);
1207                         goto error_finish_iomap;
1208                 }
1209 
1210                 /*
1211                  * If we are doing synchronous page fault and inode needs fsync,
1212                  * we can insert PTE into page tables only after that happens.
1213                  * Skip insertion for now and return the pfn so that caller can
1214                  * insert it after fsync is done.
1215                  */
1216                 if (sync) {
1217                         if (WARN_ON_ONCE(!pfnp)) {
1218                                 error = -EIO;
1219                                 goto error_finish_iomap;
1220                         }
1221                         *pfnp = pfn;
1222                         vmf_ret = VM_FAULT_NEEDDSYNC | major;
1223                         goto finish_iomap;
1224                 }
1225                 trace_dax_insert_mapping(inode, vmf, entry);
1226                 if (write)
1227                         error = vm_insert_mixed_mkwrite(vma, vaddr, pfn);
1228                 else
1229                         error = vm_insert_mixed(vma, vaddr, pfn);
1230 
1231                 /* -EBUSY is fine, somebody else faulted on the same PTE */
1232                 if (error == -EBUSY)
1233                         error = 0;
1234                 break;
1235         case IOMAP_UNWRITTEN:
1236         case IOMAP_HOLE:
1237                 if (!write) {
1238                         vmf_ret = dax_load_hole(mapping, entry, vmf);
1239                         goto finish_iomap;
1240                 }
1241                 /*FALLTHRU*/
1242         default:
1243                 WARN_ON_ONCE(1);
1244                 error = -EIO;
1245                 break;
1246         }
1247 
1248  error_finish_iomap:
1249         vmf_ret = dax_fault_return(error) | major;
1250  finish_iomap:
1251         if (ops->iomap_end) {
1252                 int copied = PAGE_SIZE;
1253 
1254                 if (vmf_ret & VM_FAULT_ERROR)
1255                         copied = 0;
1256                 /*
1257                  * The fault is done by now and there's no way back (other
1258                  * thread may be already happily using PTE we have installed).
1259                  * Just ignore error from ->iomap_end since we cannot do much
1260                  * with it.
1261                  */
1262                 ops->iomap_end(inode, pos, PAGE_SIZE, copied, flags, &iomap);
1263         }
1264  unlock_entry:
1265         put_locked_mapping_entry(mapping, vmf->pgoff);
1266  out:
1267         trace_dax_pte_fault_done(inode, vmf, vmf_ret);
1268         return vmf_ret;
1269 }
1270 
1271 #ifdef CONFIG_FS_DAX_PMD
1272 /*
1273  * The 'colour' (ie low bits) within a PMD of a page offset.  This comes up
1274  * more often than one might expect in the below functions.
1275  */
1276 #define PG_PMD_COLOUR   ((PMD_SIZE >> PAGE_SHIFT) - 1)
1277 
1278 static int dax_pmd_load_hole(struct vm_fault *vmf, struct iomap *iomap,
1279                 void *entry)
1280 {
1281         struct address_space *mapping = vmf->vma->vm_file->f_mapping;
1282         unsigned long pmd_addr = vmf->address & PMD_MASK;
1283         struct inode *inode = mapping->host;
1284         struct page *zero_page;
1285         void *ret = NULL;
1286         spinlock_t *ptl;
1287         pmd_t pmd_entry;
1288 
1289         zero_page = mm_get_huge_zero_page(vmf->vma->vm_mm);
1290 
1291         if (unlikely(!zero_page))
1292                 goto fallback;
1293 
1294         ret = dax_insert_mapping_entry(mapping, vmf, entry, 0,
1295                         RADIX_DAX_PMD | RADIX_DAX_ZERO_PAGE, false);
1296         if (IS_ERR(ret))
1297                 goto fallback;
1298 
1299         ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);
1300         if (!pmd_none(*(vmf->pmd))) {
1301                 spin_unlock(ptl);
1302                 goto fallback;
1303         }
1304 
1305         pmd_entry = mk_pmd(zero_page, vmf->vma->vm_page_prot);
1306         pmd_entry = pmd_mkhuge(pmd_entry);
1307         set_pmd_at(vmf->vma->vm_mm, pmd_addr, vmf->pmd, pmd_entry);
1308         spin_unlock(ptl);
1309         trace_dax_pmd_load_hole(inode, vmf, zero_page, ret);
1310         return VM_FAULT_NOPAGE;
1311 
1312 fallback:
1313         trace_dax_pmd_load_hole_fallback(inode, vmf, zero_page, ret);
1314         return VM_FAULT_FALLBACK;
1315 }
1316 
1317 static int dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
1318                                const struct iomap_ops *ops)
1319 {
1320         struct vm_area_struct *vma = vmf->vma;
1321         struct address_space *mapping = vma->vm_file->f_mapping;
1322         unsigned long pmd_addr = vmf->address & PMD_MASK;
1323         bool write = vmf->flags & FAULT_FLAG_WRITE;
1324         bool sync;
1325         unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT;
1326         struct inode *inode = mapping->host;
1327         int result = VM_FAULT_FALLBACK;
1328         struct iomap iomap = { 0 };
1329         pgoff_t max_pgoff, pgoff;
1330         void *entry;
1331         loff_t pos;
1332         int error;
1333         pfn_t pfn;
1334 
1335         /*
1336          * Check whether offset isn't beyond end of file now. Caller is
1337          * supposed to hold locks serializing us with truncate / punch hole so
1338          * this is a reliable test.
1339          */
1340         pgoff = linear_page_index(vma, pmd_addr);
1341         max_pgoff = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
1342 
1343         trace_dax_pmd_fault(inode, vmf, max_pgoff, 0);
1344 
1345         /*
1346          * Make sure that the faulting address's PMD offset (color) matches
1347          * the PMD offset from the start of the file.  This is necessary so
1348          * that a PMD range in the page table overlaps exactly with a PMD
1349          * range in the radix tree.
1350          */
1351         if ((vmf->pgoff & PG_PMD_COLOUR) !=
1352             ((vmf->address >> PAGE_SHIFT) & PG_PMD_COLOUR))
1353                 goto fallback;
1354 
1355         /* Fall back to PTEs if we're going to COW */
1356         if (write && !(vma->vm_flags & VM_SHARED))
1357                 goto fallback;
1358 
1359         /* If the PMD would extend outside the VMA */
1360         if (pmd_addr < vma->vm_start)
1361                 goto fallback;
1362         if ((pmd_addr + PMD_SIZE) > vma->vm_end)
1363                 goto fallback;
1364 
1365         if (pgoff >= max_pgoff) {
1366                 result = VM_FAULT_SIGBUS;
1367                 goto out;
1368         }
1369 
1370         /* If the PMD would extend beyond the file size */
1371         if ((pgoff | PG_PMD_COLOUR) >= max_pgoff)
1372                 goto fallback;
1373 
1374         /*
1375          * grab_mapping_entry() will make sure we get a 2MiB empty entry, a
1376          * 2MiB zero page entry or a DAX PMD.  If it can't (because a 4k page
1377          * is already in the tree, for instance), it will return -EEXIST and
1378          * we just fall back to 4k entries.
1379          */
1380         entry = grab_mapping_entry(mapping, pgoff, RADIX_DAX_PMD);
1381         if (IS_ERR(entry))
1382                 goto fallback;
1383 
1384         /*
1385          * It is possible, particularly with mixed reads & writes to private
1386          * mappings, that we have raced with a PTE fault that overlaps with
1387          * the PMD we need to set up.  If so just return and the fault will be
1388          * retried.
1389          */
1390         if (!pmd_none(*vmf->pmd) && !pmd_trans_huge(*vmf->pmd) &&
1391                         !pmd_devmap(*vmf->pmd)) {
1392                 result = 0;
1393                 goto unlock_entry;
1394         }
1395 
1396         /*
1397          * Note that we don't use iomap_apply here.  We aren't doing I/O, only
1398          * setting up a mapping, so really we're using iomap_begin() as a way
1399          * to look up our filesystem block.
1400          */
1401         pos = (loff_t)pgoff << PAGE_SHIFT;
1402         error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap);
1403         if (error)
1404                 goto unlock_entry;
1405 
1406         if (iomap.offset + iomap.length < pos + PMD_SIZE)
1407                 goto finish_iomap;
1408 
1409         sync = dax_fault_is_synchronous(iomap_flags, vma, &iomap);
1410 
1411         switch (iomap.type) {
1412         case IOMAP_MAPPED:
1413                 error = dax_iomap_pfn(&iomap, pos, PMD_SIZE, &pfn);
1414                 if (error < 0)
1415                         goto finish_iomap;
1416 
1417                 entry = dax_insert_mapping_entry(mapping, vmf, entry,
1418                                                 dax_iomap_sector(&iomap, pos),
1419                                                 RADIX_DAX_PMD, write && !sync);
1420                 if (IS_ERR(entry))
1421                         goto finish_iomap;
1422 
1423                 /*
1424                  * If we are doing synchronous page fault and inode needs fsync,
1425                  * we can insert PMD into page tables only after that happens.
1426                  * Skip insertion for now and return the pfn so that caller can
1427                  * insert it after fsync is done.
1428                  */
1429                 if (sync) {
1430                         if (WARN_ON_ONCE(!pfnp))
1431                                 goto finish_iomap;
1432                         *pfnp = pfn;
1433                         result = VM_FAULT_NEEDDSYNC;
1434                         goto finish_iomap;
1435                 }
1436 
1437                 trace_dax_pmd_insert_mapping(inode, vmf, PMD_SIZE, pfn, entry);
1438                 result = vmf_insert_pfn_pmd(vma, vmf->address, vmf->pmd, pfn,
1439                                             write);
1440                 break;
1441         case IOMAP_UNWRITTEN:
1442         case IOMAP_HOLE:
1443                 if (WARN_ON_ONCE(write))
1444                         break;
1445                 result = dax_pmd_load_hole(vmf, &iomap, entry);
1446                 break;
1447         default:
1448                 WARN_ON_ONCE(1);
1449                 break;
1450         }
1451 
1452  finish_iomap:
1453         if (ops->iomap_end) {
1454                 int copied = PMD_SIZE;
1455 
1456                 if (result == VM_FAULT_FALLBACK)
1457                         copied = 0;
1458                 /*
1459                  * The fault is done by now and there's no way back (other
1460                  * thread may be already happily using PMD we have installed).
1461                  * Just ignore error from ->iomap_end since we cannot do much
1462                  * with it.
1463                  */
1464                 ops->iomap_end(inode, pos, PMD_SIZE, copied, iomap_flags,
1465                                 &iomap);
1466         }
1467  unlock_entry:
1468         put_locked_mapping_entry(mapping, pgoff);
1469  fallback:
1470         if (result == VM_FAULT_FALLBACK) {
1471                 split_huge_pmd(vma, vmf->pmd, vmf->address);
1472                 count_vm_event(THP_FAULT_FALLBACK);
1473         }
1474 out:
1475         trace_dax_pmd_fault_done(inode, vmf, max_pgoff, result);
1476         return result;
1477 }
1478 #else
1479 static int dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
1480                                const struct iomap_ops *ops)
1481 {
1482         return VM_FAULT_FALLBACK;
1483 }
1484 #endif /* CONFIG_FS_DAX_PMD */
1485 
1486 /**
1487  * dax_iomap_fault - handle a page fault on a DAX file
1488  * @vmf: The description of the fault
1489  * @pe_size: Size of the page to fault in
1490  * @pfnp: PFN to insert for synchronous faults if fsync is required
1491  * @ops: Iomap ops passed from the file system
1492  *
1493  * When a page fault occurs, filesystems may call this helper in
1494  * their fault handler for DAX files. dax_iomap_fault() assumes the caller
1495  * has done all the necessary locking for page fault to proceed
1496  * successfully.
1497  */
1498 int dax_iomap_fault(struct vm_fault *vmf, enum page_entry_size pe_size,
1499                     pfn_t *pfnp, const struct iomap_ops *ops)
1500 {
1501         switch (pe_size) {
1502         case PE_SIZE_PTE:
1503                 return dax_iomap_pte_fault(vmf, pfnp, ops);
1504         case PE_SIZE_PMD:
1505                 return dax_iomap_pmd_fault(vmf, pfnp, ops);
1506         default:
1507                 return VM_FAULT_FALLBACK;
1508         }
1509 }
1510 EXPORT_SYMBOL_GPL(dax_iomap_fault);
1511 
1512 /**
1513  * dax_insert_pfn_mkwrite - insert PTE or PMD entry into page tables
1514  * @vmf: The description of the fault
1515  * @pe_size: Size of entry to be inserted
1516  * @pfn: PFN to insert
1517  *
1518  * This function inserts writeable PTE or PMD entry into page tables for mmaped
1519  * DAX file.  It takes care of marking corresponding radix tree entry as dirty
1520  * as well.
1521  */
1522 static int dax_insert_pfn_mkwrite(struct vm_fault *vmf,
1523                                   enum page_entry_size pe_size,
1524                                   pfn_t pfn)
1525 {
1526         struct address_space *mapping = vmf->vma->vm_file->f_mapping;
1527         void *entry, **slot;
1528         pgoff_t index = vmf->pgoff;
1529         int vmf_ret, error;
1530 
1531         spin_lock_irq(&mapping->tree_lock);
1532         entry = get_unlocked_mapping_entry(mapping, index, &slot);
1533         /* Did we race with someone splitting entry or so? */
1534         if (!entry ||
1535             (pe_size == PE_SIZE_PTE && !dax_is_pte_entry(entry)) ||
1536             (pe_size == PE_SIZE_PMD && !dax_is_pmd_entry(entry))) {
1537                 put_unlocked_mapping_entry(mapping, index, entry);
1538                 spin_unlock_irq(&mapping->tree_lock);
1539                 trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf,
1540                                                       VM_FAULT_NOPAGE);
1541                 return VM_FAULT_NOPAGE;
1542         }
1543         radix_tree_tag_set(&mapping->page_tree, index, PAGECACHE_TAG_DIRTY);
1544         entry = lock_slot(mapping, slot);
1545         spin_unlock_irq(&mapping->tree_lock);
1546         switch (pe_size) {
1547         case PE_SIZE_PTE:
1548                 error = vm_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn);
1549                 vmf_ret = dax_fault_return(error);
1550                 break;
1551 #ifdef CONFIG_FS_DAX_PMD
1552         case PE_SIZE_PMD:
1553                 vmf_ret = vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd,
1554                         pfn, true);
1555                 break;
1556 #endif
1557         default:
1558                 vmf_ret = VM_FAULT_FALLBACK;
1559         }
1560         put_locked_mapping_entry(mapping, index);
1561         trace_dax_insert_pfn_mkwrite(mapping->host, vmf, vmf_ret);
1562         return vmf_ret;
1563 }
1564 
1565 /**
1566  * dax_finish_sync_fault - finish synchronous page fault
1567  * @vmf: The description of the fault
1568  * @pe_size: Size of entry to be inserted
1569  * @pfn: PFN to insert
1570  *
1571  * This function ensures that the file range touched by the page fault is
1572  * stored persistently on the media and handles inserting of appropriate page
1573  * table entry.
1574  */
1575 int dax_finish_sync_fault(struct vm_fault *vmf, enum page_entry_size pe_size,
1576                           pfn_t pfn)
1577 {
1578         int err;
1579         loff_t start = ((loff_t)vmf->pgoff) << PAGE_SHIFT;
1580         size_t len = 0;
1581 
1582         if (pe_size == PE_SIZE_PTE)
1583                 len = PAGE_SIZE;
1584         else if (pe_size == PE_SIZE_PMD)
1585                 len = PMD_SIZE;
1586         else
1587                 WARN_ON_ONCE(1);
1588         err = vfs_fsync_range(vmf->vma->vm_file, start, start + len - 1, 1);
1589         if (err)
1590                 return VM_FAULT_SIGBUS;
1591         return dax_insert_pfn_mkwrite(vmf, pe_size, pfn);
1592 }
1593 EXPORT_SYMBOL_GPL(dax_finish_sync_fault);
1594 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp