~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/shmem.c

Version: ~ [ linux-5.4-rc3 ] ~ [ linux-5.3.6 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.79 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.149 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.196 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.196 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.75 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Resizable virtual memory filesystem for Linux.
  3  *
  4  * Copyright (C) 2000 Linus Torvalds.
  5  *               2000 Transmeta Corp.
  6  *               2000-2001 Christoph Rohland
  7  *               2000-2001 SAP AG
  8  *               2002 Red Hat Inc.
  9  * Copyright (C) 2002-2011 Hugh Dickins.
 10  * Copyright (C) 2011 Google Inc.
 11  * Copyright (C) 2002-2005 VERITAS Software Corporation.
 12  * Copyright (C) 2004 Andi Kleen, SuSE Labs
 13  *
 14  * Extended attribute support for tmpfs:
 15  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
 16  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
 17  *
 18  * tiny-shmem:
 19  * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
 20  *
 21  * This file is released under the GPL.
 22  */
 23 
 24 #include <linux/fs.h>
 25 #include <linux/init.h>
 26 #include <linux/vfs.h>
 27 #include <linux/mount.h>
 28 #include <linux/ramfs.h>
 29 #include <linux/pagemap.h>
 30 #include <linux/file.h>
 31 #include <linux/mm.h>
 32 #include <linux/sched/signal.h>
 33 #include <linux/export.h>
 34 #include <linux/swap.h>
 35 #include <linux/uio.h>
 36 #include <linux/khugepaged.h>
 37 
 38 #include <asm/tlbflush.h> /* for arch/microblaze update_mmu_cache() */
 39 
 40 static struct vfsmount *shm_mnt;
 41 
 42 #ifdef CONFIG_SHMEM
 43 /*
 44  * This virtual memory filesystem is heavily based on the ramfs. It
 45  * extends ramfs by the ability to use swap and honor resource limits
 46  * which makes it a completely usable filesystem.
 47  */
 48 
 49 #include <linux/xattr.h>
 50 #include <linux/exportfs.h>
 51 #include <linux/posix_acl.h>
 52 #include <linux/posix_acl_xattr.h>
 53 #include <linux/mman.h>
 54 #include <linux/string.h>
 55 #include <linux/slab.h>
 56 #include <linux/backing-dev.h>
 57 #include <linux/shmem_fs.h>
 58 #include <linux/writeback.h>
 59 #include <linux/blkdev.h>
 60 #include <linux/pagevec.h>
 61 #include <linux/percpu_counter.h>
 62 #include <linux/falloc.h>
 63 #include <linux/splice.h>
 64 #include <linux/security.h>
 65 #include <linux/swapops.h>
 66 #include <linux/mempolicy.h>
 67 #include <linux/namei.h>
 68 #include <linux/ctype.h>
 69 #include <linux/migrate.h>
 70 #include <linux/highmem.h>
 71 #include <linux/seq_file.h>
 72 #include <linux/magic.h>
 73 #include <linux/syscalls.h>
 74 #include <linux/fcntl.h>
 75 #include <uapi/linux/memfd.h>
 76 #include <linux/userfaultfd_k.h>
 77 #include <linux/rmap.h>
 78 
 79 #include <linux/uaccess.h>
 80 #include <asm/pgtable.h>
 81 
 82 #include "internal.h"
 83 
 84 #define BLOCKS_PER_PAGE  (PAGE_SIZE/512)
 85 #define VM_ACCT(size)    (PAGE_ALIGN(size) >> PAGE_SHIFT)
 86 
 87 /* Pretend that each entry is of this size in directory's i_size */
 88 #define BOGO_DIRENT_SIZE 20
 89 
 90 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */
 91 #define SHORT_SYMLINK_LEN 128
 92 
 93 /*
 94  * shmem_fallocate communicates with shmem_fault or shmem_writepage via
 95  * inode->i_private (with i_mutex making sure that it has only one user at
 96  * a time): we would prefer not to enlarge the shmem inode just for that.
 97  */
 98 struct shmem_falloc {
 99         wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
100         pgoff_t start;          /* start of range currently being fallocated */
101         pgoff_t next;           /* the next page offset to be fallocated */
102         pgoff_t nr_falloced;    /* how many new pages have been fallocated */
103         pgoff_t nr_unswapped;   /* how often writepage refused to swap out */
104 };
105 
106 #ifdef CONFIG_TMPFS
107 static unsigned long shmem_default_max_blocks(void)
108 {
109         return totalram_pages / 2;
110 }
111 
112 static unsigned long shmem_default_max_inodes(void)
113 {
114         return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
115 }
116 #endif
117 
118 static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
119 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
120                                 struct shmem_inode_info *info, pgoff_t index);
121 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
122                 struct page **pagep, enum sgp_type sgp,
123                 gfp_t gfp, struct vm_area_struct *vma,
124                 struct vm_fault *vmf, int *fault_type);
125 
126 int shmem_getpage(struct inode *inode, pgoff_t index,
127                 struct page **pagep, enum sgp_type sgp)
128 {
129         return shmem_getpage_gfp(inode, index, pagep, sgp,
130                 mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL);
131 }
132 
133 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
134 {
135         return sb->s_fs_info;
136 }
137 
138 /*
139  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
140  * for shared memory and for shared anonymous (/dev/zero) mappings
141  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
142  * consistent with the pre-accounting of private mappings ...
143  */
144 static inline int shmem_acct_size(unsigned long flags, loff_t size)
145 {
146         return (flags & VM_NORESERVE) ?
147                 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
148 }
149 
150 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
151 {
152         if (!(flags & VM_NORESERVE))
153                 vm_unacct_memory(VM_ACCT(size));
154 }
155 
156 static inline int shmem_reacct_size(unsigned long flags,
157                 loff_t oldsize, loff_t newsize)
158 {
159         if (!(flags & VM_NORESERVE)) {
160                 if (VM_ACCT(newsize) > VM_ACCT(oldsize))
161                         return security_vm_enough_memory_mm(current->mm,
162                                         VM_ACCT(newsize) - VM_ACCT(oldsize));
163                 else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
164                         vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
165         }
166         return 0;
167 }
168 
169 /*
170  * ... whereas tmpfs objects are accounted incrementally as
171  * pages are allocated, in order to allow large sparse files.
172  * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
173  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
174  */
175 static inline int shmem_acct_block(unsigned long flags, long pages)
176 {
177         if (!(flags & VM_NORESERVE))
178                 return 0;
179 
180         return security_vm_enough_memory_mm(current->mm,
181                         pages * VM_ACCT(PAGE_SIZE));
182 }
183 
184 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
185 {
186         if (flags & VM_NORESERVE)
187                 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE));
188 }
189 
190 static const struct super_operations shmem_ops;
191 static const struct address_space_operations shmem_aops;
192 static const struct file_operations shmem_file_operations;
193 static const struct inode_operations shmem_inode_operations;
194 static const struct inode_operations shmem_dir_inode_operations;
195 static const struct inode_operations shmem_special_inode_operations;
196 static const struct vm_operations_struct shmem_vm_ops;
197 static struct file_system_type shmem_fs_type;
198 
199 bool vma_is_shmem(struct vm_area_struct *vma)
200 {
201         return vma->vm_ops == &shmem_vm_ops;
202 }
203 
204 static LIST_HEAD(shmem_swaplist);
205 static DEFINE_MUTEX(shmem_swaplist_mutex);
206 
207 static int shmem_reserve_inode(struct super_block *sb)
208 {
209         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
210         if (sbinfo->max_inodes) {
211                 spin_lock(&sbinfo->stat_lock);
212                 if (!sbinfo->free_inodes) {
213                         spin_unlock(&sbinfo->stat_lock);
214                         return -ENOSPC;
215                 }
216                 sbinfo->free_inodes--;
217                 spin_unlock(&sbinfo->stat_lock);
218         }
219         return 0;
220 }
221 
222 static void shmem_free_inode(struct super_block *sb)
223 {
224         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
225         if (sbinfo->max_inodes) {
226                 spin_lock(&sbinfo->stat_lock);
227                 sbinfo->free_inodes++;
228                 spin_unlock(&sbinfo->stat_lock);
229         }
230 }
231 
232 /**
233  * shmem_recalc_inode - recalculate the block usage of an inode
234  * @inode: inode to recalc
235  *
236  * We have to calculate the free blocks since the mm can drop
237  * undirtied hole pages behind our back.
238  *
239  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
240  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
241  *
242  * It has to be called with the spinlock held.
243  */
244 static void shmem_recalc_inode(struct inode *inode)
245 {
246         struct shmem_inode_info *info = SHMEM_I(inode);
247         long freed;
248 
249         freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
250         if (freed > 0) {
251                 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
252                 if (sbinfo->max_blocks)
253                         percpu_counter_add(&sbinfo->used_blocks, -freed);
254                 info->alloced -= freed;
255                 inode->i_blocks -= freed * BLOCKS_PER_PAGE;
256                 shmem_unacct_blocks(info->flags, freed);
257         }
258 }
259 
260 bool shmem_charge(struct inode *inode, long pages)
261 {
262         struct shmem_inode_info *info = SHMEM_I(inode);
263         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
264         unsigned long flags;
265 
266         if (shmem_acct_block(info->flags, pages))
267                 return false;
268         spin_lock_irqsave(&info->lock, flags);
269         info->alloced += pages;
270         inode->i_blocks += pages * BLOCKS_PER_PAGE;
271         shmem_recalc_inode(inode);
272         spin_unlock_irqrestore(&info->lock, flags);
273         inode->i_mapping->nrpages += pages;
274 
275         if (!sbinfo->max_blocks)
276                 return true;
277         if (percpu_counter_compare(&sbinfo->used_blocks,
278                                 sbinfo->max_blocks - pages) > 0) {
279                 inode->i_mapping->nrpages -= pages;
280                 spin_lock_irqsave(&info->lock, flags);
281                 info->alloced -= pages;
282                 shmem_recalc_inode(inode);
283                 spin_unlock_irqrestore(&info->lock, flags);
284                 shmem_unacct_blocks(info->flags, pages);
285                 return false;
286         }
287         percpu_counter_add(&sbinfo->used_blocks, pages);
288         return true;
289 }
290 
291 void shmem_uncharge(struct inode *inode, long pages)
292 {
293         struct shmem_inode_info *info = SHMEM_I(inode);
294         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
295         unsigned long flags;
296 
297         spin_lock_irqsave(&info->lock, flags);
298         info->alloced -= pages;
299         inode->i_blocks -= pages * BLOCKS_PER_PAGE;
300         shmem_recalc_inode(inode);
301         spin_unlock_irqrestore(&info->lock, flags);
302 
303         if (sbinfo->max_blocks)
304                 percpu_counter_sub(&sbinfo->used_blocks, pages);
305         shmem_unacct_blocks(info->flags, pages);
306 }
307 
308 /*
309  * Replace item expected in radix tree by a new item, while holding tree lock.
310  */
311 static int shmem_radix_tree_replace(struct address_space *mapping,
312                         pgoff_t index, void *expected, void *replacement)
313 {
314         struct radix_tree_node *node;
315         void **pslot;
316         void *item;
317 
318         VM_BUG_ON(!expected);
319         VM_BUG_ON(!replacement);
320         item = __radix_tree_lookup(&mapping->page_tree, index, &node, &pslot);
321         if (!item)
322                 return -ENOENT;
323         if (item != expected)
324                 return -ENOENT;
325         __radix_tree_replace(&mapping->page_tree, node, pslot,
326                              replacement, NULL, NULL);
327         return 0;
328 }
329 
330 /*
331  * Sometimes, before we decide whether to proceed or to fail, we must check
332  * that an entry was not already brought back from swap by a racing thread.
333  *
334  * Checking page is not enough: by the time a SwapCache page is locked, it
335  * might be reused, and again be SwapCache, using the same swap as before.
336  */
337 static bool shmem_confirm_swap(struct address_space *mapping,
338                                pgoff_t index, swp_entry_t swap)
339 {
340         void *item;
341 
342         rcu_read_lock();
343         item = radix_tree_lookup(&mapping->page_tree, index);
344         rcu_read_unlock();
345         return item == swp_to_radix_entry(swap);
346 }
347 
348 /*
349  * Definitions for "huge tmpfs": tmpfs mounted with the huge= option
350  *
351  * SHMEM_HUGE_NEVER:
352  *      disables huge pages for the mount;
353  * SHMEM_HUGE_ALWAYS:
354  *      enables huge pages for the mount;
355  * SHMEM_HUGE_WITHIN_SIZE:
356  *      only allocate huge pages if the page will be fully within i_size,
357  *      also respect fadvise()/madvise() hints;
358  * SHMEM_HUGE_ADVISE:
359  *      only allocate huge pages if requested with fadvise()/madvise();
360  */
361 
362 #define SHMEM_HUGE_NEVER        0
363 #define SHMEM_HUGE_ALWAYS       1
364 #define SHMEM_HUGE_WITHIN_SIZE  2
365 #define SHMEM_HUGE_ADVISE       3
366 
367 /*
368  * Special values.
369  * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled:
370  *
371  * SHMEM_HUGE_DENY:
372  *      disables huge on shm_mnt and all mounts, for emergency use;
373  * SHMEM_HUGE_FORCE:
374  *      enables huge on shm_mnt and all mounts, w/o needing option, for testing;
375  *
376  */
377 #define SHMEM_HUGE_DENY         (-1)
378 #define SHMEM_HUGE_FORCE        (-2)
379 
380 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
381 /* ifdef here to avoid bloating shmem.o when not necessary */
382 
383 int shmem_huge __read_mostly;
384 
385 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS)
386 static int shmem_parse_huge(const char *str)
387 {
388         if (!strcmp(str, "never"))
389                 return SHMEM_HUGE_NEVER;
390         if (!strcmp(str, "always"))
391                 return SHMEM_HUGE_ALWAYS;
392         if (!strcmp(str, "within_size"))
393                 return SHMEM_HUGE_WITHIN_SIZE;
394         if (!strcmp(str, "advise"))
395                 return SHMEM_HUGE_ADVISE;
396         if (!strcmp(str, "deny"))
397                 return SHMEM_HUGE_DENY;
398         if (!strcmp(str, "force"))
399                 return SHMEM_HUGE_FORCE;
400         return -EINVAL;
401 }
402 
403 static const char *shmem_format_huge(int huge)
404 {
405         switch (huge) {
406         case SHMEM_HUGE_NEVER:
407                 return "never";
408         case SHMEM_HUGE_ALWAYS:
409                 return "always";
410         case SHMEM_HUGE_WITHIN_SIZE:
411                 return "within_size";
412         case SHMEM_HUGE_ADVISE:
413                 return "advise";
414         case SHMEM_HUGE_DENY:
415                 return "deny";
416         case SHMEM_HUGE_FORCE:
417                 return "force";
418         default:
419                 VM_BUG_ON(1);
420                 return "bad_val";
421         }
422 }
423 #endif
424 
425 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
426                 struct shrink_control *sc, unsigned long nr_to_split)
427 {
428         LIST_HEAD(list), *pos, *next;
429         LIST_HEAD(to_remove);
430         struct inode *inode;
431         struct shmem_inode_info *info;
432         struct page *page;
433         unsigned long batch = sc ? sc->nr_to_scan : 128;
434         int removed = 0, split = 0;
435 
436         if (list_empty(&sbinfo->shrinklist))
437                 return SHRINK_STOP;
438 
439         spin_lock(&sbinfo->shrinklist_lock);
440         list_for_each_safe(pos, next, &sbinfo->shrinklist) {
441                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
442 
443                 /* pin the inode */
444                 inode = igrab(&info->vfs_inode);
445 
446                 /* inode is about to be evicted */
447                 if (!inode) {
448                         list_del_init(&info->shrinklist);
449                         removed++;
450                         goto next;
451                 }
452 
453                 /* Check if there's anything to gain */
454                 if (round_up(inode->i_size, PAGE_SIZE) ==
455                                 round_up(inode->i_size, HPAGE_PMD_SIZE)) {
456                         list_move(&info->shrinklist, &to_remove);
457                         removed++;
458                         goto next;
459                 }
460 
461                 list_move(&info->shrinklist, &list);
462 next:
463                 if (!--batch)
464                         break;
465         }
466         spin_unlock(&sbinfo->shrinklist_lock);
467 
468         list_for_each_safe(pos, next, &to_remove) {
469                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
470                 inode = &info->vfs_inode;
471                 list_del_init(&info->shrinklist);
472                 iput(inode);
473         }
474 
475         list_for_each_safe(pos, next, &list) {
476                 int ret;
477 
478                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
479                 inode = &info->vfs_inode;
480 
481                 if (nr_to_split && split >= nr_to_split) {
482                         iput(inode);
483                         continue;
484                 }
485 
486                 page = find_lock_page(inode->i_mapping,
487                                 (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT);
488                 if (!page)
489                         goto drop;
490 
491                 if (!PageTransHuge(page)) {
492                         unlock_page(page);
493                         put_page(page);
494                         goto drop;
495                 }
496 
497                 ret = split_huge_page(page);
498                 unlock_page(page);
499                 put_page(page);
500 
501                 if (ret) {
502                         /* split failed: leave it on the list */
503                         iput(inode);
504                         continue;
505                 }
506 
507                 split++;
508 drop:
509                 list_del_init(&info->shrinklist);
510                 removed++;
511                 iput(inode);
512         }
513 
514         spin_lock(&sbinfo->shrinklist_lock);
515         list_splice_tail(&list, &sbinfo->shrinklist);
516         sbinfo->shrinklist_len -= removed;
517         spin_unlock(&sbinfo->shrinklist_lock);
518 
519         return split;
520 }
521 
522 static long shmem_unused_huge_scan(struct super_block *sb,
523                 struct shrink_control *sc)
524 {
525         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
526 
527         if (!READ_ONCE(sbinfo->shrinklist_len))
528                 return SHRINK_STOP;
529 
530         return shmem_unused_huge_shrink(sbinfo, sc, 0);
531 }
532 
533 static long shmem_unused_huge_count(struct super_block *sb,
534                 struct shrink_control *sc)
535 {
536         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
537         return READ_ONCE(sbinfo->shrinklist_len);
538 }
539 #else /* !CONFIG_TRANSPARENT_HUGE_PAGECACHE */
540 
541 #define shmem_huge SHMEM_HUGE_DENY
542 
543 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
544                 struct shrink_control *sc, unsigned long nr_to_split)
545 {
546         return 0;
547 }
548 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */
549 
550 /*
551  * Like add_to_page_cache_locked, but error if expected item has gone.
552  */
553 static int shmem_add_to_page_cache(struct page *page,
554                                    struct address_space *mapping,
555                                    pgoff_t index, void *expected)
556 {
557         int error, nr = hpage_nr_pages(page);
558 
559         VM_BUG_ON_PAGE(PageTail(page), page);
560         VM_BUG_ON_PAGE(index != round_down(index, nr), page);
561         VM_BUG_ON_PAGE(!PageLocked(page), page);
562         VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
563         VM_BUG_ON(expected && PageTransHuge(page));
564 
565         page_ref_add(page, nr);
566         page->mapping = mapping;
567         page->index = index;
568 
569         spin_lock_irq(&mapping->tree_lock);
570         if (PageTransHuge(page)) {
571                 void __rcu **results;
572                 pgoff_t idx;
573                 int i;
574 
575                 error = 0;
576                 if (radix_tree_gang_lookup_slot(&mapping->page_tree,
577                                         &results, &idx, index, 1) &&
578                                 idx < index + HPAGE_PMD_NR) {
579                         error = -EEXIST;
580                 }
581 
582                 if (!error) {
583                         for (i = 0; i < HPAGE_PMD_NR; i++) {
584                                 error = radix_tree_insert(&mapping->page_tree,
585                                                 index + i, page + i);
586                                 VM_BUG_ON(error);
587                         }
588                         count_vm_event(THP_FILE_ALLOC);
589                 }
590         } else if (!expected) {
591                 error = radix_tree_insert(&mapping->page_tree, index, page);
592         } else {
593                 error = shmem_radix_tree_replace(mapping, index, expected,
594                                                                  page);
595         }
596 
597         if (!error) {
598                 mapping->nrpages += nr;
599                 if (PageTransHuge(page))
600                         __inc_node_page_state(page, NR_SHMEM_THPS);
601                 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr);
602                 __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr);
603                 spin_unlock_irq(&mapping->tree_lock);
604         } else {
605                 page->mapping = NULL;
606                 spin_unlock_irq(&mapping->tree_lock);
607                 page_ref_sub(page, nr);
608         }
609         return error;
610 }
611 
612 /*
613  * Like delete_from_page_cache, but substitutes swap for page.
614  */
615 static void shmem_delete_from_page_cache(struct page *page, void *radswap)
616 {
617         struct address_space *mapping = page->mapping;
618         int error;
619 
620         VM_BUG_ON_PAGE(PageCompound(page), page);
621 
622         spin_lock_irq(&mapping->tree_lock);
623         error = shmem_radix_tree_replace(mapping, page->index, page, radswap);
624         page->mapping = NULL;
625         mapping->nrpages--;
626         __dec_node_page_state(page, NR_FILE_PAGES);
627         __dec_node_page_state(page, NR_SHMEM);
628         spin_unlock_irq(&mapping->tree_lock);
629         put_page(page);
630         BUG_ON(error);
631 }
632 
633 /*
634  * Remove swap entry from radix tree, free the swap and its page cache.
635  */
636 static int shmem_free_swap(struct address_space *mapping,
637                            pgoff_t index, void *radswap)
638 {
639         void *old;
640 
641         spin_lock_irq(&mapping->tree_lock);
642         old = radix_tree_delete_item(&mapping->page_tree, index, radswap);
643         spin_unlock_irq(&mapping->tree_lock);
644         if (old != radswap)
645                 return -ENOENT;
646         free_swap_and_cache(radix_to_swp_entry(radswap));
647         return 0;
648 }
649 
650 /*
651  * Determine (in bytes) how many of the shmem object's pages mapped by the
652  * given offsets are swapped out.
653  *
654  * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU,
655  * as long as the inode doesn't go away and racy results are not a problem.
656  */
657 unsigned long shmem_partial_swap_usage(struct address_space *mapping,
658                                                 pgoff_t start, pgoff_t end)
659 {
660         struct radix_tree_iter iter;
661         void **slot;
662         struct page *page;
663         unsigned long swapped = 0;
664 
665         rcu_read_lock();
666 
667         radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
668                 if (iter.index >= end)
669                         break;
670 
671                 page = radix_tree_deref_slot(slot);
672 
673                 if (radix_tree_deref_retry(page)) {
674                         slot = radix_tree_iter_retry(&iter);
675                         continue;
676                 }
677 
678                 if (radix_tree_exceptional_entry(page))
679                         swapped++;
680 
681                 if (need_resched()) {
682                         slot = radix_tree_iter_resume(slot, &iter);
683                         cond_resched_rcu();
684                 }
685         }
686 
687         rcu_read_unlock();
688 
689         return swapped << PAGE_SHIFT;
690 }
691 
692 /*
693  * Determine (in bytes) how many of the shmem object's pages mapped by the
694  * given vma is swapped out.
695  *
696  * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU,
697  * as long as the inode doesn't go away and racy results are not a problem.
698  */
699 unsigned long shmem_swap_usage(struct vm_area_struct *vma)
700 {
701         struct inode *inode = file_inode(vma->vm_file);
702         struct shmem_inode_info *info = SHMEM_I(inode);
703         struct address_space *mapping = inode->i_mapping;
704         unsigned long swapped;
705 
706         /* Be careful as we don't hold info->lock */
707         swapped = READ_ONCE(info->swapped);
708 
709         /*
710          * The easier cases are when the shmem object has nothing in swap, or
711          * the vma maps it whole. Then we can simply use the stats that we
712          * already track.
713          */
714         if (!swapped)
715                 return 0;
716 
717         if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
718                 return swapped << PAGE_SHIFT;
719 
720         /* Here comes the more involved part */
721         return shmem_partial_swap_usage(mapping,
722                         linear_page_index(vma, vma->vm_start),
723                         linear_page_index(vma, vma->vm_end));
724 }
725 
726 /*
727  * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
728  */
729 void shmem_unlock_mapping(struct address_space *mapping)
730 {
731         struct pagevec pvec;
732         pgoff_t indices[PAGEVEC_SIZE];
733         pgoff_t index = 0;
734 
735         pagevec_init(&pvec, 0);
736         /*
737          * Minor point, but we might as well stop if someone else SHM_LOCKs it.
738          */
739         while (!mapping_unevictable(mapping)) {
740                 /*
741                  * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it
742                  * has finished, if it hits a row of PAGEVEC_SIZE swap entries.
743                  */
744                 pvec.nr = find_get_entries(mapping, index,
745                                            PAGEVEC_SIZE, pvec.pages, indices);
746                 if (!pvec.nr)
747                         break;
748                 index = indices[pvec.nr - 1] + 1;
749                 pagevec_remove_exceptionals(&pvec);
750                 check_move_unevictable_pages(pvec.pages, pvec.nr);
751                 pagevec_release(&pvec);
752                 cond_resched();
753         }
754 }
755 
756 /*
757  * Remove range of pages and swap entries from radix tree, and free them.
758  * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
759  */
760 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
761                                                                  bool unfalloc)
762 {
763         struct address_space *mapping = inode->i_mapping;
764         struct shmem_inode_info *info = SHMEM_I(inode);
765         pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
766         pgoff_t end = (lend + 1) >> PAGE_SHIFT;
767         unsigned int partial_start = lstart & (PAGE_SIZE - 1);
768         unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1);
769         struct pagevec pvec;
770         pgoff_t indices[PAGEVEC_SIZE];
771         long nr_swaps_freed = 0;
772         pgoff_t index;
773         int i;
774 
775         if (lend == -1)
776                 end = -1;       /* unsigned, so actually very big */
777 
778         pagevec_init(&pvec, 0);
779         index = start;
780         while (index < end) {
781                 pvec.nr = find_get_entries(mapping, index,
782                         min(end - index, (pgoff_t)PAGEVEC_SIZE),
783                         pvec.pages, indices);
784                 if (!pvec.nr)
785                         break;
786                 for (i = 0; i < pagevec_count(&pvec); i++) {
787                         struct page *page = pvec.pages[i];
788 
789                         index = indices[i];
790                         if (index >= end)
791                                 break;
792 
793                         if (radix_tree_exceptional_entry(page)) {
794                                 if (unfalloc)
795                                         continue;
796                                 nr_swaps_freed += !shmem_free_swap(mapping,
797                                                                 index, page);
798                                 continue;
799                         }
800 
801                         VM_BUG_ON_PAGE(page_to_pgoff(page) != index, page);
802 
803                         if (!trylock_page(page))
804                                 continue;
805 
806                         if (PageTransTail(page)) {
807                                 /* Middle of THP: zero out the page */
808                                 clear_highpage(page);
809                                 unlock_page(page);
810                                 continue;
811                         } else if (PageTransHuge(page)) {
812                                 if (index == round_down(end, HPAGE_PMD_NR)) {
813                                         /*
814                                          * Range ends in the middle of THP:
815                                          * zero out the page
816                                          */
817                                         clear_highpage(page);
818                                         unlock_page(page);
819                                         continue;
820                                 }
821                                 index += HPAGE_PMD_NR - 1;
822                                 i += HPAGE_PMD_NR - 1;
823                         }
824 
825                         if (!unfalloc || !PageUptodate(page)) {
826                                 VM_BUG_ON_PAGE(PageTail(page), page);
827                                 if (page_mapping(page) == mapping) {
828                                         VM_BUG_ON_PAGE(PageWriteback(page), page);
829                                         truncate_inode_page(mapping, page);
830                                 }
831                         }
832                         unlock_page(page);
833                 }
834                 pagevec_remove_exceptionals(&pvec);
835                 pagevec_release(&pvec);
836                 cond_resched();
837                 index++;
838         }
839 
840         if (partial_start) {
841                 struct page *page = NULL;
842                 shmem_getpage(inode, start - 1, &page, SGP_READ);
843                 if (page) {
844                         unsigned int top = PAGE_SIZE;
845                         if (start > end) {
846                                 top = partial_end;
847                                 partial_end = 0;
848                         }
849                         zero_user_segment(page, partial_start, top);
850                         set_page_dirty(page);
851                         unlock_page(page);
852                         put_page(page);
853                 }
854         }
855         if (partial_end) {
856                 struct page *page = NULL;
857                 shmem_getpage(inode, end, &page, SGP_READ);
858                 if (page) {
859                         zero_user_segment(page, 0, partial_end);
860                         set_page_dirty(page);
861                         unlock_page(page);
862                         put_page(page);
863                 }
864         }
865         if (start >= end)
866                 return;
867 
868         index = start;
869         while (index < end) {
870                 cond_resched();
871 
872                 pvec.nr = find_get_entries(mapping, index,
873                                 min(end - index, (pgoff_t)PAGEVEC_SIZE),
874                                 pvec.pages, indices);
875                 if (!pvec.nr) {
876                         /* If all gone or hole-punch or unfalloc, we're done */
877                         if (index == start || end != -1)
878                                 break;
879                         /* But if truncating, restart to make sure all gone */
880                         index = start;
881                         continue;
882                 }
883                 for (i = 0; i < pagevec_count(&pvec); i++) {
884                         struct page *page = pvec.pages[i];
885 
886                         index = indices[i];
887                         if (index >= end)
888                                 break;
889 
890                         if (radix_tree_exceptional_entry(page)) {
891                                 if (unfalloc)
892                                         continue;
893                                 if (shmem_free_swap(mapping, index, page)) {
894                                         /* Swap was replaced by page: retry */
895                                         index--;
896                                         break;
897                                 }
898                                 nr_swaps_freed++;
899                                 continue;
900                         }
901 
902                         lock_page(page);
903 
904                         if (PageTransTail(page)) {
905                                 /* Middle of THP: zero out the page */
906                                 clear_highpage(page);
907                                 unlock_page(page);
908                                 /*
909                                  * Partial thp truncate due 'start' in middle
910                                  * of THP: don't need to look on these pages
911                                  * again on !pvec.nr restart.
912                                  */
913                                 if (index != round_down(end, HPAGE_PMD_NR))
914                                         start++;
915                                 continue;
916                         } else if (PageTransHuge(page)) {
917                                 if (index == round_down(end, HPAGE_PMD_NR)) {
918                                         /*
919                                          * Range ends in the middle of THP:
920                                          * zero out the page
921                                          */
922                                         clear_highpage(page);
923                                         unlock_page(page);
924                                         continue;
925                                 }
926                                 index += HPAGE_PMD_NR - 1;
927                                 i += HPAGE_PMD_NR - 1;
928                         }
929 
930                         if (!unfalloc || !PageUptodate(page)) {
931                                 VM_BUG_ON_PAGE(PageTail(page), page);
932                                 if (page_mapping(page) == mapping) {
933                                         VM_BUG_ON_PAGE(PageWriteback(page), page);
934                                         truncate_inode_page(mapping, page);
935                                 } else {
936                                         /* Page was replaced by swap: retry */
937                                         unlock_page(page);
938                                         index--;
939                                         break;
940                                 }
941                         }
942                         unlock_page(page);
943                 }
944                 pagevec_remove_exceptionals(&pvec);
945                 pagevec_release(&pvec);
946                 index++;
947         }
948 
949         spin_lock_irq(&info->lock);
950         info->swapped -= nr_swaps_freed;
951         shmem_recalc_inode(inode);
952         spin_unlock_irq(&info->lock);
953 }
954 
955 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
956 {
957         shmem_undo_range(inode, lstart, lend, false);
958         inode->i_ctime = inode->i_mtime = current_time(inode);
959 }
960 EXPORT_SYMBOL_GPL(shmem_truncate_range);
961 
962 static int shmem_getattr(const struct path *path, struct kstat *stat,
963                          u32 request_mask, unsigned int query_flags)
964 {
965         struct inode *inode = path->dentry->d_inode;
966         struct shmem_inode_info *info = SHMEM_I(inode);
967 
968         if (info->alloced - info->swapped != inode->i_mapping->nrpages) {
969                 spin_lock_irq(&info->lock);
970                 shmem_recalc_inode(inode);
971                 spin_unlock_irq(&info->lock);
972         }
973         generic_fillattr(inode, stat);
974         return 0;
975 }
976 
977 static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
978 {
979         struct inode *inode = d_inode(dentry);
980         struct shmem_inode_info *info = SHMEM_I(inode);
981         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
982         int error;
983 
984         error = setattr_prepare(dentry, attr);
985         if (error)
986                 return error;
987 
988         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
989                 loff_t oldsize = inode->i_size;
990                 loff_t newsize = attr->ia_size;
991 
992                 /* protected by i_mutex */
993                 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
994                     (newsize > oldsize && (info->seals & F_SEAL_GROW)))
995                         return -EPERM;
996 
997                 if (newsize != oldsize) {
998                         error = shmem_reacct_size(SHMEM_I(inode)->flags,
999                                         oldsize, newsize);
1000                         if (error)
1001                                 return error;
1002                         i_size_write(inode, newsize);
1003                         inode->i_ctime = inode->i_mtime = current_time(inode);
1004                 }
1005                 if (newsize <= oldsize) {
1006                         loff_t holebegin = round_up(newsize, PAGE_SIZE);
1007                         if (oldsize > holebegin)
1008                                 unmap_mapping_range(inode->i_mapping,
1009                                                         holebegin, 0, 1);
1010                         if (info->alloced)
1011                                 shmem_truncate_range(inode,
1012                                                         newsize, (loff_t)-1);
1013                         /* unmap again to remove racily COWed private pages */
1014                         if (oldsize > holebegin)
1015                                 unmap_mapping_range(inode->i_mapping,
1016                                                         holebegin, 0, 1);
1017 
1018                         /*
1019                          * Part of the huge page can be beyond i_size: subject
1020                          * to shrink under memory pressure.
1021                          */
1022                         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) {
1023                                 spin_lock(&sbinfo->shrinklist_lock);
1024                                 if (list_empty(&info->shrinklist)) {
1025                                         list_add_tail(&info->shrinklist,
1026                                                         &sbinfo->shrinklist);
1027                                         sbinfo->shrinklist_len++;
1028                                 }
1029                                 spin_unlock(&sbinfo->shrinklist_lock);
1030                         }
1031                 }
1032         }
1033 
1034         setattr_copy(inode, attr);
1035         if (attr->ia_valid & ATTR_MODE)
1036                 error = posix_acl_chmod(inode, inode->i_mode);
1037         return error;
1038 }
1039 
1040 static void shmem_evict_inode(struct inode *inode)
1041 {
1042         struct shmem_inode_info *info = SHMEM_I(inode);
1043         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1044 
1045         if (inode->i_mapping->a_ops == &shmem_aops) {
1046                 shmem_unacct_size(info->flags, inode->i_size);
1047                 inode->i_size = 0;
1048                 shmem_truncate_range(inode, 0, (loff_t)-1);
1049                 if (!list_empty(&info->shrinklist)) {
1050                         spin_lock(&sbinfo->shrinklist_lock);
1051                         if (!list_empty(&info->shrinklist)) {
1052                                 list_del_init(&info->shrinklist);
1053                                 sbinfo->shrinklist_len--;
1054                         }
1055                         spin_unlock(&sbinfo->shrinklist_lock);
1056                 }
1057                 if (!list_empty(&info->swaplist)) {
1058                         mutex_lock(&shmem_swaplist_mutex);
1059                         list_del_init(&info->swaplist);
1060                         mutex_unlock(&shmem_swaplist_mutex);
1061                 }
1062         }
1063 
1064         simple_xattrs_free(&info->xattrs);
1065         WARN_ON(inode->i_blocks);
1066         shmem_free_inode(inode->i_sb);
1067         clear_inode(inode);
1068 }
1069 
1070 static unsigned long find_swap_entry(struct radix_tree_root *root, void *item)
1071 {
1072         struct radix_tree_iter iter;
1073         void **slot;
1074         unsigned long found = -1;
1075         unsigned int checked = 0;
1076 
1077         rcu_read_lock();
1078         radix_tree_for_each_slot(slot, root, &iter, 0) {
1079                 if (*slot == item) {
1080                         found = iter.index;
1081                         break;
1082                 }
1083                 checked++;
1084                 if ((checked % 4096) != 0)
1085                         continue;
1086                 slot = radix_tree_iter_resume(slot, &iter);
1087                 cond_resched_rcu();
1088         }
1089 
1090         rcu_read_unlock();
1091         return found;
1092 }
1093 
1094 /*
1095  * If swap found in inode, free it and move page from swapcache to filecache.
1096  */
1097 static int shmem_unuse_inode(struct shmem_inode_info *info,
1098                              swp_entry_t swap, struct page **pagep)
1099 {
1100         struct address_space *mapping = info->vfs_inode.i_mapping;
1101         void *radswap;
1102         pgoff_t index;
1103         gfp_t gfp;
1104         int error = 0;
1105 
1106         radswap = swp_to_radix_entry(swap);
1107         index = find_swap_entry(&mapping->page_tree, radswap);
1108         if (index == -1)
1109                 return -EAGAIN; /* tell shmem_unuse we found nothing */
1110 
1111         /*
1112          * Move _head_ to start search for next from here.
1113          * But be careful: shmem_evict_inode checks list_empty without taking
1114          * mutex, and there's an instant in list_move_tail when info->swaplist
1115          * would appear empty, if it were the only one on shmem_swaplist.
1116          */
1117         if (shmem_swaplist.next != &info->swaplist)
1118                 list_move_tail(&shmem_swaplist, &info->swaplist);
1119 
1120         gfp = mapping_gfp_mask(mapping);
1121         if (shmem_should_replace_page(*pagep, gfp)) {
1122                 mutex_unlock(&shmem_swaplist_mutex);
1123                 error = shmem_replace_page(pagep, gfp, info, index);
1124                 mutex_lock(&shmem_swaplist_mutex);
1125                 /*
1126                  * We needed to drop mutex to make that restrictive page
1127                  * allocation, but the inode might have been freed while we
1128                  * dropped it: although a racing shmem_evict_inode() cannot
1129                  * complete without emptying the radix_tree, our page lock
1130                  * on this swapcache page is not enough to prevent that -
1131                  * free_swap_and_cache() of our swap entry will only
1132                  * trylock_page(), removing swap from radix_tree whatever.
1133                  *
1134                  * We must not proceed to shmem_add_to_page_cache() if the
1135                  * inode has been freed, but of course we cannot rely on
1136                  * inode or mapping or info to check that.  However, we can
1137                  * safely check if our swap entry is still in use (and here
1138                  * it can't have got reused for another page): if it's still
1139                  * in use, then the inode cannot have been freed yet, and we
1140                  * can safely proceed (if it's no longer in use, that tells
1141                  * nothing about the inode, but we don't need to unuse swap).
1142                  */
1143                 if (!page_swapcount(*pagep))
1144                         error = -ENOENT;
1145         }
1146 
1147         /*
1148          * We rely on shmem_swaplist_mutex, not only to protect the swaplist,
1149          * but also to hold up shmem_evict_inode(): so inode cannot be freed
1150          * beneath us (pagelock doesn't help until the page is in pagecache).
1151          */
1152         if (!error)
1153                 error = shmem_add_to_page_cache(*pagep, mapping, index,
1154                                                 radswap);
1155         if (error != -ENOMEM) {
1156                 /*
1157                  * Truncation and eviction use free_swap_and_cache(), which
1158                  * only does trylock page: if we raced, best clean up here.
1159                  */
1160                 delete_from_swap_cache(*pagep);
1161                 set_page_dirty(*pagep);
1162                 if (!error) {
1163                         spin_lock_irq(&info->lock);
1164                         info->swapped--;
1165                         spin_unlock_irq(&info->lock);
1166                         swap_free(swap);
1167                 }
1168         }
1169         return error;
1170 }
1171 
1172 /*
1173  * Search through swapped inodes to find and replace swap by page.
1174  */
1175 int shmem_unuse(swp_entry_t swap, struct page *page)
1176 {
1177         struct list_head *this, *next;
1178         struct shmem_inode_info *info;
1179         struct mem_cgroup *memcg;
1180         int error = 0;
1181 
1182         /*
1183          * There's a faint possibility that swap page was replaced before
1184          * caller locked it: caller will come back later with the right page.
1185          */
1186         if (unlikely(!PageSwapCache(page) || page_private(page) != swap.val))
1187                 goto out;
1188 
1189         /*
1190          * Charge page using GFP_KERNEL while we can wait, before taking
1191          * the shmem_swaplist_mutex which might hold up shmem_writepage().
1192          * Charged back to the user (not to caller) when swap account is used.
1193          */
1194         error = mem_cgroup_try_charge(page, current->mm, GFP_KERNEL, &memcg,
1195                         false);
1196         if (error)
1197                 goto out;
1198         /* No radix_tree_preload: swap entry keeps a place for page in tree */
1199         error = -EAGAIN;
1200 
1201         mutex_lock(&shmem_swaplist_mutex);
1202         list_for_each_safe(this, next, &shmem_swaplist) {
1203                 info = list_entry(this, struct shmem_inode_info, swaplist);
1204                 if (info->swapped)
1205                         error = shmem_unuse_inode(info, swap, &page);
1206                 else
1207                         list_del_init(&info->swaplist);
1208                 cond_resched();
1209                 if (error != -EAGAIN)
1210                         break;
1211                 /* found nothing in this: move on to search the next */
1212         }
1213         mutex_unlock(&shmem_swaplist_mutex);
1214 
1215         if (error) {
1216                 if (error != -ENOMEM)
1217                         error = 0;
1218                 mem_cgroup_cancel_charge(page, memcg, false);
1219         } else
1220                 mem_cgroup_commit_charge(page, memcg, true, false);
1221 out:
1222         unlock_page(page);
1223         put_page(page);
1224         return error;
1225 }
1226 
1227 /*
1228  * Move the page from the page cache to the swap cache.
1229  */
1230 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1231 {
1232         struct shmem_inode_info *info;
1233         struct address_space *mapping;
1234         struct inode *inode;
1235         swp_entry_t swap;
1236         pgoff_t index;
1237 
1238         VM_BUG_ON_PAGE(PageCompound(page), page);
1239         BUG_ON(!PageLocked(page));
1240         mapping = page->mapping;
1241         index = page->index;
1242         inode = mapping->host;
1243         info = SHMEM_I(inode);
1244         if (info->flags & VM_LOCKED)
1245                 goto redirty;
1246         if (!total_swap_pages)
1247                 goto redirty;
1248 
1249         /*
1250          * Our capabilities prevent regular writeback or sync from ever calling
1251          * shmem_writepage; but a stacking filesystem might use ->writepage of
1252          * its underlying filesystem, in which case tmpfs should write out to
1253          * swap only in response to memory pressure, and not for the writeback
1254          * threads or sync.
1255          */
1256         if (!wbc->for_reclaim) {
1257                 WARN_ON_ONCE(1);        /* Still happens? Tell us about it! */
1258                 goto redirty;
1259         }
1260 
1261         /*
1262          * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
1263          * value into swapfile.c, the only way we can correctly account for a
1264          * fallocated page arriving here is now to initialize it and write it.
1265          *
1266          * That's okay for a page already fallocated earlier, but if we have
1267          * not yet completed the fallocation, then (a) we want to keep track
1268          * of this page in case we have to undo it, and (b) it may not be a
1269          * good idea to continue anyway, once we're pushing into swap.  So
1270          * reactivate the page, and let shmem_fallocate() quit when too many.
1271          */
1272         if (!PageUptodate(page)) {
1273                 if (inode->i_private) {
1274                         struct shmem_falloc *shmem_falloc;
1275                         spin_lock(&inode->i_lock);
1276                         shmem_falloc = inode->i_private;
1277                         if (shmem_falloc &&
1278                             !shmem_falloc->waitq &&
1279                             index >= shmem_falloc->start &&
1280                             index < shmem_falloc->next)
1281                                 shmem_falloc->nr_unswapped++;
1282                         else
1283                                 shmem_falloc = NULL;
1284                         spin_unlock(&inode->i_lock);
1285                         if (shmem_falloc)
1286                                 goto redirty;
1287                 }
1288                 clear_highpage(page);
1289                 flush_dcache_page(page);
1290                 SetPageUptodate(page);
1291         }
1292 
1293         swap = get_swap_page();
1294         if (!swap.val)
1295                 goto redirty;
1296 
1297         if (mem_cgroup_try_charge_swap(page, swap))
1298                 goto free_swap;
1299 
1300         /*
1301          * Add inode to shmem_unuse()'s list of swapped-out inodes,
1302          * if it's not already there.  Do it now before the page is
1303          * moved to swap cache, when its pagelock no longer protects
1304          * the inode from eviction.  But don't unlock the mutex until
1305          * we've incremented swapped, because shmem_unuse_inode() will
1306          * prune a !swapped inode from the swaplist under this mutex.
1307          */
1308         mutex_lock(&shmem_swaplist_mutex);
1309         if (list_empty(&info->swaplist))
1310                 list_add_tail(&info->swaplist, &shmem_swaplist);
1311 
1312         if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
1313                 spin_lock_irq(&info->lock);
1314                 shmem_recalc_inode(inode);
1315                 info->swapped++;
1316                 spin_unlock_irq(&info->lock);
1317 
1318                 swap_shmem_alloc(swap);
1319                 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));
1320 
1321                 mutex_unlock(&shmem_swaplist_mutex);
1322                 BUG_ON(page_mapped(page));
1323                 swap_writepage(page, wbc);
1324                 return 0;
1325         }
1326 
1327         mutex_unlock(&shmem_swaplist_mutex);
1328 free_swap:
1329         swapcache_free(swap);
1330 redirty:
1331         set_page_dirty(page);
1332         if (wbc->for_reclaim)
1333                 return AOP_WRITEPAGE_ACTIVATE;  /* Return with page locked */
1334         unlock_page(page);
1335         return 0;
1336 }
1337 
1338 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
1339 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1340 {
1341         char buffer[64];
1342 
1343         if (!mpol || mpol->mode == MPOL_DEFAULT)
1344                 return;         /* show nothing */
1345 
1346         mpol_to_str(buffer, sizeof(buffer), mpol);
1347 
1348         seq_printf(seq, ",mpol=%s", buffer);
1349 }
1350 
1351 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1352 {
1353         struct mempolicy *mpol = NULL;
1354         if (sbinfo->mpol) {
1355                 spin_lock(&sbinfo->stat_lock);  /* prevent replace/use races */
1356                 mpol = sbinfo->mpol;
1357                 mpol_get(mpol);
1358                 spin_unlock(&sbinfo->stat_lock);
1359         }
1360         return mpol;
1361 }
1362 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */
1363 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1364 {
1365 }
1366 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1367 {
1368         return NULL;
1369 }
1370 #endif /* CONFIG_NUMA && CONFIG_TMPFS */
1371 #ifndef CONFIG_NUMA
1372 #define vm_policy vm_private_data
1373 #endif
1374 
1375 static void shmem_pseudo_vma_init(struct vm_area_struct *vma,
1376                 struct shmem_inode_info *info, pgoff_t index)
1377 {
1378         /* Create a pseudo vma that just contains the policy */
1379         vma->vm_start = 0;
1380         /* Bias interleave by inode number to distribute better across nodes */
1381         vma->vm_pgoff = index + info->vfs_inode.i_ino;
1382         vma->vm_ops = NULL;
1383         vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index);
1384 }
1385 
1386 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma)
1387 {
1388         /* Drop reference taken by mpol_shared_policy_lookup() */
1389         mpol_cond_put(vma->vm_policy);
1390 }
1391 
1392 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
1393                         struct shmem_inode_info *info, pgoff_t index)
1394 {
1395         struct vm_area_struct pvma;
1396         struct page *page;
1397 
1398         shmem_pseudo_vma_init(&pvma, info, index);
1399         page = swapin_readahead(swap, gfp, &pvma, 0);
1400         shmem_pseudo_vma_destroy(&pvma);
1401 
1402         return page;
1403 }
1404 
1405 static struct page *shmem_alloc_hugepage(gfp_t gfp,
1406                 struct shmem_inode_info *info, pgoff_t index)
1407 {
1408         struct vm_area_struct pvma;
1409         struct inode *inode = &info->vfs_inode;
1410         struct address_space *mapping = inode->i_mapping;
1411         pgoff_t idx, hindex;
1412         void __rcu **results;
1413         struct page *page;
1414 
1415         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
1416                 return NULL;
1417 
1418         hindex = round_down(index, HPAGE_PMD_NR);
1419         rcu_read_lock();
1420         if (radix_tree_gang_lookup_slot(&mapping->page_tree, &results, &idx,
1421                                 hindex, 1) && idx < hindex + HPAGE_PMD_NR) {
1422                 rcu_read_unlock();
1423                 return NULL;
1424         }
1425         rcu_read_unlock();
1426 
1427         shmem_pseudo_vma_init(&pvma, info, hindex);
1428         page = alloc_pages_vma(gfp | __GFP_COMP | __GFP_NORETRY | __GFP_NOWARN,
1429                         HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(), true);
1430         shmem_pseudo_vma_destroy(&pvma);
1431         if (page)
1432                 prep_transhuge_page(page);
1433         return page;
1434 }
1435 
1436 static struct page *shmem_alloc_page(gfp_t gfp,
1437                         struct shmem_inode_info *info, pgoff_t index)
1438 {
1439         struct vm_area_struct pvma;
1440         struct page *page;
1441 
1442         shmem_pseudo_vma_init(&pvma, info, index);
1443         page = alloc_page_vma(gfp, &pvma, 0);
1444         shmem_pseudo_vma_destroy(&pvma);
1445 
1446         return page;
1447 }
1448 
1449 static struct page *shmem_alloc_and_acct_page(gfp_t gfp,
1450                 struct shmem_inode_info *info, struct shmem_sb_info *sbinfo,
1451                 pgoff_t index, bool huge)
1452 {
1453         struct page *page;
1454         int nr;
1455         int err = -ENOSPC;
1456 
1457         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
1458                 huge = false;
1459         nr = huge ? HPAGE_PMD_NR : 1;
1460 
1461         if (shmem_acct_block(info->flags, nr))
1462                 goto failed;
1463         if (sbinfo->max_blocks) {
1464                 if (percpu_counter_compare(&sbinfo->used_blocks,
1465                                         sbinfo->max_blocks - nr) > 0)
1466                         goto unacct;
1467                 percpu_counter_add(&sbinfo->used_blocks, nr);
1468         }
1469 
1470         if (huge)
1471                 page = shmem_alloc_hugepage(gfp, info, index);
1472         else
1473                 page = shmem_alloc_page(gfp, info, index);
1474         if (page) {
1475                 __SetPageLocked(page);
1476                 __SetPageSwapBacked(page);
1477                 return page;
1478         }
1479 
1480         err = -ENOMEM;
1481         if (sbinfo->max_blocks)
1482                 percpu_counter_add(&sbinfo->used_blocks, -nr);
1483 unacct:
1484         shmem_unacct_blocks(info->flags, nr);
1485 failed:
1486         return ERR_PTR(err);
1487 }
1488 
1489 /*
1490  * When a page is moved from swapcache to shmem filecache (either by the
1491  * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of
1492  * shmem_unuse_inode()), it may have been read in earlier from swap, in
1493  * ignorance of the mapping it belongs to.  If that mapping has special
1494  * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
1495  * we may need to copy to a suitable page before moving to filecache.
1496  *
1497  * In a future release, this may well be extended to respect cpuset and
1498  * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
1499  * but for now it is a simple matter of zone.
1500  */
1501 static bool shmem_should_replace_page(struct page *page, gfp_t gfp)
1502 {
1503         return page_zonenum(page) > gfp_zone(gfp);
1504 }
1505 
1506 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
1507                                 struct shmem_inode_info *info, pgoff_t index)
1508 {
1509         struct page *oldpage, *newpage;
1510         struct address_space *swap_mapping;
1511         pgoff_t swap_index;
1512         int error;
1513 
1514         oldpage = *pagep;
1515         swap_index = page_private(oldpage);
1516         swap_mapping = page_mapping(oldpage);
1517 
1518         /*
1519          * We have arrived here because our zones are constrained, so don't
1520          * limit chance of success by further cpuset and node constraints.
1521          */
1522         gfp &= ~GFP_CONSTRAINT_MASK;
1523         newpage = shmem_alloc_page(gfp, info, index);
1524         if (!newpage)
1525                 return -ENOMEM;
1526 
1527         get_page(newpage);
1528         copy_highpage(newpage, oldpage);
1529         flush_dcache_page(newpage);
1530 
1531         __SetPageLocked(newpage);
1532         __SetPageSwapBacked(newpage);
1533         SetPageUptodate(newpage);
1534         set_page_private(newpage, swap_index);
1535         SetPageSwapCache(newpage);
1536 
1537         /*
1538          * Our caller will very soon move newpage out of swapcache, but it's
1539          * a nice clean interface for us to replace oldpage by newpage there.
1540          */
1541         spin_lock_irq(&swap_mapping->tree_lock);
1542         error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage,
1543                                                                    newpage);
1544         if (!error) {
1545                 __inc_node_page_state(newpage, NR_FILE_PAGES);
1546                 __dec_node_page_state(oldpage, NR_FILE_PAGES);
1547         }
1548         spin_unlock_irq(&swap_mapping->tree_lock);
1549 
1550         if (unlikely(error)) {
1551                 /*
1552                  * Is this possible?  I think not, now that our callers check
1553                  * both PageSwapCache and page_private after getting page lock;
1554                  * but be defensive.  Reverse old to newpage for clear and free.
1555                  */
1556                 oldpage = newpage;
1557         } else {
1558                 mem_cgroup_migrate(oldpage, newpage);
1559                 lru_cache_add_anon(newpage);
1560                 *pagep = newpage;
1561         }
1562 
1563         ClearPageSwapCache(oldpage);
1564         set_page_private(oldpage, 0);
1565 
1566         unlock_page(oldpage);
1567         put_page(oldpage);
1568         put_page(oldpage);
1569         return error;
1570 }
1571 
1572 /*
1573  * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
1574  *
1575  * If we allocate a new one we do not mark it dirty. That's up to the
1576  * vm. If we swap it in we mark it dirty since we also free the swap
1577  * entry since a page cannot live in both the swap and page cache.
1578  *
1579  * fault_mm and fault_type are only supplied by shmem_fault:
1580  * otherwise they are NULL.
1581  */
1582 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
1583         struct page **pagep, enum sgp_type sgp, gfp_t gfp,
1584         struct vm_area_struct *vma, struct vm_fault *vmf, int *fault_type)
1585 {
1586         struct address_space *mapping = inode->i_mapping;
1587         struct shmem_inode_info *info = SHMEM_I(inode);
1588         struct shmem_sb_info *sbinfo;
1589         struct mm_struct *charge_mm;
1590         struct mem_cgroup *memcg;
1591         struct page *page;
1592         swp_entry_t swap;
1593         enum sgp_type sgp_huge = sgp;
1594         pgoff_t hindex = index;
1595         int error;
1596         int once = 0;
1597         int alloced = 0;
1598 
1599         if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
1600                 return -EFBIG;
1601         if (sgp == SGP_NOHUGE || sgp == SGP_HUGE)
1602                 sgp = SGP_CACHE;
1603 repeat:
1604         swap.val = 0;
1605         page = find_lock_entry(mapping, index);
1606         if (radix_tree_exceptional_entry(page)) {
1607                 swap = radix_to_swp_entry(page);
1608                 page = NULL;
1609         }
1610 
1611         if (sgp <= SGP_CACHE &&
1612             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1613                 error = -EINVAL;
1614                 goto unlock;
1615         }
1616 
1617         if (page && sgp == SGP_WRITE)
1618                 mark_page_accessed(page);
1619 
1620         /* fallocated page? */
1621         if (page && !PageUptodate(page)) {
1622                 if (sgp != SGP_READ)
1623                         goto clear;
1624                 unlock_page(page);
1625                 put_page(page);
1626                 page = NULL;
1627         }
1628         if (page || (sgp == SGP_READ && !swap.val)) {
1629                 *pagep = page;
1630                 return 0;
1631         }
1632 
1633         /*
1634          * Fast cache lookup did not find it:
1635          * bring it back from swap or allocate.
1636          */
1637         sbinfo = SHMEM_SB(inode->i_sb);
1638         charge_mm = vma ? vma->vm_mm : current->mm;
1639 
1640         if (swap.val) {
1641                 /* Look it up and read it in.. */
1642                 page = lookup_swap_cache(swap);
1643                 if (!page) {
1644                         /* Or update major stats only when swapin succeeds?? */
1645                         if (fault_type) {
1646                                 *fault_type |= VM_FAULT_MAJOR;
1647                                 count_vm_event(PGMAJFAULT);
1648                                 mem_cgroup_count_vm_event(charge_mm,
1649                                                           PGMAJFAULT);
1650                         }
1651                         /* Here we actually start the io */
1652                         page = shmem_swapin(swap, gfp, info, index);
1653                         if (!page) {
1654                                 error = -ENOMEM;
1655                                 goto failed;
1656                         }
1657                 }
1658 
1659                 /* We have to do this with page locked to prevent races */
1660                 lock_page(page);
1661                 if (!PageSwapCache(page) || page_private(page) != swap.val ||
1662                     !shmem_confirm_swap(mapping, index, swap)) {
1663                         error = -EEXIST;        /* try again */
1664                         goto unlock;
1665                 }
1666                 if (!PageUptodate(page)) {
1667                         error = -EIO;
1668                         goto failed;
1669                 }
1670                 wait_on_page_writeback(page);
1671 
1672                 if (shmem_should_replace_page(page, gfp)) {
1673                         error = shmem_replace_page(&page, gfp, info, index);
1674                         if (error)
1675                                 goto failed;
1676                 }
1677 
1678                 error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg,
1679                                 false);
1680                 if (!error) {
1681                         error = shmem_add_to_page_cache(page, mapping, index,
1682                                                 swp_to_radix_entry(swap));
1683                         /*
1684                          * We already confirmed swap under page lock, and make
1685                          * no memory allocation here, so usually no possibility
1686                          * of error; but free_swap_and_cache() only trylocks a
1687                          * page, so it is just possible that the entry has been
1688                          * truncated or holepunched since swap was confirmed.
1689                          * shmem_undo_range() will have done some of the
1690                          * unaccounting, now delete_from_swap_cache() will do
1691                          * the rest.
1692                          * Reset swap.val? No, leave it so "failed" goes back to
1693                          * "repeat": reading a hole and writing should succeed.
1694                          */
1695                         if (error) {
1696                                 mem_cgroup_cancel_charge(page, memcg, false);
1697                                 delete_from_swap_cache(page);
1698                         }
1699                 }
1700                 if (error)
1701                         goto failed;
1702 
1703                 mem_cgroup_commit_charge(page, memcg, true, false);
1704 
1705                 spin_lock_irq(&info->lock);
1706                 info->swapped--;
1707                 shmem_recalc_inode(inode);
1708                 spin_unlock_irq(&info->lock);
1709 
1710                 if (sgp == SGP_WRITE)
1711                         mark_page_accessed(page);
1712 
1713                 delete_from_swap_cache(page);
1714                 set_page_dirty(page);
1715                 swap_free(swap);
1716 
1717         } else {
1718                 if (vma && userfaultfd_missing(vma)) {
1719                         *fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
1720                         return 0;
1721                 }
1722 
1723                 /* shmem_symlink() */
1724                 if (mapping->a_ops != &shmem_aops)
1725                         goto alloc_nohuge;
1726                 if (shmem_huge == SHMEM_HUGE_DENY || sgp_huge == SGP_NOHUGE)
1727                         goto alloc_nohuge;
1728                 if (shmem_huge == SHMEM_HUGE_FORCE)
1729                         goto alloc_huge;
1730                 switch (sbinfo->huge) {
1731                         loff_t i_size;
1732                         pgoff_t off;
1733                 case SHMEM_HUGE_NEVER:
1734                         goto alloc_nohuge;
1735                 case SHMEM_HUGE_WITHIN_SIZE:
1736                         off = round_up(index, HPAGE_PMD_NR);
1737                         i_size = round_up(i_size_read(inode), PAGE_SIZE);
1738                         if (i_size >= HPAGE_PMD_SIZE &&
1739                                         i_size >> PAGE_SHIFT >= off)
1740                                 goto alloc_huge;
1741                         /* fallthrough */
1742                 case SHMEM_HUGE_ADVISE:
1743                         if (sgp_huge == SGP_HUGE)
1744                                 goto alloc_huge;
1745                         /* TODO: implement fadvise() hints */
1746                         goto alloc_nohuge;
1747                 }
1748 
1749 alloc_huge:
1750                 page = shmem_alloc_and_acct_page(gfp, info, sbinfo,
1751                                 index, true);
1752                 if (IS_ERR(page)) {
1753 alloc_nohuge:           page = shmem_alloc_and_acct_page(gfp, info, sbinfo,
1754                                         index, false);
1755                 }
1756                 if (IS_ERR(page)) {
1757                         int retry = 5;
1758                         error = PTR_ERR(page);
1759                         page = NULL;
1760                         if (error != -ENOSPC)
1761                                 goto failed;
1762                         /*
1763                          * Try to reclaim some spece by splitting a huge page
1764                          * beyond i_size on the filesystem.
1765                          */
1766                         while (retry--) {
1767                                 int ret;
1768                                 ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
1769                                 if (ret == SHRINK_STOP)
1770                                         break;
1771                                 if (ret)
1772                                         goto alloc_nohuge;
1773                         }
1774                         goto failed;
1775                 }
1776 
1777                 if (PageTransHuge(page))
1778                         hindex = round_down(index, HPAGE_PMD_NR);
1779                 else
1780                         hindex = index;
1781 
1782                 if (sgp == SGP_WRITE)
1783                         __SetPageReferenced(page);
1784 
1785                 error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg,
1786                                 PageTransHuge(page));
1787                 if (error)
1788                         goto unacct;
1789                 error = radix_tree_maybe_preload_order(gfp & GFP_RECLAIM_MASK,
1790                                 compound_order(page));
1791                 if (!error) {
1792                         error = shmem_add_to_page_cache(page, mapping, hindex,
1793                                                         NULL);
1794                         radix_tree_preload_end();
1795                 }
1796                 if (error) {
1797                         mem_cgroup_cancel_charge(page, memcg,
1798                                         PageTransHuge(page));
1799                         goto unacct;
1800                 }
1801                 mem_cgroup_commit_charge(page, memcg, false,
1802                                 PageTransHuge(page));
1803                 lru_cache_add_anon(page);
1804 
1805                 spin_lock_irq(&info->lock);
1806                 info->alloced += 1 << compound_order(page);
1807                 inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page);
1808                 shmem_recalc_inode(inode);
1809                 spin_unlock_irq(&info->lock);
1810                 alloced = true;
1811 
1812                 if (PageTransHuge(page) &&
1813                                 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
1814                                 hindex + HPAGE_PMD_NR - 1) {
1815                         /*
1816                          * Part of the huge page is beyond i_size: subject
1817                          * to shrink under memory pressure.
1818                          */
1819                         spin_lock(&sbinfo->shrinklist_lock);
1820                         if (list_empty(&info->shrinklist)) {
1821                                 list_add_tail(&info->shrinklist,
1822                                                 &sbinfo->shrinklist);
1823                                 sbinfo->shrinklist_len++;
1824                         }
1825                         spin_unlock(&sbinfo->shrinklist_lock);
1826                 }
1827 
1828                 /*
1829                  * Let SGP_FALLOC use the SGP_WRITE optimization on a new page.
1830                  */
1831                 if (sgp == SGP_FALLOC)
1832                         sgp = SGP_WRITE;
1833 clear:
1834                 /*
1835                  * Let SGP_WRITE caller clear ends if write does not fill page;
1836                  * but SGP_FALLOC on a page fallocated earlier must initialize
1837                  * it now, lest undo on failure cancel our earlier guarantee.
1838                  */
1839                 if (sgp != SGP_WRITE && !PageUptodate(page)) {
1840                         struct page *head = compound_head(page);
1841                         int i;
1842 
1843                         for (i = 0; i < (1 << compound_order(head)); i++) {
1844                                 clear_highpage(head + i);
1845                                 flush_dcache_page(head + i);
1846                         }
1847                         SetPageUptodate(head);
1848                 }
1849         }
1850 
1851         /* Perhaps the file has been truncated since we checked */
1852         if (sgp <= SGP_CACHE &&
1853             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1854                 if (alloced) {
1855                         ClearPageDirty(page);
1856                         delete_from_page_cache(page);
1857                         spin_lock_irq(&info->lock);
1858                         shmem_recalc_inode(inode);
1859                         spin_unlock_irq(&info->lock);
1860                 }
1861                 error = -EINVAL;
1862                 goto unlock;
1863         }
1864         *pagep = page + index - hindex;
1865         return 0;
1866 
1867         /*
1868          * Error recovery.
1869          */
1870 unacct:
1871         if (sbinfo->max_blocks)
1872                 percpu_counter_sub(&sbinfo->used_blocks,
1873                                 1 << compound_order(page));
1874         shmem_unacct_blocks(info->flags, 1 << compound_order(page));
1875 
1876         if (PageTransHuge(page)) {
1877                 unlock_page(page);
1878                 put_page(page);
1879                 goto alloc_nohuge;
1880         }
1881 failed:
1882         if (swap.val && !shmem_confirm_swap(mapping, index, swap))
1883                 error = -EEXIST;
1884 unlock:
1885         if (page) {
1886                 unlock_page(page);
1887                 put_page(page);
1888         }
1889         if (error == -ENOSPC && !once++) {
1890                 spin_lock_irq(&info->lock);
1891                 shmem_recalc_inode(inode);
1892                 spin_unlock_irq(&info->lock);
1893                 goto repeat;
1894         }
1895         if (error == -EEXIST)   /* from above or from radix_tree_insert */
1896                 goto repeat;
1897         return error;
1898 }
1899 
1900 /*
1901  * This is like autoremove_wake_function, but it removes the wait queue
1902  * entry unconditionally - even if something else had already woken the
1903  * target.
1904  */
1905 static int synchronous_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key)
1906 {
1907         int ret = default_wake_function(wait, mode, sync, key);
1908         list_del_init(&wait->task_list);
1909         return ret;
1910 }
1911 
1912 static int shmem_fault(struct vm_fault *vmf)
1913 {
1914         struct vm_area_struct *vma = vmf->vma;
1915         struct inode *inode = file_inode(vma->vm_file);
1916         gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
1917         enum sgp_type sgp;
1918         int error;
1919         int ret = VM_FAULT_LOCKED;
1920 
1921         /*
1922          * Trinity finds that probing a hole which tmpfs is punching can
1923          * prevent the hole-punch from ever completing: which in turn
1924          * locks writers out with its hold on i_mutex.  So refrain from
1925          * faulting pages into the hole while it's being punched.  Although
1926          * shmem_undo_range() does remove the additions, it may be unable to
1927          * keep up, as each new page needs its own unmap_mapping_range() call,
1928          * and the i_mmap tree grows ever slower to scan if new vmas are added.
1929          *
1930          * It does not matter if we sometimes reach this check just before the
1931          * hole-punch begins, so that one fault then races with the punch:
1932          * we just need to make racing faults a rare case.
1933          *
1934          * The implementation below would be much simpler if we just used a
1935          * standard mutex or completion: but we cannot take i_mutex in fault,
1936          * and bloating every shmem inode for this unlikely case would be sad.
1937          */
1938         if (unlikely(inode->i_private)) {
1939                 struct shmem_falloc *shmem_falloc;
1940 
1941                 spin_lock(&inode->i_lock);
1942                 shmem_falloc = inode->i_private;
1943                 if (shmem_falloc &&
1944                     shmem_falloc->waitq &&
1945                     vmf->pgoff >= shmem_falloc->start &&
1946                     vmf->pgoff < shmem_falloc->next) {
1947                         wait_queue_head_t *shmem_falloc_waitq;
1948                         DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
1949 
1950                         ret = VM_FAULT_NOPAGE;
1951                         if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) &&
1952                            !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) {
1953                                 /* It's polite to up mmap_sem if we can */
1954                                 up_read(&vma->vm_mm->mmap_sem);
1955                                 ret = VM_FAULT_RETRY;
1956                         }
1957 
1958                         shmem_falloc_waitq = shmem_falloc->waitq;
1959                         prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
1960                                         TASK_UNINTERRUPTIBLE);
1961                         spin_unlock(&inode->i_lock);
1962                         schedule();
1963 
1964                         /*
1965                          * shmem_falloc_waitq points into the shmem_fallocate()
1966                          * stack of the hole-punching task: shmem_falloc_waitq
1967                          * is usually invalid by the time we reach here, but
1968                          * finish_wait() does not dereference it in that case;
1969                          * though i_lock needed lest racing with wake_up_all().
1970                          */
1971                         spin_lock(&inode->i_lock);
1972                         finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
1973                         spin_unlock(&inode->i_lock);
1974                         return ret;
1975                 }
1976                 spin_unlock(&inode->i_lock);
1977         }
1978 
1979         sgp = SGP_CACHE;
1980         if (vma->vm_flags & VM_HUGEPAGE)
1981                 sgp = SGP_HUGE;
1982         else if (vma->vm_flags & VM_NOHUGEPAGE)
1983                 sgp = SGP_NOHUGE;
1984 
1985         error = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp,
1986                                   gfp, vma, vmf, &ret);
1987         if (error)
1988                 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
1989         return ret;
1990 }
1991 
1992 unsigned long shmem_get_unmapped_area(struct file *file,
1993                                       unsigned long uaddr, unsigned long len,
1994                                       unsigned long pgoff, unsigned long flags)
1995 {
1996         unsigned long (*get_area)(struct file *,
1997                 unsigned long, unsigned long, unsigned long, unsigned long);
1998         unsigned long addr;
1999         unsigned long offset;
2000         unsigned long inflated_len;
2001         unsigned long inflated_addr;
2002         unsigned long inflated_offset;
2003 
2004         if (len > TASK_SIZE)
2005                 return -ENOMEM;
2006 
2007         get_area = current->mm->get_unmapped_area;
2008         addr = get_area(file, uaddr, len, pgoff, flags);
2009 
2010         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
2011                 return addr;
2012         if (IS_ERR_VALUE(addr))
2013                 return addr;
2014         if (addr & ~PAGE_MASK)
2015                 return addr;
2016         if (addr > TASK_SIZE - len)
2017                 return addr;
2018 
2019         if (shmem_huge == SHMEM_HUGE_DENY)
2020                 return addr;
2021         if (len < HPAGE_PMD_SIZE)
2022                 return addr;
2023         if (flags & MAP_FIXED)
2024                 return addr;
2025         /*
2026          * Our priority is to support MAP_SHARED mapped hugely;
2027          * and support MAP_PRIVATE mapped hugely too, until it is COWed.
2028          * But if caller specified an address hint, respect that as before.
2029          */
2030         if (uaddr)
2031                 return addr;
2032 
2033         if (shmem_huge != SHMEM_HUGE_FORCE) {
2034                 struct super_block *sb;
2035 
2036                 if (file) {
2037                         VM_BUG_ON(file->f_op != &shmem_file_operations);
2038                         sb = file_inode(file)->i_sb;
2039                 } else {
2040                         /*
2041                          * Called directly from mm/mmap.c, or drivers/char/mem.c
2042                          * for "/dev/zero", to create a shared anonymous object.
2043                          */
2044                         if (IS_ERR(shm_mnt))
2045                                 return addr;
2046                         sb = shm_mnt->mnt_sb;
2047                 }
2048                 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER)
2049                         return addr;
2050         }
2051 
2052         offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1);
2053         if (offset && offset + len < 2 * HPAGE_PMD_SIZE)
2054                 return addr;
2055         if ((addr & (HPAGE_PMD_SIZE-1)) == offset)
2056                 return addr;
2057 
2058         inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE;
2059         if (inflated_len > TASK_SIZE)
2060                 return addr;
2061         if (inflated_len < len)
2062                 return addr;
2063 
2064         inflated_addr = get_area(NULL, 0, inflated_len, 0, flags);
2065         if (IS_ERR_VALUE(inflated_addr))
2066                 return addr;
2067         if (inflated_addr & ~PAGE_MASK)
2068                 return addr;
2069 
2070         inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1);
2071         inflated_addr += offset - inflated_offset;
2072         if (inflated_offset > offset)
2073                 inflated_addr += HPAGE_PMD_SIZE;
2074 
2075         if (inflated_addr > TASK_SIZE - len)
2076                 return addr;
2077         return inflated_addr;
2078 }
2079 
2080 #ifdef CONFIG_NUMA
2081 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
2082 {
2083         struct inode *inode = file_inode(vma->vm_file);
2084         return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
2085 }
2086 
2087 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
2088                                           unsigned long addr)
2089 {
2090         struct inode *inode = file_inode(vma->vm_file);
2091         pgoff_t index;
2092 
2093         index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2094         return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
2095 }
2096 #endif
2097 
2098 int shmem_lock(struct file *file, int lock, struct user_struct *user)
2099 {
2100         struct inode *inode = file_inode(file);
2101         struct shmem_inode_info *info = SHMEM_I(inode);
2102         int retval = -ENOMEM;
2103 
2104         spin_lock_irq(&info->lock);
2105         if (lock && !(info->flags & VM_LOCKED)) {
2106                 if (!user_shm_lock(inode->i_size, user))
2107                         goto out_nomem;
2108                 info->flags |= VM_LOCKED;
2109                 mapping_set_unevictable(file->f_mapping);
2110         }
2111         if (!lock && (info->flags & VM_LOCKED) && user) {
2112                 user_shm_unlock(inode->i_size, user);
2113                 info->flags &= ~VM_LOCKED;
2114                 mapping_clear_unevictable(file->f_mapping);
2115         }
2116         retval = 0;
2117 
2118 out_nomem:
2119         spin_unlock_irq(&info->lock);
2120         return retval;
2121 }
2122 
2123 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
2124 {
2125         file_accessed(file);
2126         vma->vm_ops = &shmem_vm_ops;
2127         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) &&
2128                         ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
2129                         (vma->vm_end & HPAGE_PMD_MASK)) {
2130                 khugepaged_enter(vma, vma->vm_flags);
2131         }
2132         return 0;
2133 }
2134 
2135 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
2136                                      umode_t mode, dev_t dev, unsigned long flags)
2137 {
2138         struct inode *inode;
2139         struct shmem_inode_info *info;
2140         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2141 
2142         if (shmem_reserve_inode(sb))
2143                 return NULL;
2144 
2145         inode = new_inode(sb);
2146         if (inode) {
2147                 inode->i_ino = get_next_ino();
2148                 inode_init_owner(inode, dir, mode);
2149                 inode->i_blocks = 0;
2150                 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
2151                 inode->i_generation = get_seconds();
2152                 info = SHMEM_I(inode);
2153                 memset(info, 0, (char *)inode - (char *)info);
2154                 spin_lock_init(&info->lock);
2155                 info->seals = F_SEAL_SEAL;
2156                 info->flags = flags & VM_NORESERVE;
2157                 INIT_LIST_HEAD(&info->shrinklist);
2158                 INIT_LIST_HEAD(&info->swaplist);
2159                 simple_xattrs_init(&info->xattrs);
2160                 cache_no_acl(inode);
2161 
2162                 switch (mode & S_IFMT) {
2163                 default:
2164                         inode->i_op = &shmem_special_inode_operations;
2165                         init_special_inode(inode, mode, dev);
2166                         break;
2167                 case S_IFREG:
2168                         inode->i_mapping->a_ops = &shmem_aops;
2169                         inode->i_op = &shmem_inode_operations;
2170                         inode->i_fop = &shmem_file_operations;
2171                         mpol_shared_policy_init(&info->policy,
2172                                                  shmem_get_sbmpol(sbinfo));
2173                         break;
2174                 case S_IFDIR:
2175                         inc_nlink(inode);
2176                         /* Some things misbehave if size == 0 on a directory */
2177                         inode->i_size = 2 * BOGO_DIRENT_SIZE;
2178                         inode->i_op = &shmem_dir_inode_operations;
2179                         inode->i_fop = &simple_dir_operations;
2180                         break;
2181                 case S_IFLNK:
2182                         /*
2183                          * Must not load anything in the rbtree,
2184                          * mpol_free_shared_policy will not be called.
2185                          */
2186                         mpol_shared_policy_init(&info->policy, NULL);
2187                         break;
2188                 }
2189         } else
2190                 shmem_free_inode(sb);
2191         return inode;
2192 }
2193 
2194 bool shmem_mapping(struct address_space *mapping)
2195 {
2196         return mapping->a_ops == &shmem_aops;
2197 }
2198 
2199 int shmem_mcopy_atomic_pte(struct mm_struct *dst_mm,
2200                            pmd_t *dst_pmd,
2201                            struct vm_area_struct *dst_vma,
2202                            unsigned long dst_addr,
2203                            unsigned long src_addr,
2204                            struct page **pagep)
2205 {
2206         struct inode *inode = file_inode(dst_vma->vm_file);
2207         struct shmem_inode_info *info = SHMEM_I(inode);
2208         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
2209         struct address_space *mapping = inode->i_mapping;
2210         gfp_t gfp = mapping_gfp_mask(mapping);
2211         pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
2212         struct mem_cgroup *memcg;
2213         spinlock_t *ptl;
2214         void *page_kaddr;
2215         struct page *page;
2216         pte_t _dst_pte, *dst_pte;
2217         int ret;
2218 
2219         ret = -ENOMEM;
2220         if (shmem_acct_block(info->flags, 1))
2221                 goto out;
2222         if (sbinfo->max_blocks) {
2223                 if (percpu_counter_compare(&sbinfo->used_blocks,
2224                                            sbinfo->max_blocks) >= 0)
2225                         goto out_unacct_blocks;
2226                 percpu_counter_inc(&sbinfo->used_blocks);
2227         }
2228 
2229         if (!*pagep) {
2230                 page = shmem_alloc_page(gfp, info, pgoff);
2231                 if (!page)
2232                         goto out_dec_used_blocks;
2233 
2234                 page_kaddr = kmap_atomic(page);
2235                 ret = copy_from_user(page_kaddr, (const void __user *)src_addr,
2236                                      PAGE_SIZE);
2237                 kunmap_atomic(page_kaddr);
2238 
2239                 /* fallback to copy_from_user outside mmap_sem */
2240                 if (unlikely(ret)) {
2241                         *pagep = page;
2242                         if (sbinfo->max_blocks)
2243                                 percpu_counter_add(&sbinfo->used_blocks, -1);
2244                         shmem_unacct_blocks(info->flags, 1);
2245                         /* don't free the page */
2246                         return -EFAULT;
2247                 }
2248         } else {
2249                 page = *pagep;
2250                 *pagep = NULL;
2251         }
2252 
2253         VM_BUG_ON(PageLocked(page) || PageSwapBacked(page));
2254         __SetPageLocked(page);
2255         __SetPageSwapBacked(page);
2256         __SetPageUptodate(page);
2257 
2258         ret = mem_cgroup_try_charge(page, dst_mm, gfp, &memcg, false);
2259         if (ret)
2260                 goto out_release;
2261 
2262         ret = radix_tree_maybe_preload(gfp & GFP_RECLAIM_MASK);
2263         if (!ret) {
2264                 ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL);
2265                 radix_tree_preload_end();
2266         }
2267         if (ret)
2268                 goto out_release_uncharge;
2269 
2270         mem_cgroup_commit_charge(page, memcg, false, false);
2271 
2272         _dst_pte = mk_pte(page, dst_vma->vm_page_prot);
2273         if (dst_vma->vm_flags & VM_WRITE)
2274                 _dst_pte = pte_mkwrite(pte_mkdirty(_dst_pte));
2275 
2276         ret = -EEXIST;
2277         dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
2278         if (!pte_none(*dst_pte))
2279                 goto out_release_uncharge_unlock;
2280 
2281         lru_cache_add_anon(page);
2282 
2283         spin_lock(&info->lock);
2284         info->alloced++;
2285         inode->i_blocks += BLOCKS_PER_PAGE;
2286         shmem_recalc_inode(inode);
2287         spin_unlock(&info->lock);
2288 
2289         inc_mm_counter(dst_mm, mm_counter_file(page));
2290         page_add_file_rmap(page, false);
2291         set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
2292 
2293         /* No need to invalidate - it was non-present before */
2294         update_mmu_cache(dst_vma, dst_addr, dst_pte);
2295         unlock_page(page);
2296         pte_unmap_unlock(dst_pte, ptl);
2297         ret = 0;
2298 out:
2299         return ret;
2300 out_release_uncharge_unlock:
2301         pte_unmap_unlock(dst_pte, ptl);
2302 out_release_uncharge:
2303         mem_cgroup_cancel_charge(page, memcg, false);
2304 out_release:
2305         unlock_page(page);
2306         put_page(page);
2307 out_dec_used_blocks:
2308         if (sbinfo->max_blocks)
2309                 percpu_counter_add(&sbinfo->used_blocks, -1);
2310 out_unacct_blocks:
2311         shmem_unacct_blocks(info->flags, 1);
2312         goto out;
2313 }
2314 
2315 #ifdef CONFIG_TMPFS
2316 static const struct inode_operations shmem_symlink_inode_operations;
2317 static const struct inode_operations shmem_short_symlink_operations;
2318 
2319 #ifdef CONFIG_TMPFS_XATTR
2320 static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
2321 #else
2322 #define shmem_initxattrs NULL
2323 #endif
2324 
2325 static int
2326 shmem_write_begin(struct file *file, struct address_space *mapping,
2327                         loff_t pos, unsigned len, unsigned flags,
2328                         struct page **pagep, void **fsdata)
2329 {
2330         struct inode *inode = mapping->host;
2331         struct shmem_inode_info *info = SHMEM_I(inode);
2332         pgoff_t index = pos >> PAGE_SHIFT;
2333 
2334         /* i_mutex is held by caller */
2335         if (unlikely(info->seals & (F_SEAL_WRITE | F_SEAL_GROW))) {
2336                 if (info->seals & F_SEAL_WRITE)
2337                         return -EPERM;
2338                 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
2339                         return -EPERM;
2340         }
2341 
2342         return shmem_getpage(inode, index, pagep, SGP_WRITE);
2343 }
2344 
2345 static int
2346 shmem_write_end(struct file *file, struct address_space *mapping,
2347                         loff_t pos, unsigned len, unsigned copied,
2348                         struct page *page, void *fsdata)
2349 {
2350         struct inode *inode = mapping->host;
2351 
2352         if (pos + copied > inode->i_size)
2353                 i_size_write(inode, pos + copied);
2354 
2355         if (!PageUptodate(page)) {
2356                 struct page *head = compound_head(page);
2357                 if (PageTransCompound(page)) {
2358                         int i;
2359 
2360                         for (i = 0; i < HPAGE_PMD_NR; i++) {
2361                                 if (head + i == page)
2362                                         continue;
2363                                 clear_highpage(head + i);
2364                                 flush_dcache_page(head + i);
2365                         }
2366                 }
2367                 if (copied < PAGE_SIZE) {
2368                         unsigned from = pos & (PAGE_SIZE - 1);
2369                         zero_user_segments(page, 0, from,
2370                                         from + copied, PAGE_SIZE);
2371                 }
2372                 SetPageUptodate(head);
2373         }
2374         set_page_dirty(page);
2375         unlock_page(page);
2376         put_page(page);
2377 
2378         return copied;
2379 }
2380 
2381 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
2382 {
2383         struct file *file = iocb->ki_filp;
2384         struct inode *inode = file_inode(file);
2385         struct address_space *mapping = inode->i_mapping;
2386         pgoff_t index;
2387         unsigned long offset;
2388         enum sgp_type sgp = SGP_READ;
2389         int error = 0;
2390         ssize_t retval = 0;
2391         loff_t *ppos = &iocb->ki_pos;
2392 
2393         /*
2394          * Might this read be for a stacking filesystem?  Then when reading
2395          * holes of a sparse file, we actually need to allocate those pages,
2396          * and even mark them dirty, so it cannot exceed the max_blocks limit.
2397          */
2398         if (!iter_is_iovec(to))
2399                 sgp = SGP_CACHE;
2400 
2401         index = *ppos >> PAGE_SHIFT;
2402         offset = *ppos & ~PAGE_MASK;
2403 
2404         for (;;) {
2405                 struct page *page = NULL;
2406                 pgoff_t end_index;
2407                 unsigned long nr, ret;
2408                 loff_t i_size = i_size_read(inode);
2409 
2410                 end_index = i_size >> PAGE_SHIFT;
2411                 if (index > end_index)
2412                         break;
2413                 if (index == end_index) {
2414                         nr = i_size & ~PAGE_MASK;
2415                         if (nr <= offset)
2416                                 break;
2417                 }
2418 
2419                 error = shmem_getpage(inode, index, &page, sgp);
2420                 if (error) {
2421                         if (error == -EINVAL)
2422                                 error = 0;
2423                         break;
2424                 }
2425                 if (page) {
2426                         if (sgp == SGP_CACHE)
2427                                 set_page_dirty(page);
2428                         unlock_page(page);
2429                 }
2430 
2431                 /*
2432                  * We must evaluate after, since reads (unlike writes)
2433                  * are called without i_mutex protection against truncate
2434                  */
2435                 nr = PAGE_SIZE;
2436                 i_size = i_size_read(inode);
2437                 end_index = i_size >> PAGE_SHIFT;
2438                 if (index == end_index) {
2439                         nr = i_size & ~PAGE_MASK;
2440                         if (nr <= offset) {
2441                                 if (page)
2442                                         put_page(page);
2443                                 break;
2444                         }
2445                 }
2446                 nr -= offset;
2447 
2448                 if (page) {
2449                         /*
2450                          * If users can be writing to this page using arbitrary
2451                          * virtual addresses, take care about potential aliasing
2452                          * before reading the page on the kernel side.
2453                          */
2454                         if (mapping_writably_mapped(mapping))
2455                                 flush_dcache_page(page);
2456                         /*
2457                          * Mark the page accessed if we read the beginning.
2458                          */
2459                         if (!offset)
2460                                 mark_page_accessed(page);
2461                 } else {
2462                         page = ZERO_PAGE(0);
2463                         get_page(page);
2464                 }
2465 
2466                 /*
2467                  * Ok, we have the page, and it's up-to-date, so
2468                  * now we can copy it to user space...
2469                  */
2470                 ret = copy_page_to_iter(page, offset, nr, to);
2471                 retval += ret;
2472                 offset += ret;
2473                 index += offset >> PAGE_SHIFT;
2474                 offset &= ~PAGE_MASK;
2475 
2476                 put_page(page);
2477                 if (!iov_iter_count(to))
2478                         break;
2479                 if (ret < nr) {
2480                         error = -EFAULT;
2481                         break;
2482                 }
2483                 cond_resched();
2484         }
2485 
2486         *ppos = ((loff_t) index << PAGE_SHIFT) + offset;
2487         file_accessed(file);
2488         return retval ? retval : error;
2489 }
2490 
2491 /*
2492  * llseek SEEK_DATA or SEEK_HOLE through the radix_tree.
2493  */
2494 static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
2495                                     pgoff_t index, pgoff_t end, int whence)
2496 {
2497         struct page *page;
2498         struct pagevec pvec;
2499         pgoff_t indices[PAGEVEC_SIZE];
2500         bool done = false;
2501         int i;
2502 
2503         pagevec_init(&pvec, 0);
2504         pvec.nr = 1;            /* start small: we may be there already */
2505         while (!done) {
2506                 pvec.nr = find_get_entries(mapping, index,
2507                                         pvec.nr, pvec.pages, indices);
2508                 if (!pvec.nr) {
2509                         if (whence == SEEK_DATA)
2510                                 index = end;
2511                         break;
2512                 }
2513                 for (i = 0; i < pvec.nr; i++, index++) {
2514                         if (index < indices[i]) {
2515                                 if (whence == SEEK_HOLE) {
2516                                         done = true;
2517                                         break;
2518                                 }
2519                                 index = indices[i];
2520                         }
2521                         page = pvec.pages[i];
2522                         if (page && !radix_tree_exceptional_entry(page)) {
2523                                 if (!PageUptodate(page))
2524                                         page = NULL;
2525                         }
2526                         if (index >= end ||
2527                             (page && whence == SEEK_DATA) ||
2528                             (!page && whence == SEEK_HOLE)) {
2529                                 done = true;
2530                                 break;
2531                         }
2532                 }
2533                 pagevec_remove_exceptionals(&pvec);
2534                 pagevec_release(&pvec);
2535                 pvec.nr = PAGEVEC_SIZE;
2536                 cond_resched();
2537         }
2538         return index;
2539 }
2540 
2541 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
2542 {
2543         struct address_space *mapping = file->f_mapping;
2544         struct inode *inode = mapping->host;
2545         pgoff_t start, end;
2546         loff_t new_offset;
2547 
2548         if (whence != SEEK_DATA && whence != SEEK_HOLE)
2549                 return generic_file_llseek_size(file, offset, whence,
2550                                         MAX_LFS_FILESIZE, i_size_read(inode));
2551         inode_lock(inode);
2552         /* We're holding i_mutex so we can access i_size directly */
2553 
2554         if (offset < 0)
2555                 offset = -EINVAL;
2556         else if (offset >= inode->i_size)
2557                 offset = -ENXIO;
2558         else {
2559                 start = offset >> PAGE_SHIFT;
2560                 end = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT;
2561                 new_offset = shmem_seek_hole_data(mapping, start, end, whence);
2562                 new_offset <<= PAGE_SHIFT;
2563                 if (new_offset > offset) {
2564                         if (new_offset < inode->i_size)
2565                                 offset = new_offset;
2566                         else if (whence == SEEK_DATA)
2567                                 offset = -ENXIO;
2568                         else
2569                                 offset = inode->i_size;
2570                 }
2571         }
2572 
2573         if (offset >= 0)
2574                 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
2575         inode_unlock(inode);
2576         return offset;
2577 }
2578 
2579 /*
2580  * We need a tag: a new tag would expand every radix_tree_node by 8 bytes,
2581  * so reuse a tag which we firmly believe is never set or cleared on shmem.
2582  */
2583 #define SHMEM_TAG_PINNED        PAGECACHE_TAG_TOWRITE
2584 #define LAST_SCAN               4       /* about 150ms max */
2585 
2586 static void shmem_tag_pins(struct address_space *mapping)
2587 {
2588         struct radix_tree_iter iter;
2589         void **slot;
2590         pgoff_t start;
2591         struct page *page;
2592 
2593         lru_add_drain();
2594         start = 0;
2595         rcu_read_lock();
2596 
2597         radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
2598                 page = radix_tree_deref_slot(slot);
2599                 if (!page || radix_tree_exception(page)) {
2600                         if (radix_tree_deref_retry(page)) {
2601                                 slot = radix_tree_iter_retry(&iter);
2602                                 continue;
2603                         }
2604                 } else if (page_count(page) - page_mapcount(page) > 1) {
2605                         spin_lock_irq(&mapping->tree_lock);
2606                         radix_tree_tag_set(&mapping->page_tree, iter.index,
2607                                            SHMEM_TAG_PINNED);
2608                         spin_unlock_irq(&mapping->tree_lock);
2609                 }
2610 
2611                 if (need_resched()) {
2612                         slot = radix_tree_iter_resume(slot, &iter);
2613                         cond_resched_rcu();
2614                 }
2615         }
2616         rcu_read_unlock();
2617 }
2618 
2619 /*
2620  * Setting SEAL_WRITE requires us to verify there's no pending writer. However,
2621  * via get_user_pages(), drivers might have some pending I/O without any active
2622  * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all pages
2623  * and see whether it has an elevated ref-count. If so, we tag them and wait for
2624  * them to be dropped.
2625  * The caller must guarantee that no new user will acquire writable references
2626  * to those pages to avoid races.
2627  */
2628 static int shmem_wait_for_pins(struct address_space *mapping)
2629 {
2630         struct radix_tree_iter iter;
2631         void **slot;
2632         pgoff_t start;
2633         struct page *page;
2634         int error, scan;
2635 
2636         shmem_tag_pins(mapping);
2637 
2638         error = 0;
2639         for (scan = 0; scan <= LAST_SCAN; scan++) {
2640                 if (!radix_tree_tagged(&mapping->page_tree, SHMEM_TAG_PINNED))
2641                         break;
2642 
2643                 if (!scan)
2644                         lru_add_drain_all();
2645                 else if (schedule_timeout_killable((HZ << scan) / 200))
2646                         scan = LAST_SCAN;
2647 
2648                 start = 0;
2649                 rcu_read_lock();
2650                 radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter,
2651                                            start, SHMEM_TAG_PINNED) {
2652 
2653                         page = radix_tree_deref_slot(slot);
2654                         if (radix_tree_exception(page)) {
2655                                 if (radix_tree_deref_retry(page)) {
2656                                         slot = radix_tree_iter_retry(&iter);
2657                                         continue;
2658                                 }
2659 
2660                                 page = NULL;
2661                         }
2662 
2663                         if (page &&
2664                             page_count(page) - page_mapcount(page) != 1) {
2665                                 if (scan < LAST_SCAN)
2666                                         goto continue_resched;
2667 
2668                                 /*
2669                                  * On the last scan, we clean up all those tags
2670                                  * we inserted; but make a note that we still
2671                                  * found pages pinned.
2672                                  */
2673                                 error = -EBUSY;
2674                         }
2675 
2676                         spin_lock_irq(&mapping->tree_lock);
2677                         radix_tree_tag_clear(&mapping->page_tree,
2678                                              iter.index, SHMEM_TAG_PINNED);
2679                         spin_unlock_irq(&mapping->tree_lock);
2680 continue_resched:
2681                         if (need_resched()) {
2682                                 slot = radix_tree_iter_resume(slot, &iter);
2683                                 cond_resched_rcu();
2684                         }
2685                 }
2686                 rcu_read_unlock();
2687         }
2688 
2689         return error;
2690 }
2691 
2692 #define F_ALL_SEALS (F_SEAL_SEAL | \
2693                      F_SEAL_SHRINK | \
2694                      F_SEAL_GROW | \
2695                      F_SEAL_WRITE)
2696 
2697 int shmem_add_seals(struct file *file, unsigned int seals)
2698 {
2699         struct inode *inode = file_inode(file);
2700         struct shmem_inode_info *info = SHMEM_I(inode);
2701         int error;
2702 
2703         /*
2704          * SEALING
2705          * Sealing allows multiple parties to share a shmem-file but restrict
2706          * access to a specific subset of file operations. Seals can only be
2707          * added, but never removed. This way, mutually untrusted parties can
2708          * share common memory regions with a well-defined policy. A malicious
2709          * peer can thus never perform unwanted operations on a shared object.
2710          *
2711          * Seals are only supported on special shmem-files and always affect
2712          * the whole underlying inode. Once a seal is set, it may prevent some
2713          * kinds of access to the file. Currently, the following seals are
2714          * defined:
2715          *   SEAL_SEAL: Prevent further seals from being set on this file
2716          *   SEAL_SHRINK: Prevent the file from shrinking
2717          *   SEAL_GROW: Prevent the file from growing
2718          *   SEAL_WRITE: Prevent write access to the file
2719          *
2720          * As we don't require any trust relationship between two parties, we
2721          * must prevent seals from being removed. Therefore, sealing a file
2722          * only adds a given set of seals to the file, it never touches
2723          * existing seals. Furthermore, the "setting seals"-operation can be
2724          * sealed itself, which basically prevents any further seal from being
2725          * added.
2726          *
2727          * Semantics of sealing are only defined on volatile files. Only
2728          * anonymous shmem files support sealing. More importantly, seals are
2729          * never written to disk. Therefore, there's no plan to support it on
2730          * other file types.
2731          */
2732 
2733         if (file->f_op != &shmem_file_operations)
2734                 return -EINVAL;
2735         if (!(file->f_mode & FMODE_WRITE))
2736                 return -EPERM;
2737         if (seals & ~(unsigned int)F_ALL_SEALS)
2738                 return -EINVAL;
2739 
2740         inode_lock(inode);
2741 
2742         if (info->seals & F_SEAL_SEAL) {
2743                 error = -EPERM;
2744                 goto unlock;
2745         }
2746 
2747         if ((seals & F_SEAL_WRITE) && !(info->seals & F_SEAL_WRITE)) {
2748                 error = mapping_deny_writable(file->f_mapping);
2749                 if (error)
2750                         goto unlock;
2751 
2752                 error = shmem_wait_for_pins(file->f_mapping);
2753                 if (error) {
2754                         mapping_allow_writable(file->f_mapping);
2755                         goto unlock;
2756                 }
2757         }
2758 
2759         info->seals |= seals;
2760         error = 0;
2761 
2762 unlock:
2763         inode_unlock(inode);
2764         return error;
2765 }
2766 EXPORT_SYMBOL_GPL(shmem_add_seals);
2767 
2768 int shmem_get_seals(struct file *file)
2769 {
2770         if (file->f_op != &shmem_file_operations)
2771                 return -EINVAL;
2772 
2773         return SHMEM_I(file_inode(file))->seals;
2774 }
2775 EXPORT_SYMBOL_GPL(shmem_get_seals);
2776 
2777 long shmem_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
2778 {
2779         long error;
2780 
2781         switch (cmd) {
2782         case F_ADD_SEALS:
2783                 /* disallow upper 32bit */
2784                 if (arg > UINT_MAX)
2785                         return -EINVAL;
2786 
2787                 error = shmem_add_seals(file, arg);
2788                 break;
2789         case F_GET_SEALS:
2790                 error = shmem_get_seals(file);
2791                 break;
2792         default:
2793                 error = -EINVAL;
2794                 break;
2795         }
2796 
2797         return error;
2798 }
2799 
2800 static long shmem_fallocate(struct file *file, int mode, loff_t offset,
2801                                                          loff_t len)
2802 {
2803         struct inode *inode = file_inode(file);
2804         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
2805         struct shmem_inode_info *info = SHMEM_I(inode);
2806         struct shmem_falloc shmem_falloc;
2807         pgoff_t start, index, end;
2808         int error;
2809 
2810         if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
2811                 return -EOPNOTSUPP;
2812 
2813         inode_lock(inode);
2814 
2815         if (mode & FALLOC_FL_PUNCH_HOLE) {
2816                 struct address_space *mapping = file->f_mapping;
2817                 loff_t unmap_start = round_up(offset, PAGE_SIZE);
2818                 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
2819                 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
2820 
2821                 /* protected by i_mutex */
2822                 if (info->seals & F_SEAL_WRITE) {
2823                         error = -EPERM;
2824                         goto out;
2825                 }
2826 
2827                 shmem_falloc.waitq = &shmem_falloc_waitq;
2828                 shmem_falloc.start = unmap_start >> PAGE_SHIFT;
2829                 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
2830                 spin_lock(&inode->i_lock);
2831                 inode->i_private = &shmem_falloc;
2832                 spin_unlock(&inode->i_lock);
2833 
2834                 if ((u64)unmap_end > (u64)unmap_start)
2835                         unmap_mapping_range(mapping, unmap_start,
2836                                             1 + unmap_end - unmap_start, 0);
2837                 shmem_truncate_range(inode, offset, offset + len - 1);
2838                 /* No need to unmap again: hole-punching leaves COWed pages */
2839 
2840                 spin_lock(&inode->i_lock);
2841                 inode->i_private = NULL;
2842                 wake_up_all(&shmem_falloc_waitq);
2843                 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.task_list));
2844                 spin_unlock(&inode->i_lock);
2845                 error = 0;
2846                 goto out;
2847         }
2848 
2849         /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
2850         error = inode_newsize_ok(inode, offset + len);
2851         if (error)
2852                 goto out;
2853 
2854         if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
2855                 error = -EPERM;
2856                 goto out;
2857         }
2858 
2859         start = offset >> PAGE_SHIFT;
2860         end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
2861         /* Try to avoid a swapstorm if len is impossible to satisfy */
2862         if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
2863                 error = -ENOSPC;
2864                 goto out;
2865         }
2866 
2867         shmem_falloc.waitq = NULL;
2868         shmem_falloc.start = start;
2869         shmem_falloc.next  = start;
2870         shmem_falloc.nr_falloced = 0;
2871         shmem_falloc.nr_unswapped = 0;
2872         spin_lock(&inode->i_lock);
2873         inode->i_private = &shmem_falloc;
2874         spin_unlock(&inode->i_lock);
2875 
2876         for (index = start; index < end; index++) {
2877                 struct page *page;
2878 
2879                 /*
2880                  * Good, the fallocate(2) manpage permits EINTR: we may have
2881                  * been interrupted because we are using up too much memory.
2882                  */
2883                 if (signal_pending(current))
2884                         error = -EINTR;
2885                 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
2886                         error = -ENOMEM;
2887                 else
2888                         error = shmem_getpage(inode, index, &page, SGP_FALLOC);
2889                 if (error) {
2890                         /* Remove the !PageUptodate pages we added */
2891                         if (index > start) {
2892                                 shmem_undo_range(inode,
2893                                     (loff_t)start << PAGE_SHIFT,
2894                                     ((loff_t)index << PAGE_SHIFT) - 1, true);
2895                         }
2896                         goto undone;
2897                 }
2898 
2899                 /*
2900                  * Inform shmem_writepage() how far we have reached.
2901                  * No need for lock or barrier: we have the page lock.
2902                  */
2903                 shmem_falloc.next++;
2904                 if (!PageUptodate(page))
2905                         shmem_falloc.nr_falloced++;
2906 
2907                 /*
2908                  * If !PageUptodate, leave it that way so that freeable pages
2909                  * can be recognized if we need to rollback on error later.
2910                  * But set_page_dirty so that memory pressure will swap rather
2911                  * than free the pages we are allocating (and SGP_CACHE pages
2912                  * might still be clean: we now need to mark those dirty too).
2913                  */
2914                 set_page_dirty(page);
2915                 unlock_page(page);
2916                 put_page(page);
2917                 cond_resched();
2918         }
2919 
2920         if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
2921                 i_size_write(inode, offset + len);
2922         inode->i_ctime = current_time(inode);
2923 undone:
2924         spin_lock(&inode->i_lock);
2925         inode->i_private = NULL;
2926         spin_unlock(&inode->i_lock);
2927 out:
2928         inode_unlock(inode);
2929         return error;
2930 }
2931 
2932 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
2933 {
2934         struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
2935 
2936         buf->f_type = TMPFS_MAGIC;
2937         buf->f_bsize = PAGE_SIZE;
2938         buf->f_namelen = NAME_MAX;
2939         if (sbinfo->max_blocks) {
2940                 buf->f_blocks = sbinfo->max_blocks;
2941                 buf->f_bavail =
2942                 buf->f_bfree  = sbinfo->max_blocks -
2943                                 percpu_counter_sum(&sbinfo->used_blocks);
2944         }
2945         if (sbinfo->max_inodes) {
2946                 buf->f_files = sbinfo->max_inodes;
2947                 buf->f_ffree = sbinfo->free_inodes;
2948         }
2949         /* else leave those fields 0 like simple_statfs */
2950         return 0;
2951 }
2952 
2953 /*
2954  * File creation. Allocate an inode, and we're done..
2955  */
2956 static int
2957 shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
2958 {
2959         struct inode *inode;
2960         int error = -ENOSPC;
2961 
2962         inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
2963         if (inode) {
2964                 error = simple_acl_create(dir, inode);
2965                 if (error)
2966                         goto out_iput;
2967                 error = security_inode_init_security(inode, dir,
2968                                                      &dentry->d_name,
2969                                                      shmem_initxattrs, NULL);
2970                 if (error && error != -EOPNOTSUPP)
2971                         goto out_iput;
2972 
2973                 error = 0;
2974                 dir->i_size += BOGO_DIRENT_SIZE;
2975                 dir->i_ctime = dir->i_mtime = current_time(dir);
2976                 d_instantiate(dentry, inode);
2977                 dget(dentry); /* Extra count - pin the dentry in core */
2978         }
2979         return error;
2980 out_iput:
2981         iput(inode);
2982         return error;
2983 }
2984 
2985 static int
2986 shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
2987 {
2988         struct inode *inode;
2989         int error = -ENOSPC;
2990 
2991         inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE);
2992         if (inode) {
2993                 error = security_inode_init_security(inode, dir,
2994                                                      NULL,
2995                                                      shmem_initxattrs, NULL);
2996                 if (error && error != -EOPNOTSUPP)
2997                         goto out_iput;
2998                 error = simple_acl_create(dir, inode);
2999                 if (error)
3000                         goto out_iput;
3001                 d_tmpfile(dentry, inode);
3002         }
3003         return error;
3004 out_iput:
3005         iput(inode);
3006         return error;
3007 }
3008 
3009 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
3010 {
3011         int error;
3012 
3013         if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
3014                 return error;
3015         inc_nlink(dir);
3016         return 0;
3017 }
3018 
3019 static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode,
3020                 bool excl)
3021 {
3022         return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
3023 }
3024 
3025 /*
3026  * Link a file..
3027  */
3028 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
3029 {
3030         struct inode *inode = d_inode(old_dentry);
3031         int ret;
3032 
3033         /*
3034          * No ordinary (disk based) filesystem counts links as inodes;
3035          * but each new link needs a new dentry, pinning lowmem, and
3036          * tmpfs dentries cannot be pruned until they are unlinked.
3037          */
3038         ret = shmem_reserve_inode(inode->i_sb);
3039         if (ret)
3040                 goto out;
3041 
3042         dir->i_size += BOGO_DIRENT_SIZE;
3043         inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
3044         inc_nlink(inode);
3045         ihold(inode);   /* New dentry reference */
3046         dget(dentry);           /* Extra pinning count for the created dentry */
3047         d_instantiate(dentry, inode);
3048 out:
3049         return ret;
3050 }
3051 
3052 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
3053 {
3054         struct inode *inode = d_inode(dentry);
3055 
3056         if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
3057                 shmem_free_inode(inode->i_sb);
3058 
3059         dir->i_size -= BOGO_DIRENT_SIZE;
3060         inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
3061         drop_nlink(inode);
3062         dput(dentry);   /* Undo the count from "create" - this does all the work */
3063         return 0;
3064 }
3065 
3066 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
3067 {
3068         if (!simple_empty(dentry))
3069                 return -ENOTEMPTY;
3070 
3071         drop_nlink(d_inode(dentry));
3072         drop_nlink(dir);
3073         return shmem_unlink(dir, dentry);
3074 }
3075 
3076 static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
3077 {
3078         bool old_is_dir = d_is_dir(old_dentry);
3079         bool new_is_dir = d_is_dir(new_dentry);
3080 
3081         if (old_dir != new_dir && old_is_dir != new_is_dir) {
3082                 if (old_is_dir) {
3083                         drop_nlink(old_dir);
3084                         inc_nlink(new_dir);
3085                 } else {
3086                         drop_nlink(new_dir);
3087                         inc_nlink(old_dir);
3088                 }
3089         }
3090         old_dir->i_ctime = old_dir->i_mtime =
3091         new_dir->i_ctime = new_dir->i_mtime =
3092         d_inode(old_dentry)->i_ctime =
3093         d_inode(new_dentry)->i_ctime = current_time(old_dir);
3094 
3095         return 0;
3096 }
3097 
3098 static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry)
3099 {
3100         struct dentry *whiteout;
3101         int error;
3102 
3103         whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
3104         if (!whiteout)
3105                 return -ENOMEM;
3106 
3107         error = shmem_mknod(old_dir, whiteout,
3108                             S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
3109         dput(whiteout);
3110         if (error)
3111                 return error;
3112 
3113         /*
3114          * Cheat and hash the whiteout while the old dentry is still in
3115          * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
3116          *
3117          * d_lookup() will consistently find one of them at this point,
3118          * not sure which one, but that isn't even important.
3119          */
3120         d_rehash(whiteout);
3121         return 0;
3122 }
3123 
3124 /*
3125  * The VFS layer already does all the dentry stuff for rename,
3126  * we just have to decrement the usage count for the target if
3127  * it exists so that the VFS layer correctly free's it when it
3128  * gets overwritten.
3129  */
3130 static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags)
3131 {
3132         struct inode *inode = d_inode(old_dentry);
3133         int they_are_dirs = S_ISDIR(inode->i_mode);
3134 
3135         if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
3136                 return -EINVAL;
3137 
3138         if (flags & RENAME_EXCHANGE)
3139                 return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry);
3140 
3141         if (!simple_empty(new_dentry))
3142                 return -ENOTEMPTY;
3143 
3144         if (flags & RENAME_WHITEOUT) {
3145                 int error;
3146 
3147                 error = shmem_whiteout(old_dir, old_dentry);
3148                 if (error)
3149                         return error;
3150         }
3151 
3152         if (d_really_is_positive(new_dentry)) {
3153                 (void) shmem_unlink(new_dir, new_dentry);
3154                 if (they_are_dirs) {
3155                         drop_nlink(d_inode(new_dentry));
3156                         drop_nlink(old_dir);
3157                 }
3158         } else if (they_are_dirs) {
3159                 drop_nlink(old_dir);
3160                 inc_nlink(new_dir);
3161         }
3162 
3163         old_dir->i_size -= BOGO_DIRENT_SIZE;
3164         new_dir->i_size += BOGO_DIRENT_SIZE;
3165         old_dir->i_ctime = old_dir->i_mtime =
3166         new_dir->i_ctime = new_dir->i_mtime =
3167         inode->i_ctime = current_time(old_dir);
3168         return 0;
3169 }
3170 
3171 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
3172 {
3173         int error;
3174         int len;
3175         struct inode *inode;
3176         struct page *page;
3177         struct shmem_inode_info *info;
3178 
3179         len = strlen(symname) + 1;
3180         if (len > PAGE_SIZE)
3181                 return -ENAMETOOLONG;
3182 
3183         inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
3184         if (!inode)
3185                 return -ENOSPC;
3186 
3187         error = security_inode_init_security(inode, dir, &dentry->d_name,
3188                                              shmem_initxattrs, NULL);
3189         if (error) {
3190                 if (error != -EOPNOTSUPP) {
3191                         iput(inode);
3192                         return error;
3193                 }
3194                 error = 0;
3195         }
3196 
3197         info = SHMEM_I(inode);
3198         inode->i_size = len-1;
3199         if (len <= SHORT_SYMLINK_LEN) {
3200                 inode->i_link = kmemdup(symname, len, GFP_KERNEL);
3201                 if (!inode->i_link) {
3202                         iput(inode);
3203                         return -ENOMEM;
3204                 }
3205                 inode->i_op = &shmem_short_symlink_operations;
3206         } else {
3207                 inode_nohighmem(inode);
3208                 error = shmem_getpage(inode, 0, &page, SGP_WRITE);
3209                 if (error) {
3210                         iput(inode);
3211                         return error;
3212                 }
3213                 inode->i_mapping->a_ops = &shmem_aops;
3214                 inode->i_op = &shmem_symlink_inode_operations;
3215                 memcpy(page_address(page), symname, len);
3216                 SetPageUptodate(page);
3217                 set_page_dirty(page);
3218                 unlock_page(page);
3219                 put_page(page);
3220         }
3221         dir->i_size += BOGO_DIRENT_SIZE;
3222         dir->i_ctime = dir->i_mtime = current_time(dir);
3223         d_instantiate(dentry, inode);
3224         dget(dentry);
3225         return 0;
3226 }
3227 
3228 static void shmem_put_link(void *arg)
3229 {
3230         mark_page_accessed(arg);
3231         put_page(arg);
3232 }
3233 
3234 static const char *shmem_get_link(struct dentry *dentry,
3235                                   struct inode *inode,
3236                                   struct delayed_call *done)
3237 {
3238         struct page *page = NULL;
3239         int error;
3240         if (!dentry) {
3241                 page = find_get_page(inode->i_mapping, 0);
3242                 if (!page)
3243                         return ERR_PTR(-ECHILD);
3244                 if (!PageUptodate(page)) {
3245                         put_page(page);
3246                         return ERR_PTR(-ECHILD);
3247                 }
3248         } else {
3249                 error = shmem_getpage(inode, 0, &page, SGP_READ);
3250                 if (error)
3251                         return ERR_PTR(error);
3252                 unlock_page(page);
3253         }
3254         set_delayed_call(done, shmem_put_link, page);
3255         return page_address(page);
3256 }
3257 
3258 #ifdef CONFIG_TMPFS_XATTR
3259 /*
3260  * Superblocks without xattr inode operations may get some security.* xattr
3261  * support from the LSM "for free". As soon as we have any other xattrs
3262  * like ACLs, we also need to implement the security.* handlers at
3263  * filesystem level, though.
3264  */
3265 
3266 /*
3267  * Callback for security_inode_init_security() for acquiring xattrs.
3268  */
3269 static int shmem_initxattrs(struct inode *inode,
3270                             const struct xattr *xattr_array,
3271                             void *fs_info)
3272 {
3273         struct shmem_inode_info *info = SHMEM_I(inode);
3274         const struct xattr *xattr;
3275         struct simple_xattr *new_xattr;
3276         size_t len;
3277 
3278         for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3279                 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
3280                 if (!new_xattr)
3281                         return -ENOMEM;
3282 
3283                 len = strlen(xattr->name) + 1;
3284                 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
3285                                           GFP_KERNEL);
3286                 if (!new_xattr->name) {
3287                         kfree(new_xattr);
3288                         return -ENOMEM;
3289                 }
3290 
3291                 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
3292                        XATTR_SECURITY_PREFIX_LEN);
3293                 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
3294                        xattr->name, len);
3295 
3296                 simple_xattr_list_add(&info->xattrs, new_xattr);
3297         }
3298 
3299         return 0;
3300 }
3301 
3302 static int shmem_xattr_handler_get(const struct xattr_handler *handler,
3303                                    struct dentry *unused, struct inode *inode,
3304                                    const char *name, void *buffer, size_t size)
3305 {
3306         struct shmem_inode_info *info = SHMEM_I(inode);
3307 
3308         name = xattr_full_name(handler, name);
3309         return simple_xattr_get(&info->xattrs, name, buffer, size);
3310 }
3311 
3312 static int shmem_xattr_handler_set(const struct xattr_handler *handler,
3313                                    struct dentry *unused, struct inode *inode,
3314                                    const char *name, const void *value,
3315                                    size_t size, int flags)
3316 {
3317         struct shmem_inode_info *info = SHMEM_I(inode);
3318 
3319         name = xattr_full_name(handler, name);
3320         return simple_xattr_set(&info->xattrs, name, value, size, flags);
3321 }
3322 
3323 static const struct xattr_handler shmem_security_xattr_handler = {
3324         .prefix = XATTR_SECURITY_PREFIX,
3325         .get = shmem_xattr_handler_get,
3326         .set = shmem_xattr_handler_set,
3327 };
3328 
3329 static const struct xattr_handler shmem_trusted_xattr_handler = {
3330         .prefix = XATTR_TRUSTED_PREFIX,
3331         .get = shmem_xattr_handler_get,
3332         .set = shmem_xattr_handler_set,
3333 };
3334 
3335 static const struct xattr_handler *shmem_xattr_handlers[] = {
3336 #ifdef CONFIG_TMPFS_POSIX_ACL
3337         &posix_acl_access_xattr_handler,
3338         &posix_acl_default_xattr_handler,
3339 #endif
3340         &shmem_security_xattr_handler,
3341         &shmem_trusted_xattr_handler,
3342         NULL
3343 };
3344 
3345 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
3346 {
3347         struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3348         return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size);
3349 }
3350 #endif /* CONFIG_TMPFS_XATTR */
3351 
3352 static const struct inode_operations shmem_short_symlink_operations = {
3353         .get_link       = simple_get_link,
3354 #ifdef CONFIG_TMPFS_XATTR
3355         .listxattr      = shmem_listxattr,
3356 #endif
3357 };
3358 
3359 static const struct inode_operations shmem_symlink_inode_operations = {
3360         .get_link       = shmem_get_link,
3361 #ifdef CONFIG_TMPFS_XATTR
3362         .listxattr      = shmem_listxattr,
3363 #endif
3364 };
3365 
3366 static struct dentry *shmem_get_parent(struct dentry *child)
3367 {
3368         return ERR_PTR(-ESTALE);
3369 }
3370 
3371 static int shmem_match(struct inode *ino, void *vfh)
3372 {
3373         __u32 *fh = vfh;
3374         __u64 inum = fh[2];
3375         inum = (inum << 32) | fh[1];
3376         return ino->i_ino == inum && fh[0] == ino->i_generation;
3377 }
3378 
3379 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
3380                 struct fid *fid, int fh_len, int fh_type)
3381 {
3382         struct inode *inode;
3383         struct dentry *dentry = NULL;
3384         u64 inum;
3385 
3386         if (fh_len < 3)
3387                 return NULL;
3388 
3389         inum = fid->raw[2];
3390         inum = (inum << 32) | fid->raw[1];
3391 
3392         inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
3393                         shmem_match, fid->raw);
3394         if (inode) {
3395                 dentry = d_find_alias(inode);
3396                 iput(inode);
3397         }
3398 
3399         return dentry;
3400 }
3401 
3402 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
3403                                 struct inode *parent)
3404 {
3405         if (*len < 3) {
3406                 *len = 3;
3407                 return FILEID_INVALID;
3408         }
3409 
3410         if (inode_unhashed(inode)) {
3411                 /* Unfortunately insert_inode_hash is not idempotent,
3412                  * so as we hash inodes here rather than at creation
3413                  * time, we need a lock to ensure we only try
3414                  * to do it once
3415                  */
3416                 static DEFINE_SPINLOCK(lock);
3417                 spin_lock(&lock);
3418                 if (inode_unhashed(inode))
3419                         __insert_inode_hash(inode,
3420                                             inode->i_ino + inode->i_generation);
3421                 spin_unlock(&lock);
3422         }
3423 
3424         fh[0] = inode->i_generation;
3425         fh[1] = inode->i_ino;
3426         fh[2] = ((__u64)inode->i_ino) >> 32;
3427 
3428         *len = 3;
3429         return 1;
3430 }
3431 
3432 static const struct export_operations shmem_export_ops = {
3433         .get_parent     = shmem_get_parent,
3434         .encode_fh      = shmem_encode_fh,
3435         .fh_to_dentry   = shmem_fh_to_dentry,
3436 };
3437 
3438 static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
3439                                bool remount)
3440 {
3441         char *this_char, *value, *rest;
3442         struct mempolicy *mpol = NULL;
3443         uid_t uid;
3444         gid_t gid;
3445 
3446         while (options != NULL) {
3447                 this_char = options;
3448                 for (;;) {
3449                         /*
3450                          * NUL-terminate this option: unfortunately,
3451                          * mount options form a comma-separated list,
3452                          * but mpol's nodelist may also contain commas.
3453                          */
3454                         options = strchr(options, ',');
3455                         if (options == NULL)
3456                                 break;
3457                         options++;
3458                         if (!isdigit(*options)) {
3459                                 options[-1] = '\0';
3460                                 break;
3461                         }
3462                 }
3463                 if (!*this_char)
3464                         continue;
3465                 if ((value = strchr(this_char,'=')) != NULL) {
3466                         *value++ = 0;
3467                 } else {
3468                         pr_err("tmpfs: No value for mount option '%s'\n",
3469                                this_char);
3470                         goto error;
3471                 }
3472 
3473                 if (!strcmp(this_char,"size")) {
3474                         unsigned long long size;
3475                         size = memparse(value,&rest);
3476                         if (*rest == '%') {
3477                                 size <<= PAGE_SHIFT;
3478                                 size *= totalram_pages;
3479                                 do_div(size, 100);
3480                                 rest++;
3481                         }
3482                         if (*rest)
3483                                 goto bad_val;
3484                         sbinfo->max_blocks =
3485                                 DIV_ROUND_UP(size, PAGE_SIZE);
3486                 } else if (!strcmp(this_char,"nr_blocks")) {
3487                         sbinfo->max_blocks = memparse(value, &rest);
3488                         if (*rest)
3489                                 goto bad_val;
3490                 } else if (!strcmp(this_char,"nr_inodes")) {
3491                         sbinfo->max_inodes = memparse(value, &rest);
3492                         if (*rest)
3493                                 goto bad_val;
3494                 } else if (!strcmp(this_char,"mode")) {
3495                         if (remount)
3496                                 continue;
3497                         sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
3498                         if (*rest)
3499                                 goto bad_val;
3500                 } else if (!strcmp(this_char,"uid")) {
3501                         if (remount)
3502                                 continue;
3503                         uid = simple_strtoul(value, &rest, 0);
3504                         if (*rest)
3505                                 goto bad_val;
3506                         sbinfo->uid = make_kuid(current_user_ns(), uid);
3507                         if (!uid_valid(sbinfo->uid))
3508                                 goto bad_val;
3509                 } else if (!strcmp(this_char,"gid")) {
3510                         if (remount)
3511                                 continue;
3512                         gid = simple_strtoul(value, &rest, 0);
3513                         if (*rest)
3514                                 goto bad_val;
3515                         sbinfo->gid = make_kgid(current_user_ns(), gid);
3516                         if (!gid_valid(sbinfo->gid))
3517                                 goto bad_val;
3518 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
3519                 } else if (!strcmp(this_char, "huge")) {
3520                         int huge;
3521                         huge = shmem_parse_huge(value);
3522                         if (huge < 0)
3523                                 goto bad_val;
3524                         if (!has_transparent_hugepage() &&
3525                                         huge != SHMEM_HUGE_NEVER)
3526                                 goto bad_val;
3527                         sbinfo->huge = huge;
3528 #endif
3529 #ifdef CONFIG_NUMA
3530                 } else if (!strcmp(this_char,"mpol")) {
3531                         mpol_put(mpol);
3532                         mpol = NULL;
3533                         if (mpol_parse_str(value, &mpol))
3534                                 goto bad_val;
3535 #endif
3536                 } else {
3537                         pr_err("tmpfs: Bad mount option %s\n", this_char);
3538                         goto error;
3539                 }
3540         }
3541         sbinfo->mpol = mpol;
3542         return 0;
3543 
3544 bad_val:
3545         pr_err("tmpfs: Bad value '%s' for mount option '%s'\n",
3546                value, this_char);
3547 error:
3548         mpol_put(mpol);
3549         return 1;
3550 
3551 }
3552 
3553 static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
3554 {
3555         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
3556         struct shmem_sb_info config = *sbinfo;
3557         unsigned long inodes;
3558         int error = -EINVAL;
3559 
3560         config.mpol = NULL;
3561         if (shmem_parse_options(data, &config, true))
3562                 return error;
3563 
3564         spin_lock(&sbinfo->stat_lock);
3565         inodes = sbinfo->max_inodes - sbinfo->free_inodes;
3566         if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
3567                 goto out;
3568         if (config.max_inodes < inodes)
3569                 goto out;
3570         /*
3571          * Those tests disallow limited->unlimited while any are in use;
3572          * but we must separately disallow unlimited->limited, because
3573          * in that case we have no record of how much is already in use.
3574          */
3575         if (config.max_blocks && !sbinfo->max_blocks)
3576                 goto out;
3577         if (config.max_inodes && !sbinfo->max_inodes)
3578                 goto out;
3579 
3580         error = 0;
3581         sbinfo->huge = config.huge;
3582         sbinfo->max_blocks  = config.max_blocks;
3583         sbinfo->max_inodes  = config.max_inodes;
3584         sbinfo->free_inodes = config.max_inodes - inodes;
3585 
3586         /*
3587          * Preserve previous mempolicy unless mpol remount option was specified.
3588          */
3589         if (config.mpol) {
3590                 mpol_put(sbinfo->mpol);
3591                 sbinfo->mpol = config.mpol;     /* transfers initial ref */
3592         }
3593 out:
3594         spin_unlock(&sbinfo->stat_lock);
3595         return error;
3596 }
3597 
3598 static int shmem_show_options(struct seq_file *seq, struct dentry *root)
3599 {
3600         struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
3601 
3602         if (sbinfo->max_blocks != shmem_default_max_blocks())
3603                 seq_printf(seq, ",size=%luk",
3604                         sbinfo->max_blocks << (PAGE_SHIFT - 10));
3605         if (sbinfo->max_inodes != shmem_default_max_inodes())
3606                 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
3607         if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
3608                 seq_printf(seq, ",mode=%03ho", sbinfo->mode);
3609         if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
3610                 seq_printf(seq, ",uid=%u",
3611                                 from_kuid_munged(&init_user_ns, sbinfo->uid));
3612         if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
3613                 seq_printf(seq, ",gid=%u",
3614                                 from_kgid_munged(&init_user_ns, sbinfo->gid));
3615 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
3616         /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */
3617         if (sbinfo->huge)
3618                 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge));
3619 #endif
3620         shmem_show_mpol(seq, sbinfo->mpol);
3621         return 0;
3622 }
3623 
3624 #define MFD_NAME_PREFIX "memfd:"
3625 #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1)
3626 #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN)
3627 
3628 #define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING)
3629 
3630 SYSCALL_DEFINE2(memfd_create,
3631                 const char __user *, uname,
3632                 unsigned int, flags)
3633 {
3634         struct shmem_inode_info *info;
3635         struct file *file;
3636         int fd, error;
3637         char *name;
3638         long len;
3639 
3640         if (flags & ~(unsigned int)MFD_ALL_FLAGS)
3641                 return -EINVAL;
3642 
3643         /* length includes terminating zero */
3644         len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1);
3645         if (len <= 0)
3646                 return -EFAULT;
3647         if (len > MFD_NAME_MAX_LEN + 1)
3648                 return -EINVAL;
3649 
3650         name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_TEMPORARY);
3651         if (!name)
3652                 return -ENOMEM;
3653 
3654         strcpy(name, MFD_NAME_PREFIX);
3655         if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) {
3656                 error = -EFAULT;
3657                 goto err_name;
3658         }
3659 
3660         /* terminating-zero may have changed after strnlen_user() returned */
3661         if (name[len + MFD_NAME_PREFIX_LEN - 1]) {
3662                 error = -EFAULT;
3663                 goto err_name;
3664         }
3665 
3666         fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0);
3667         if (fd < 0) {
3668                 error = fd;
3669                 goto err_name;
3670         }
3671 
3672         file = shmem_file_setup(name, 0, VM_NORESERVE);
3673         if (IS_ERR(file)) {
3674                 error = PTR_ERR(file);
3675                 goto err_fd;
3676         }
3677         info = SHMEM_I(file_inode(file));
3678         file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE;
3679         file->f_flags |= O_RDWR | O_LARGEFILE;
3680         if (flags & MFD_ALLOW_SEALING)
3681                 info->seals &= ~F_SEAL_SEAL;
3682 
3683         fd_install(fd, file);
3684         kfree(name);
3685         return fd;
3686 
3687 err_fd:
3688         put_unused_fd(fd);
3689 err_name:
3690         kfree(name);
3691         return error;
3692 }
3693 
3694 #endif /* CONFIG_TMPFS */
3695 
3696 static void shmem_put_super(struct super_block *sb)
3697 {
3698         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
3699 
3700         percpu_counter_destroy(&sbinfo->used_blocks);
3701         mpol_put(sbinfo->mpol);
3702         kfree(sbinfo);
3703         sb->s_fs_info = NULL;
3704 }
3705 
3706 int shmem_fill_super(struct super_block *sb, void *data, int silent)
3707 {
3708         struct inode *inode;
3709         struct shmem_sb_info *sbinfo;
3710         int err = -ENOMEM;
3711 
3712         /* Round up to L1_CACHE_BYTES to resist false sharing */
3713         sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
3714                                 L1_CACHE_BYTES), GFP_KERNEL);
3715         if (!sbinfo)
3716                 return -ENOMEM;
3717 
3718         sbinfo->mode = S_IRWXUGO | S_ISVTX;
3719         sbinfo->uid = current_fsuid();
3720         sbinfo->gid = current_fsgid();
3721         sb->s_fs_info = sbinfo;
3722 
3723 #ifdef CONFIG_TMPFS
3724         /*
3725          * Per default we only allow half of the physical ram per
3726          * tmpfs instance, limiting inodes to one per page of lowmem;
3727          * but the internal instance is left unlimited.
3728          */
3729         if (!(sb->s_flags & MS_KERNMOUNT)) {
3730                 sbinfo->max_blocks = shmem_default_max_blocks();
3731                 sbinfo->max_inodes = shmem_default_max_inodes();
3732                 if (shmem_parse_options(data, sbinfo, false)) {
3733                         err = -EINVAL;
3734                         goto failed;
3735                 }
3736         } else {
3737                 sb->s_flags |= MS_NOUSER;
3738         }
3739         sb->s_export_op = &shmem_export_ops;
3740         sb->s_flags |= MS_NOSEC;
3741 #else
3742         sb->s_flags |= MS_NOUSER;
3743 #endif
3744 
3745         spin_lock_init(&sbinfo->stat_lock);
3746         if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
3747                 goto failed;
3748         sbinfo->free_inodes = sbinfo->max_inodes;
3749         spin_lock_init(&sbinfo->shrinklist_lock);
3750         INIT_LIST_HEAD(&sbinfo->shrinklist);
3751 
3752         sb->s_maxbytes = MAX_LFS_FILESIZE;
3753         sb->s_blocksize = PAGE_SIZE;
3754         sb->s_blocksize_bits = PAGE_SHIFT;
3755         sb->s_magic = TMPFS_MAGIC;
3756         sb->s_op = &shmem_ops;
3757         sb->s_time_gran = 1;
3758 #ifdef CONFIG_TMPFS_XATTR
3759         sb->s_xattr = shmem_xattr_handlers;
3760 #endif
3761 #ifdef CONFIG_TMPFS_POSIX_ACL
3762         sb->s_flags |= MS_POSIXACL;
3763 #endif
3764 
3765         inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
3766         if (!inode)
3767                 goto failed;
3768         inode->i_uid = sbinfo->uid;
3769         inode->i_gid = sbinfo->gid;
3770         sb->s_root = d_make_root(inode);
3771         if (!sb->s_root)
3772                 goto failed;
3773         return 0;
3774 
3775 failed:
3776         shmem_put_super(sb);
3777         return err;
3778 }
3779 
3780 static struct kmem_cache *shmem_inode_cachep;
3781 
3782 static struct inode *shmem_alloc_inode(struct super_block *sb)
3783 {
3784         struct shmem_inode_info *info;
3785         info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
3786         if (!info)
3787                 return NULL;
3788         return &info->vfs_inode;
3789 }
3790 
3791 static void shmem_destroy_callback(struct rcu_head *head)
3792 {
3793         struct inode *inode = container_of(head, struct inode, i_rcu);
3794         if (S_ISLNK(inode->i_mode))
3795                 kfree(inode->i_link);
3796         kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
3797 }
3798 
3799 static void shmem_destroy_inode(struct inode *inode)
3800 {
3801         if (S_ISREG(inode->i_mode))
3802                 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
3803         call_rcu(&inode->i_rcu, shmem_destroy_callback);
3804 }
3805 
3806 static void shmem_init_inode(void *foo)
3807 {
3808         struct shmem_inode_info *info = foo;
3809         inode_init_once(&info->vfs_inode);
3810 }
3811 
3812 static int shmem_init_inodecache(void)
3813 {
3814         shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
3815                                 sizeof(struct shmem_inode_info),
3816                                 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode);
3817         return 0;
3818 }
3819 
3820 static void shmem_destroy_inodecache(void)
3821 {
3822         kmem_cache_destroy(shmem_inode_cachep);
3823 }
3824 
3825 static const struct address_space_operations shmem_aops = {
3826         .writepage      = shmem_writepage,
3827         .set_page_dirty = __set_page_dirty_no_writeback,
3828 #ifdef CONFIG_TMPFS
3829         .write_begin    = shmem_write_begin,
3830         .write_end      = shmem_write_end,
3831 #endif
3832 #ifdef CONFIG_MIGRATION
3833         .migratepage    = migrate_page,
3834 #endif
3835         .error_remove_page = generic_error_remove_page,
3836 };
3837 
3838 static const struct file_operations shmem_file_operations = {
3839         .mmap           = shmem_mmap,
3840         .get_unmapped_area = shmem_get_unmapped_area,
3841 #ifdef CONFIG_TMPFS
3842         .llseek         = shmem_file_llseek,
3843         .read_iter      = shmem_file_read_iter,
3844         .write_iter     = generic_file_write_iter,
3845         .fsync          = noop_fsync,
3846         .splice_read    = generic_file_splice_read,
3847         .splice_write   = iter_file_splice_write,
3848         .fallocate      = shmem_fallocate,
3849 #endif
3850 };
3851 
3852 static const struct inode_operations shmem_inode_operations = {
3853         .getattr        = shmem_getattr,
3854         .setattr        = shmem_setattr,
3855 #ifdef CONFIG_TMPFS_XATTR
3856         .listxattr      = shmem_listxattr,
3857         .set_acl        = simple_set_acl,
3858 #endif
3859 };
3860 
3861 static const struct inode_operations shmem_dir_inode_operations = {
3862 #ifdef CONFIG_TMPFS
3863         .create         = shmem_create,
3864         .lookup         = simple_lookup,
3865         .link           = shmem_link,
3866         .unlink         = shmem_unlink,
3867         .symlink        = shmem_symlink,
3868         .mkdir          = shmem_mkdir,
3869         .rmdir          = shmem_rmdir,
3870         .mknod          = shmem_mknod,
3871         .rename         = shmem_rename2,
3872         .tmpfile        = shmem_tmpfile,
3873 #endif
3874 #ifdef CONFIG_TMPFS_XATTR
3875         .listxattr      = shmem_listxattr,
3876 #endif
3877 #ifdef CONFIG_TMPFS_POSIX_ACL
3878         .setattr        = shmem_setattr,
3879         .set_acl        = simple_set_acl,
3880 #endif
3881 };
3882 
3883 static const struct inode_operations shmem_special_inode_operations = {
3884 #ifdef CONFIG_TMPFS_XATTR
3885         .listxattr      = shmem_listxattr,
3886 #endif
3887 #ifdef CONFIG_TMPFS_POSIX_ACL
3888         .setattr        = shmem_setattr,
3889         .set_acl        = simple_set_acl,
3890 #endif
3891 };
3892 
3893 static const struct super_operations shmem_ops = {
3894         .alloc_inode    = shmem_alloc_inode,
3895         .destroy_inode  = shmem_destroy_inode,
3896 #ifdef CONFIG_TMPFS
3897         .statfs         = shmem_statfs,
3898         .remount_fs     = shmem_remount_fs,
3899         .show_options   = shmem_show_options,
3900 #endif
3901         .evict_inode    = shmem_evict_inode,
3902         .drop_inode     = generic_delete_inode,
3903         .put_super      = shmem_put_super,
3904 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
3905         .nr_cached_objects      = shmem_unused_huge_count,
3906         .free_cached_objects    = shmem_unused_huge_scan,
3907 #endif
3908 };
3909 
3910 static const struct vm_operations_struct shmem_vm_ops = {
3911         .fault          = shmem_fault,
3912         .map_pages      = filemap_map_pages,
3913 #ifdef CONFIG_NUMA
3914         .set_policy     = shmem_set_policy,
3915         .get_policy     = shmem_get_policy,
3916 #endif
3917 };
3918 
3919 static struct dentry *shmem_mount(struct file_system_type *fs_type,
3920         int flags, const char *dev_name, void *data)
3921 {
3922         return mount_nodev(fs_type, flags, data, shmem_fill_super);
3923 }
3924 
3925 static struct file_system_type shmem_fs_type = {
3926         .owner          = THIS_MODULE,
3927         .name           = "tmpfs",
3928         .mount          = shmem_mount,
3929         .kill_sb        = kill_litter_super,
3930         .fs_flags       = FS_USERNS_MOUNT,
3931 };
3932 
3933 int __init shmem_init(void)
3934 {
3935         int error;
3936 
3937         /* If rootfs called this, don't re-init */
3938         if (shmem_inode_cachep)
3939                 return 0;
3940 
3941         error = shmem_init_inodecache();
3942         if (error)
3943                 goto out3;
3944 
3945         error = register_filesystem(&shmem_fs_type);
3946         if (error) {
3947                 pr_err("Could not register tmpfs\n");
3948                 goto out2;
3949         }
3950 
3951         shm_mnt = kern_mount(&shmem_fs_type);
3952         if (IS_ERR(shm_mnt)) {
3953                 error = PTR_ERR(shm_mnt);
3954                 pr_err("Could not kern_mount tmpfs\n");
3955                 goto out1;
3956         }
3957 
3958 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
3959         if (has_transparent_hugepage() && shmem_huge < SHMEM_HUGE_DENY)
3960                 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
3961         else
3962                 shmem_huge = 0; /* just in case it was patched */
3963 #endif
3964         return 0;
3965 
3966 out1:
3967         unregister_filesystem(&shmem_fs_type);
3968 out2:
3969         shmem_destroy_inodecache();
3970 out3:
3971         shm_mnt = ERR_PTR(error);
3972         return error;
3973 }
3974 
3975 #if defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && defined(CONFIG_SYSFS)
3976 static ssize_t shmem_enabled_show(struct kobject *kobj,
3977                 struct kobj_attribute *attr, char *buf)
3978 {
3979         int values[] = {
3980                 SHMEM_HUGE_ALWAYS,
3981                 SHMEM_HUGE_WITHIN_SIZE,
3982                 SHMEM_HUGE_ADVISE,
3983                 SHMEM_HUGE_NEVER,
3984                 SHMEM_HUGE_DENY,
3985                 SHMEM_HUGE_FORCE,
3986         };
3987         int i, count;
3988 
3989         for (i = 0, count = 0; i < ARRAY_SIZE(values); i++) {
3990                 const char *fmt = shmem_huge == values[i] ? "[%s] " : "%s ";
3991 
3992                 count += sprintf(buf + count, fmt,
3993                                 shmem_format_huge(values[i]));
3994         }
3995         buf[count - 1] = '\n';
3996         return count;
3997 }
3998 
3999 static ssize_t shmem_enabled_store(struct kobject *kobj,
4000                 struct kobj_attribute *attr, const char *buf, size_t count)
4001 {
4002         char tmp[16];
4003         int huge;
4004 
4005         if (count + 1 > sizeof(tmp))
4006                 return -EINVAL;
4007         memcpy(tmp, buf, count);
4008         tmp[count] = '\0';
4009         if (count && tmp[count - 1] == '\n')
4010                 tmp[count - 1] = '\0';
4011 
4012         huge = shmem_parse_huge(tmp);
4013         if (huge == -EINVAL)
4014                 return -EINVAL;
4015         if (!has_transparent_hugepage() &&
4016                         huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY)
4017                 return -EINVAL;
4018 
4019         shmem_huge = huge;
4020         if (shmem_huge < SHMEM_HUGE_DENY)
4021                 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
4022         return count;
4023 }
4024 
4025 struct kobj_attribute shmem_enabled_attr =
4026         __ATTR(shmem_enabled, 0644, shmem_enabled_show, shmem_enabled_store);
4027 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE && CONFIG_SYSFS */
4028 
4029 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
4030 bool shmem_huge_enabled(struct vm_area_struct *vma)
4031 {
4032         struct inode *inode = file_inode(vma->vm_file);
4033         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
4034         loff_t i_size;
4035         pgoff_t off;
4036 
4037         if (shmem_huge == SHMEM_HUGE_FORCE)
4038                 return true;
4039         if (shmem_huge == SHMEM_HUGE_DENY)
4040                 return false;
4041         switch (sbinfo->huge) {
4042                 case SHMEM_HUGE_NEVER:
4043                         return false;
4044                 case SHMEM_HUGE_ALWAYS:
4045                         return true;
4046                 case SHMEM_HUGE_WITHIN_SIZE:
4047                         off = round_up(vma->vm_pgoff, HPAGE_PMD_NR);
4048                         i_size = round_up(i_size_read(inode), PAGE_SIZE);
4049                         if (i_size >= HPAGE_PMD_SIZE &&
4050                                         i_size >> PAGE_SHIFT >= off)
4051                                 return true;
4052                 case SHMEM_HUGE_ADVISE:
4053                         /* TODO: implement fadvise() hints */
4054                         return (vma->vm_flags & VM_HUGEPAGE);
4055                 default:
4056                         VM_BUG_ON(1);
4057                         return false;
4058         }
4059 }
4060 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */
4061 
4062 #else /* !CONFIG_SHMEM */
4063 
4064 /*
4065  * tiny-shmem: simple shmemfs and tmpfs using ramfs code
4066  *
4067  * This is intended for small system where the benefits of the full
4068  * shmem code (swap-backed and resource-limited) are outweighed by
4069  * their complexity. On systems without swap this code should be
4070  * effectively equivalent, but much lighter weight.
4071  */
4072 
4073 static struct file_system_type shmem_fs_type = {
4074         .name           = "tmpfs",
4075         .mount          = ramfs_mount,
4076         .kill_sb        = kill_litter_super,
4077         .fs_flags       = FS_USERNS_MOUNT,
4078 };
4079 
4080 int __init shmem_init(void)
4081 {
4082         BUG_ON(register_filesystem(&shmem_fs_type) != 0);
4083 
4084         shm_mnt = kern_mount(&shmem_fs_type);
4085         BUG_ON(IS_ERR(shm_mnt));
4086 
4087         return 0;
4088 }
4089 
4090 int shmem_unuse(swp_entry_t swap, struct page *page)
4091 {
4092         return 0;
4093 }
4094 
4095 int shmem_lock(struct file *file, int lock, struct user_struct *user)
4096 {
4097         return 0;
4098 }
4099 
4100 void shmem_unlock_mapping(struct address_space *mapping)
4101 {
4102 }
4103 
4104 #ifdef CONFIG_MMU
4105 unsigned long shmem_get_unmapped_area(struct file *file,
4106                                       unsigned long addr, unsigned long len,
4107                                       unsigned long pgoff, unsigned long flags)
4108 {
4109         return current->mm->get_unmapped_area(file, addr, len, pgoff, flags);
4110 }
4111 #endif
4112 
4113 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
4114 {
4115         truncate_inode_pages_range(inode->i_mapping, lstart, lend);
4116 }
4117 EXPORT_SYMBOL_GPL(shmem_truncate_range);
4118 
4119 #define shmem_vm_ops                            generic_file_vm_ops
4120 #define shmem_file_operations                   ramfs_file_operations
4121 #define shmem_get_inode(sb, dir, mode, dev, flags)      ramfs_get_inode(sb, dir, mode, dev)
4122 #define shmem_acct_size(flags, size)            0
4123 #define shmem_unacct_size(flags, size)          do {} while (0)
4124 
4125 #endif /* CONFIG_SHMEM */
4126 
4127 /* common code */
4128 
4129 static const struct dentry_operations anon_ops = {
4130         .d_dname = simple_dname
4131 };
4132 
4133 static struct file *__shmem_file_setup(const char *name, loff_t size,
4134                                        unsigned long flags, unsigned int i_flags)
4135 {
4136         struct file *res;
4137         struct inode *inode;
4138         struct path path;
4139         struct super_block *sb;
4140         struct qstr this;
4141 
4142         if (IS_ERR(shm_mnt))
4143                 return ERR_CAST(shm_mnt);
4144 
4145         if (size < 0 || size > MAX_LFS_FILESIZE)
4146                 return ERR_PTR(-EINVAL);
4147 
4148         if (shmem_acct_size(flags, size))
4149                 return ERR_PTR(-ENOMEM);
4150 
4151         res = ERR_PTR(-ENOMEM);
4152         this.name = name;
4153         this.len = strlen(name);
4154         this.hash = 0; /* will go */
4155         sb = shm_mnt->mnt_sb;
4156         path.mnt = mntget(shm_mnt);
4157         path.dentry = d_alloc_pseudo(sb, &this);
4158         if (!path.dentry)
4159                 goto put_memory;
4160         d_set_d_op(path.dentry, &anon_ops);
4161 
4162         res = ERR_PTR(-ENOSPC);
4163         inode = shmem_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0, flags);
4164         if (!inode)
4165                 goto put_memory;
4166 
4167         inode->i_flags |= i_flags;
4168         d_instantiate(path.dentry, inode);
4169         inode->i_size = size;
4170         clear_nlink(inode);     /* It is unlinked */
4171         res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size));
4172         if (IS_ERR(res))
4173                 goto put_path;
4174 
4175         res = alloc_file(&path, FMODE_WRITE | FMODE_READ,
4176                   &shmem_file_operations);
4177         if (IS_ERR(res))
4178                 goto put_path;
4179 
4180         return res;
4181 
4182 put_memory:
4183         shmem_unacct_size(flags, size);
4184 put_path:
4185         path_put(&path);
4186         return res;
4187 }
4188 
4189 /**
4190  * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be
4191  *      kernel internal.  There will be NO LSM permission checks against the
4192  *      underlying inode.  So users of this interface must do LSM checks at a
4193  *      higher layer.  The users are the big_key and shm implementations.  LSM
4194  *      checks are provided at the key or shm level rather than the inode.
4195  * @name: name for dentry (to be seen in /proc/<pid>/maps
4196  * @size: size to be set for the file
4197  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4198  */
4199 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
4200 {
4201         return __shmem_file_setup(name, size, flags, S_PRIVATE);
4202 }
4203 
4204 /**
4205  * shmem_file_setup - get an unlinked file living in tmpfs
4206  * @name: name for dentry (to be seen in /proc/<pid>/maps
4207  * @size: size to be set for the file
4208  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4209  */
4210 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
4211 {
4212         return __shmem_file_setup(name, size, flags, 0);
4213 }
4214 EXPORT_SYMBOL_GPL(shmem_file_setup);
4215 
4216 /**
4217  * shmem_zero_setup - setup a shared anonymous mapping
4218  * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
4219  */
4220 int shmem_zero_setup(struct vm_area_struct *vma)
4221 {
4222         struct file *file;
4223         loff_t size = vma->vm_end - vma->vm_start;
4224 
4225         /*
4226          * Cloning a new file under mmap_sem leads to a lock ordering conflict
4227          * between XFS directory reading and selinux: since this file is only
4228          * accessible to the user through its mapping, use S_PRIVATE flag to
4229          * bypass file security, in the same way as shmem_kernel_file_setup().
4230          */
4231         file = __shmem_file_setup("dev/zero", size, vma->vm_flags, S_PRIVATE);
4232         if (IS_ERR(file))
4233                 return PTR_ERR(file);
4234 
4235         if (vma->vm_file)
4236                 fput(vma->vm_file);
4237         vma->vm_file = file;
4238         vma->vm_ops = &shmem_vm_ops;
4239 
4240         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) &&
4241                         ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
4242                         (vma->vm_end & HPAGE_PMD_MASK)) {
4243                 khugepaged_enter(vma, vma->vm_flags);
4244         }
4245 
4246         return 0;
4247 }
4248 
4249 /**
4250  * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
4251  * @mapping:    the page's address_space
4252  * @index:      the page index
4253  * @gfp:        the page allocator flags to use if allocating
4254  *
4255  * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
4256  * with any new page allocations done using the specified allocation flags.
4257  * But read_cache_page_gfp() uses the ->readpage() method: which does not
4258  * suit tmpfs, since it may have pages in swapcache, and needs to find those
4259  * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
4260  *
4261  * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
4262  * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
4263  */
4264 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
4265                                          pgoff_t index, gfp_t gfp)
4266 {
4267 #ifdef CONFIG_SHMEM
4268         struct inode *inode = mapping->host;
4269         struct page *page;
4270         int error;
4271 
4272         BUG_ON(mapping->a_ops != &shmem_aops);
4273         error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE,
4274                                   gfp, NULL, NULL, NULL);
4275         if (error)
4276                 page = ERR_PTR(error);
4277         else
4278                 unlock_page(page);
4279         return page;
4280 #else
4281         /*
4282          * The tiny !SHMEM case uses ramfs without swap
4283          */
4284         return read_cache_page_gfp(mapping, index, gfp);
4285 #endif
4286 }
4287 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);
4288 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp